Skip to content

Commit a910f84

Browse files
ShareASmileShareASmile
authored
Merge pull request #65 from Stypox/lockup-podcast
[YouTube] Support lockup content type podcast
2 parents 2e45729 + 35cc5a7 commit a910f84

52 files changed

Lines changed: 5100 additions & 2391 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎build.gradle‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -40,12 +40,20 @@ dependencies {
4040
}
4141

4242
subprojects {
43-
task sourcesJar(type: Jar, dependsOn: classes) {
43+
tasks.register('sourcesJar', Jar) {
44+
dependsOn classes
4445
archiveClassifier.set('sources')
4546
from sourceSets.main.allSource
4647
}
4748

48-
tasks.withType(Test) {
49+
// Protobuf files would uselessly end up in the JAR otherwise, see
50+
// https://github.com/google/protobuf-gradle-plugin/issues/390
51+
tasks.withType(Jar).configureEach {
52+
exclude '**/*.proto'
53+
includeEmptyDirs false
54+
}
55+
56+
tasks.withType(Test).configureEach {
4957
testLogging {
5058
events "skipped", "failed"
5159
showStandardStreams = true
@@ -59,8 +67,8 @@ subprojects {
5967
}
6068

6169
// https://discuss.gradle.org/t/best-approach-gradle-multi-module-project-generate-just-one-global-javadoc/18657/21
62-
task aggregatedJavadocs(type: Javadoc, group: 'Documentation') {
63-
destinationDir = file("$buildDir/docs/javadoc")
70+
tasks.register('aggregatedJavadocs', Javadoc) {
71+
destinationDir = file("${layout.buildDirectory}/docs/javadoc")
6472
title = "$project.name $version"
6573
// options.memberLevel = JavadocMemberLevel.PRIVATE
6674
options.links 'https://docs.oracle.com/javase/8/docs/api/'

‎extractor/build.gradle‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
plugins {
2-
id 'checkstyle'
2+
id "checkstyle"
3+
id "com.google.protobuf" version "0.9.5"
34
}
45

56
test {
@@ -18,12 +19,16 @@ checkstyle {
1819
toolVersion checkstyleVersion
1920
}
2021

22+
// Exclude Protobuf generated files from Checkstyle
23+
checkstyleMain.exclude("org/schabi/newpipe/extractor/services/youtube/protos")
24+
2125
checkstyleTest {
2226
enabled false // do not checkstyle test files
2327
}
2428

2529
ext {
2630
rhinoVersion = '1.8.0'
31+
protobufVersion = '4.30.2'
2732
}
2833

2934
dependencies {
@@ -32,6 +37,7 @@ dependencies {
3237
implementation "com.github.TeamNewPipe:nanojson:$nanojsonVersion"
3338
implementation 'org.jsoup:jsoup:1.21.1'
3439
implementation "com.google.code.findbugs:jsr305:$jsr305Version"
40+
implementation "com.google.protobuf:protobuf-javalite:$protobufVersion"
3541

3642
implementation "org.mozilla:rhino:$rhinoVersion"
3743
implementation "org.mozilla:rhino-engine:$rhinoVersion"
@@ -47,3 +53,19 @@ dependencies {
4753
testImplementation "com.squareup.okhttp3:okhttp:4.12.0"
4854
testImplementation 'com.google.code.gson:gson:2.13.1'
4955
}
56+
57+
protobuf {
58+
protoc {
59+
artifact = "com.google.protobuf:protoc:$protobufVersion"
60+
}
61+
62+
generateProtoTasks {
63+
all().configureEach { task ->
64+
task.builtins {
65+
java {
66+
option "lite"
67+
}
68+
}
69+
}
70+
}
71+
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java‎

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1017,9 +1017,9 @@ public static String getValidJsonResponseBody(@Nonnull final Response response)
10171017
return responseBody;
10181018
}
10191019

1020-
public static JsonObject getJsonPostResponse(final String endpoint,
1020+
public static JsonObject getJsonPostResponse(@Nonnull final String endpoint,
10211021
final byte[] body,
1022-
final Localization localization)
1022+
@Nonnull final Localization localization)
10231023
throws IOException, ExtractionException {
10241024
final var headers = getYouTubeHeaders();
10251025

@@ -1028,6 +1028,26 @@ public static JsonObject getJsonPostResponse(final String endpoint,
10281028
+ DISABLE_PRETTY_PRINT_PARAMETER, headers, body, localization)));
10291029
}
10301030

1031+
public static JsonObject getJsonPostResponse(@Nonnull final String endpoint,
1032+
@Nonnull final List<String> queryParameters,
1033+
final byte[] body,
1034+
@Nonnull final Localization localization)
1035+
throws IOException, ExtractionException {
1036+
final var headers = getYouTubeHeaders();
1037+
1038+
final String queryParametersString;
1039+
if (queryParameters.isEmpty()) {
1040+
queryParametersString = "?" + DISABLE_PRETTY_PRINT_PARAMETER;
1041+
} else {
1042+
queryParametersString = "?" + String.join("&", queryParameters)
1043+
+ "&" + DISABLE_PRETTY_PRINT_PARAMETER;
1044+
}
1045+
1046+
return JsonUtils.toJsonObject(getValidJsonResponseBody(
1047+
getDownloader().postWithContentTypeJson(YOUTUBEI_V1_URL + endpoint
1048+
+ queryParametersString, headers, body, localization)));
1049+
}
1050+
10311051
@Nonnull
10321052
public static JsonBuilder<JsonObject> prepareDesktopJsonBuilder(
10331053
@Nonnull final Localization localization,

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java‎

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -313,9 +313,14 @@ private Optional<JsonObject> collectItem(@Nonnull final MultiInfoItemsCollector
313313
.getArray("items"), channelVerifiedStatus, channelName, channelUrl);
314314
} else if (item.has("lockupViewModel")) {
315315
final JsonObject lockupViewModel = item.getObject("lockupViewModel");
316-
if ("LOCKUP_CONTENT_TYPE_PLAYLIST".equals(lockupViewModel.getString("contentType"))) {
316+
final String contentType = lockupViewModel.getString("contentType");
317+
if ("LOCKUP_CONTENT_TYPE_PLAYLIST".equals(contentType)
318+
|| "LOCKUP_CONTENT_TYPE_PODCAST".equals(contentType)) {
317319
commitPlaylistLockup(collector, lockupViewModel, channelVerifiedStatus,
318320
channelName, channelUrl);
321+
} else if ("LOCKUP_CONTENT_TYPE_VIDEO".equals(contentType)) {
322+
commitVideoLockup(collector, timeAgoParser, lockupViewModel, channelVerifiedStatus,
323+
channelName, channelUrl);
319324
}
320325
} else if (item.has("continuationItemRenderer")) {
321326
return Optional.ofNullable(item.getObject("continuationItemRenderer"));
@@ -372,6 +377,31 @@ public boolean isUploaderVerified() {
372377
});
373378
}
374379

380+
private static void commitVideoLockup(@Nonnull final MultiInfoItemsCollector collector,
381+
@Nonnull final TimeAgoParser timeAgoParser,
382+
@Nonnull final JsonObject lockupViewModel,
383+
@Nonnull final VerifiedStatus channelVerifiedStatus,
384+
@Nullable final String channelName,
385+
@Nullable final String channelUrl) {
386+
collector.commit(
387+
new YoutubeStreamInfoItemLockupExtractor(lockupViewModel, timeAgoParser) {
388+
@Override
389+
public String getUploaderName() throws ParsingException {
390+
return isNullOrEmpty(channelName) ? super.getUploaderName() : channelName;
391+
}
392+
393+
@Override
394+
public String getUploaderUrl() throws ParsingException {
395+
return isNullOrEmpty(channelUrl) ? super.getUploaderName() : channelUrl;
396+
}
397+
398+
@Override
399+
public boolean isUploaderVerified() {
400+
return channelVerifiedStatus == VerifiedStatus.VERIFIED;
401+
}
402+
});
403+
}
404+
375405
private void commitPlaylistLockup(@Nonnull final MultiInfoItemsCollector collector,
376406
@Nonnull final JsonObject playlistLockupViewModel,
377407
@Nonnull final VerifiedStatus channelVerifiedStatus,

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java‎

Lines changed: 47 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@
88
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray;
99
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
1010
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
11+
import static org.schabi.newpipe.extractor.services.youtube.protos.playlist.PlaylistProtobufContinuation.ContinuationParams;
12+
import static org.schabi.newpipe.extractor.services.youtube.protos.playlist.PlaylistProtobufContinuation.PlaylistContinuation;
1113
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
1214

1315
import com.grack.nanojson.JsonArray;
@@ -33,6 +35,7 @@
3335

3436
import java.io.IOException;
3537
import java.nio.charset.StandardCharsets;
38+
import java.util.Base64;
3639
import java.util.List;
3740

3841
import javax.annotation.Nonnull;
@@ -41,14 +44,17 @@
4144
public class YoutubePlaylistExtractor extends PlaylistExtractor {
4245
// Names of some objects in JSON response frequently used in this class
4346
private static final String PLAYLIST_VIDEO_RENDERER = "playlistVideoRenderer";
44-
private static final String PLAYLIST_VIDEO_LIST_RENDERER = "playlistVideoListRenderer";
45-
private static final String RICH_GRID_RENDERER = "richGridRenderer";
4647
private static final String RICH_ITEM_RENDERER = "richItemRenderer";
4748
private static final String REEL_ITEM_RENDERER = "reelItemRenderer";
4849
private static final String SIDEBAR = "sidebar";
50+
private static final String HEADER = "header";
4951
private static final String VIDEO_OWNER_RENDERER = "videoOwnerRenderer";
52+
private static final String MICROFORMAT = "microformat";
53+
// Continuation properties requesting first page and showing unavailable videos
54+
private static final String PLAYLIST_CONTINUATION_PROPERTIES_BASE64 = "CADCBgIIAA%3D%3D";
5055

51-
private JsonObject browseResponse;
56+
private JsonObject browseMetadataResponse;
57+
private JsonObject initialBrowseContinuationResponse;
5258

5359
private JsonObject playlistInfo;
5460
private JsonObject uploaderInfo;
@@ -64,17 +70,40 @@ public YoutubePlaylistExtractor(final StreamingService service,
6470
@Override
6571
public void onFetchPage(@Nonnull final Downloader downloader) throws IOException,
6672
ExtractionException {
73+
final String playlistId = getId();
74+
6775
final Localization localization = getExtractorLocalization();
6876
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(localization,
6977
getExtractorContentCountry())
70-
.value("browseId", "VL" + getId())
78+
.value("browseId", "VL" + playlistId)
7179
.value("params", "wgYCCAA%3D") // Show unavailable videos
7280
.done())
7381
.getBytes(StandardCharsets.UTF_8);
7482

75-
browseResponse = getJsonPostResponse("browse", body, localization);
76-
YoutubeParsingHelper.defaultAlertsCheck(browseResponse);
83+
browseMetadataResponse = getJsonPostResponse("browse",
84+
List.of("$fields=" + SIDEBAR + "," + HEADER + "," + MICROFORMAT + ",alerts"),
85+
body,
86+
localization);
87+
88+
YoutubeParsingHelper.defaultAlertsCheck(browseMetadataResponse);
7789
isNewPlaylistInterface = checkIfResponseIsNewPlaylistInterface();
90+
91+
final PlaylistContinuation playlistContinuation = PlaylistContinuation.newBuilder()
92+
.setParameters(ContinuationParams.newBuilder()
93+
.setBrowseId("VL" + playlistId)
94+
.setPlaylistId(playlistId)
95+
.setContinuationProperties(PLAYLIST_CONTINUATION_PROPERTIES_BASE64)
96+
.build())
97+
.build();
98+
99+
initialBrowseContinuationResponse = getJsonPostResponse("browse",
100+
JsonWriter.string(prepareDesktopJsonBuilder(localization,
101+
getExtractorContentCountry())
102+
.value("continuation", Utils.encodeUrlUtf8(Base64.getUrlEncoder()
103+
.encodeToString(playlistContinuation.toByteArray())))
104+
.done())
105+
.getBytes(StandardCharsets.UTF_8),
106+
localization);
78107
}
79108

80109
/**
@@ -93,13 +122,13 @@ public void onFetchPage(@Nonnull final Downloader downloader) throws IOException
93122
*/
94123
private boolean checkIfResponseIsNewPlaylistInterface() {
95124
// The "old" playlist UI can be also returned with the new one
96-
return browseResponse.has("header") && !browseResponse.has(SIDEBAR);
125+
return browseMetadataResponse.has(HEADER) && !browseMetadataResponse.has(SIDEBAR);
97126
}
98127

99128
@Nonnull
100129
private JsonObject getUploaderInfo() throws ParsingException {
101130
if (uploaderInfo == null) {
102-
uploaderInfo = browseResponse.getObject(SIDEBAR)
131+
uploaderInfo = browseMetadataResponse.getObject(SIDEBAR)
103132
.getObject("playlistSidebarRenderer")
104133
.getArray("items")
105134
.stream()
@@ -121,7 +150,7 @@ private JsonObject getUploaderInfo() throws ParsingException {
121150
@Nonnull
122151
private JsonObject getPlaylistInfo() throws ParsingException {
123152
if (playlistInfo == null) {
124-
playlistInfo = browseResponse.getObject(SIDEBAR)
153+
playlistInfo = browseMetadataResponse.getObject(SIDEBAR)
125154
.getObject("playlistSidebarRenderer")
126155
.getArray("items")
127156
.stream()
@@ -139,7 +168,7 @@ private JsonObject getPlaylistInfo() throws ParsingException {
139168
@Nonnull
140169
private JsonObject getPlaylistHeader() {
141170
if (playlistHeader == null) {
142-
playlistHeader = browseResponse.getObject("header")
171+
playlistHeader = browseMetadataResponse.getObject(HEADER)
143172
.getObject("playlistHeaderRenderer");
144173
}
145174

@@ -154,7 +183,7 @@ public String getName() throws ParsingException {
154183
return name;
155184
}
156185

157-
return browseResponse.getObject("microformat")
186+
return browseMetadataResponse.getObject(MICROFORMAT)
158187
.getObject("microformatDataRenderer")
159188
.getString("title");
160189
}
@@ -180,7 +209,7 @@ public List<Image> getThumbnails() throws ParsingException {
180209
}
181210

182211
// This data structure is returned in both layouts
183-
final JsonArray microFormatThumbnailsArray = browseResponse.getObject("microformat")
212+
final JsonArray microFormatThumbnailsArray = browseMetadataResponse.getObject(MICROFORMAT)
184213
.getObject("microformatDataRenderer")
185214
.getObject("thumbnail")
186215
.getArray("thumbnails");
@@ -302,45 +331,16 @@ public Description getDescription() throws ParsingException {
302331
@Override
303332
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
304333
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
305-
Page nextPage = null;
306334

307-
final JsonArray contents = browseResponse.getObject("contents")
308-
.getObject("twoColumnBrowseResultsRenderer")
309-
.getArray("tabs")
335+
final JsonArray initialItems = initialBrowseContinuationResponse
336+
.getArray("onResponseReceivedActions")
310337
.getObject(0)
311-
.getObject("tabRenderer")
312-
.getObject("content")
313-
.getObject("sectionListRenderer")
314-
.getArray("contents");
315-
316-
final JsonObject videoPlaylistObject = contents.stream()
317-
.filter(JsonObject.class::isInstance)
318-
.map(JsonObject.class::cast)
319-
.map(content -> content.getObject("itemSectionRenderer")
320-
.getArray("contents")
321-
.getObject(0))
322-
.filter(content -> content.has(PLAYLIST_VIDEO_LIST_RENDERER)
323-
|| content.has(RICH_GRID_RENDERER))
324-
.findFirst()
325-
.orElse(null);
326-
327-
if (videoPlaylistObject != null) {
328-
final JsonObject renderer;
329-
if (videoPlaylistObject.has(PLAYLIST_VIDEO_LIST_RENDERER)) {
330-
renderer = videoPlaylistObject.getObject(PLAYLIST_VIDEO_LIST_RENDERER);
331-
} else if (videoPlaylistObject.has(RICH_GRID_RENDERER)) {
332-
renderer = videoPlaylistObject.getObject(RICH_GRID_RENDERER);
333-
} else {
334-
return new InfoItemsPage<>(collector, null);
335-
}
336-
337-
final JsonArray videosArray = renderer.getArray("contents");
338-
collectStreamsFrom(collector, videosArray);
338+
.getObject("reloadContinuationItemsCommand")
339+
.getArray("continuationItems");
339340

340-
nextPage = getNextPageFrom(videosArray);
341-
}
341+
collectStreamsFrom(collector, initialItems);
342342

343-
return new InfoItemsPage<>(collector, nextPage);
343+
return new InfoItemsPage<>(collector, getNextPageFrom(initialItems));
344344
}
345345

346346
@Override

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -247,10 +247,14 @@ private void collectStreamsFrom(final MultiInfoItemsCollector collector,
247247
item.getObject("showRenderer")));
248248
} else if (item.has("lockupViewModel")) {
249249
final JsonObject lockupViewModel = item.getObject("lockupViewModel");
250-
if ("LOCKUP_CONTENT_TYPE_PLAYLIST".equals(
251-
lockupViewModel.getString("contentType"))) {
250+
final String contentType = lockupViewModel.getString("contentType");
251+
if ("LOCKUP_CONTENT_TYPE_PLAYLIST".equals(contentType)
252+
|| "LOCKUP_CONTENT_TYPE_PODCAST".equals(contentType)) {
252253
collector.commit(
253254
new YoutubeMixOrPlaylistLockupInfoItemExtractor(lockupViewModel));
255+
} else if ("LOCKUP_CONTENT_TYPE_VIDEO".equals(contentType)) {
256+
collector.commit(new YoutubeStreamInfoItemLockupExtractor(
257+
lockupViewModel, timeAgoParser));
254258
}
255259
}
256260
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -760,7 +760,8 @@ public MultiInfoItemsCollector getRelatedItems() throws ExtractionException {
760760
} else if (result.has("lockupViewModel")) {
761761
final JsonObject lockupViewModel = result.getObject("lockupViewModel");
762762
final String contentType = lockupViewModel.getString("contentType");
763-
if ("LOCKUP_CONTENT_TYPE_PLAYLIST".equals(contentType)) {
763+
if ("LOCKUP_CONTENT_TYPE_PLAYLIST".equals(contentType)
764+
|| "LOCKUP_CONTENT_TYPE_PODCAST".equals(contentType)) {
764765
return new YoutubeMixOrPlaylistLockupInfoItemExtractor(
765766
lockupViewModel);
766767
} else if ("LOCKUP_CONTENT_TYPE_VIDEO".equals(contentType)) {

0 commit comments

Comments
 (0)