Skip to content

Commit 8ec54ff

Browse files
litetexlitetex
committed
Basic implementation of YoutubeStreamInfoItemLockupExtractor 
1 parent 069d8c7 commit 8ec54ff

2 files changed

Lines changed: 283 additions & 2 deletions

File tree

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -759,10 +759,13 @@ public MultiInfoItemsCollector getRelatedItems() throws ExtractionException {
759759
result.getObject("compactPlaylistRenderer"));
760760
} else if (result.has("lockupViewModel")) {
761761
final JsonObject lockupViewModel = result.getObject("lockupViewModel");
762-
if ("LOCKUP_CONTENT_TYPE_PLAYLIST".equals(
763-
lockupViewModel.getString("contentType"))) {
762+
final String contentType = lockupViewModel.getString("contentType");
763+
if ("LOCKUP_CONTENT_TYPE_PLAYLIST".equals(contentType)) {
764764
return new YoutubeMixOrPlaylistLockupInfoItemExtractor(
765765
lockupViewModel);
766+
} else if ("LOCKUP_CONTENT_TYPE_VIDEO".equals(contentType)) {
767+
return new YoutubeStreamInfoItemLockupExtractor(
768+
lockupViewModel, timeAgoParser);
766769
}
767770
}
768771
return null;

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemLockupExtractor.java‎

Lines changed: 278 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,278 @@
1+
/*
2+
* Copyright (C) 2016 Christian Schabesberger <chris.schabesberger@mailbox.org>
3+
* YoutubeStreamInfoItemExtractor.java is part of NewPipe Extractor.
4+
*
5+
* NewPipe Extractor is free software: you can redistribute it and/or modify
6+
* it under the terms of the GNU General Public License as published by
7+
* the Free Software Foundation, either version 3 of the License, or
8+
* (at your option) any later version.
9+
*
10+
* NewPipe Extractor is distributed in the hope that it will be useful,
11+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
12+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13+
* GNU General Public License for more details.
14+
*
15+
* You should have received a copy of the GNU General Public License
16+
* along with NewPipe Extractor. If not, see <https://www.gnu.org/licenses/>.
17+
*/
18+
19+
package org.schabi.newpipe.extractor.services.youtube.extractors;
20+
21+
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
22+
23+
import com.grack.nanojson.JsonObject;
24+
25+
import org.schabi.newpipe.extractor.Image;
26+
import org.schabi.newpipe.extractor.exceptions.ParsingException;
27+
import org.schabi.newpipe.extractor.localization.DateWrapper;
28+
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
29+
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
30+
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
31+
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory;
32+
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
33+
import org.schabi.newpipe.extractor.stream.StreamType;
34+
import org.schabi.newpipe.extractor.utils.JsonUtils;
35+
import org.schabi.newpipe.extractor.utils.Utils;
36+
37+
import java.util.List;
38+
import java.util.Optional;
39+
import java.util.stream.Collectors;
40+
41+
import javax.annotation.Nonnull;
42+
import javax.annotation.Nullable;
43+
44+
public class YoutubeStreamInfoItemLockupExtractor implements StreamInfoItemExtractor {
45+
46+
private static final String NO_VIEWS_LOWERCASE = "no views";
47+
48+
private final JsonObject lockupViewModel;
49+
private final TimeAgoParser timeAgoParser;
50+
51+
/**
52+
* Creates an extractor of StreamInfoItems from a YouTube page.
53+
*
54+
* @param lockupViewModel The JSON page element
55+
* @param timeAgoParser A parser of the textual dates or {@code null}.
56+
*/
57+
public YoutubeStreamInfoItemLockupExtractor(final JsonObject lockupViewModel,
58+
@Nullable final TimeAgoParser timeAgoParser) {
59+
this.lockupViewModel = lockupViewModel;
60+
this.timeAgoParser = timeAgoParser;
61+
}
62+
63+
@Override
64+
public StreamType getStreamType() {
65+
// TODO only encountered video streams so far... Are there more types?
66+
return StreamType.VIDEO_STREAM;
67+
}
68+
69+
@Override
70+
public boolean isAd() throws ParsingException {
71+
if (isPremium()) {
72+
return true;
73+
}
74+
final String name = getName(); // only get it once
75+
return "[Private video]".equals(name)
76+
|| "[Deleted video]".equals(name);
77+
}
78+
79+
@Override
80+
public String getUrl() throws ParsingException {
81+
try {
82+
final String videoId = lockupViewModel.getString("contentId");
83+
return YoutubeStreamLinkHandlerFactory.getInstance().getUrl(videoId);
84+
} catch (final Exception e) {
85+
throw new ParsingException("Could not get url", e);
86+
}
87+
}
88+
89+
@Override
90+
public String getName() throws ParsingException {
91+
// TODO Is there Formatting?
92+
final String name = JsonUtils.getString(lockupViewModel,
93+
"metadata.lockupMetadataViewModel.title.content");
94+
if (!isNullOrEmpty(name)) {
95+
return name;
96+
}
97+
throw new ParsingException("Could not get name");
98+
}
99+
100+
@Override
101+
public long getDuration() throws ParsingException {
102+
final List<String> potentialDurations = lockupViewModel
103+
.getObject("contentImage")
104+
.getObject("thumbnailViewModel")
105+
.getArray("overlays")
106+
.streamAsJsonObjects()
107+
.flatMap(jsonObject -> jsonObject
108+
.getObject("thumbnailOverlayBadgeViewModel")
109+
.getArray("thumbnailBadges")
110+
.streamAsJsonObjects())
111+
.map(jsonObject -> jsonObject
112+
.getObject("thumbnailBadgeViewModel")
113+
.getString("text"))
114+
.collect(Collectors.toList());
115+
116+
if (potentialDurations.isEmpty()) {
117+
throw new ParsingException("Could not get duration: No parsable durations detected");
118+
}
119+
120+
ParsingException parsingException = null;
121+
for (final String potentialDuration : potentialDurations) {
122+
try {
123+
return YoutubeParsingHelper.parseDurationString(potentialDuration);
124+
} catch (final ParsingException ex) {
125+
parsingException = ex;
126+
}
127+
}
128+
129+
throw new ParsingException("Could not get duration", parsingException);
130+
}
131+
132+
@Override
133+
public String getUploaderName() throws ParsingException {
134+
return metadataPart(0, 0)
135+
.map(this::getTextContentFromMetadataPart)
136+
.filter(s -> !isNullOrEmpty(s))
137+
.orElseThrow(() -> new ParsingException("Could not get uploader name"));
138+
}
139+
140+
@Override
141+
public String getUploaderUrl() throws ParsingException {
142+
final String channelId = JsonUtils.getString(lockupViewModel,
143+
"metadata.lockupMetadataViewModel.image.decoratedAvatarViewModel"
144+
+ ".rendererContext.commandContext.onTap"
145+
+ ".innertubeCommand.browseEndpoint.browseId");
146+
if (isNullOrEmpty(channelId)) {
147+
throw new ParsingException("Could not get uploader url");
148+
}
149+
return YoutubeChannelLinkHandlerFactory.getInstance().getUrl(channelId);
150+
}
151+
152+
@Nonnull
153+
@Override
154+
public List<Image> getUploaderAvatars() throws ParsingException {
155+
return YoutubeParsingHelper.getImagesFromThumbnailsArray(
156+
JsonUtils.getArray(lockupViewModel,
157+
"metadata.lockupMetadataViewModel.image.decoratedAvatarViewModel"
158+
+ ".avatar.avatarViewModel.image.sources"));
159+
}
160+
161+
@Override
162+
public boolean isUploaderVerified() throws ParsingException {
163+
return metadataPart(0, 0)
164+
.stream()
165+
.flatMap(jsonObject -> jsonObject
166+
.getObject("text")
167+
.getArray("attachmentRuns")
168+
.streamAsJsonObjects())
169+
.flatMap(jsonObject -> jsonObject
170+
.getObject("element")
171+
.getObject("type")
172+
.getObject("imageType")
173+
.getObject("image")
174+
.getArray("sources")
175+
.streamAsJsonObjects())
176+
.map(jsonObject -> jsonObject
177+
.getObject("clientResource")
178+
.getString("imageName"))
179+
.map("CHECK_CIRCLE_FILLED"::equals)
180+
.findFirst()
181+
.orElse(false);
182+
}
183+
184+
@Nullable
185+
@Override
186+
public String getTextualUploadDate() throws ParsingException {
187+
return metadataPart(1, 1)
188+
.map(this::getTextContentFromMetadataPart)
189+
.orElse(null);
190+
}
191+
192+
@Nullable
193+
@Override
194+
public DateWrapper getUploadDate() throws ParsingException {
195+
if (timeAgoParser == null) {
196+
return null;
197+
}
198+
199+
return timeAgoParser.parse(getTextualUploadDate());
200+
}
201+
202+
@Override
203+
public long getViewCount() throws ParsingException {
204+
if (isPremium() || isPremiere()) {
205+
return -1;
206+
}
207+
208+
// TODO Check if this is the same for shorts
209+
final Optional<String> optTextContent = metadataPart(1, 0)
210+
.map(this::getTextContentFromMetadataPart);
211+
// We could do this inline if the ParsingException would be a RuntimeException -.-
212+
if (optTextContent.isPresent()) {
213+
return getViewCountFromViewCountText(optTextContent.get());
214+
}
215+
return -1;
216+
}
217+
218+
protected long getViewCountFromViewCountText(@Nonnull final String viewCountText)
219+
throws NumberFormatException, ParsingException {
220+
// These approaches are language dependent
221+
if (viewCountText.toLowerCase().contains(NO_VIEWS_LOWERCASE)) {
222+
return 0;
223+
} else if (viewCountText.toLowerCase().contains("recommended")) {
224+
return -1;
225+
}
226+
227+
return Utils.mixedNumberWordToLong(viewCountText);
228+
}
229+
230+
@Nonnull
231+
@Override
232+
public List<Image> getThumbnails() throws ParsingException {
233+
return YoutubeParsingHelper.getImagesFromThumbnailsArray(
234+
JsonUtils.getArray(lockupViewModel,
235+
"contentImage.thumbnailViewModel.image.sources"));
236+
}
237+
238+
protected boolean isPremium() {
239+
// TODO Detect with samples
240+
return false;
241+
}
242+
243+
protected boolean isPremiere() {
244+
// TODO Detect with samples
245+
return false;
246+
}
247+
248+
protected Optional<JsonObject> metadataPart(final int rowIndex, final int partIndex)
249+
throws ParsingException {
250+
return JsonUtils.getArray(lockupViewModel,
251+
"metadata.lockupMetadataViewModel.metadata"
252+
+ ".contentMetadataViewModel.metadataRows")
253+
.streamAsJsonObjects()
254+
.skip(rowIndex)
255+
.limit(1)
256+
.flatMap(jsonObject -> jsonObject.getArray("metadataParts")
257+
.streamAsJsonObjects()
258+
.skip(partIndex)
259+
.limit(1))
260+
.findFirst();
261+
}
262+
263+
protected String getTextContentFromMetadataPart(final JsonObject metadataPart) {
264+
return metadataPart.getObject("text").getString("content");
265+
}
266+
267+
@Nullable
268+
@Override
269+
public String getShortDescription() throws ParsingException {
270+
return null;
271+
}
272+
273+
@Override
274+
public boolean isShortFormContent() throws ParsingException {
275+
// TODO Detect with samples
276+
return false;
277+
}
278+
}

0 commit comments

Comments
 (0)