Skip to content

Commit 293c3e9

Browse files
AudricVAudricV
committed
[YouTube] Support new A/B tested comments data
Also improve current comments code by removing outdated comment renderer data.
1 parent e5b30ae commit 293c3e9

3 files changed

Lines changed: 377 additions & 71 deletions

File tree

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsEUVMInfoItemExtractor.java‎

Lines changed: 235 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,235 @@
1+
package org.schabi.newpipe.extractor.services.youtube.extractors;
2+
3+
import com.grack.nanojson.JsonObject;
4+
import org.schabi.newpipe.extractor.Image;
5+
import org.schabi.newpipe.extractor.Page;
6+
import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor;
7+
import org.schabi.newpipe.extractor.exceptions.ParsingException;
8+
import org.schabi.newpipe.extractor.localization.DateWrapper;
9+
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
10+
import org.schabi.newpipe.extractor.stream.Description;
11+
import org.schabi.newpipe.extractor.utils.Utils;
12+
13+
import javax.annotation.Nonnull;
14+
import javax.annotation.Nullable;
15+
import java.util.List;
16+
import java.util.Objects;
17+
18+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getAttributedDescription;
19+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray;
20+
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
21+
22+
/**
23+
* A {@link CommentsInfoItemExtractor} for YouTube comment data returned in a view model and entity
24+
* updates.
25+
*/
26+
class YoutubeCommentsEUVMInfoItemExtractor implements CommentsInfoItemExtractor {
27+
28+
private static final String AUTHOR = "author";
29+
private static final String PROPERTIES = "properties";
30+
31+
@Nonnull
32+
private final JsonObject commentViewModel;
33+
@Nullable
34+
private final JsonObject commentRepliesRenderer;
35+
@Nonnull
36+
private final JsonObject commentEntityPayload;
37+
@Nonnull
38+
private final JsonObject engagementToolbarStateEntityPayload;
39+
@Nonnull
40+
private final String videoUrl;
41+
@Nonnull
42+
private final TimeAgoParser timeAgoParser;
43+
44+
YoutubeCommentsEUVMInfoItemExtractor(
45+
@Nonnull final JsonObject commentViewModel,
46+
@Nullable final JsonObject commentRepliesRenderer,
47+
@Nonnull final JsonObject commentEntityPayload,
48+
@Nonnull final JsonObject engagementToolbarStateEntityPayload,
49+
@Nonnull final String videoUrl,
50+
@Nonnull final TimeAgoParser timeAgoParser) {
51+
this.commentViewModel = commentViewModel;
52+
this.commentRepliesRenderer = commentRepliesRenderer;
53+
this.commentEntityPayload = commentEntityPayload;
54+
this.engagementToolbarStateEntityPayload = engagementToolbarStateEntityPayload;
55+
this.videoUrl = videoUrl;
56+
this.timeAgoParser = timeAgoParser;
57+
}
58+
59+
@Override
60+
public String getName() throws ParsingException {
61+
return getUploaderName();
62+
}
63+
64+
@Override
65+
public String getUrl() throws ParsingException {
66+
return videoUrl;
67+
}
68+
69+
@Nonnull
70+
@Override
71+
public List<Image> getThumbnails() throws ParsingException {
72+
return getUploaderAvatars();
73+
}
74+
75+
@Override
76+
public int getLikeCount() throws ParsingException {
77+
final String textualLikeCount = getTextualLikeCount();
78+
try {
79+
if (Utils.isBlank(textualLikeCount)) {
80+
return 0;
81+
}
82+
83+
return (int) Utils.mixedNumberWordToLong(textualLikeCount);
84+
} catch (final Exception e) {
85+
throw new ParsingException(
86+
"Unexpected error while converting textual like count to like count", e);
87+
}
88+
}
89+
90+
@Override
91+
public String getTextualLikeCount() {
92+
return commentEntityPayload.getObject("toolbar")
93+
.getString("likeCountNotliked");
94+
}
95+
96+
@Override
97+
public Description getCommentText() throws ParsingException {
98+
// Comments' text work in the same way as an attributed video description
99+
return new Description(
100+
getAttributedDescription(commentEntityPayload.getObject(PROPERTIES)
101+
.getObject("content")), Description.HTML);
102+
}
103+
104+
@Override
105+
public String getTextualUploadDate() throws ParsingException {
106+
return commentEntityPayload.getObject(PROPERTIES)
107+
.getString("publishedTime");
108+
}
109+
110+
@Nullable
111+
@Override
112+
public DateWrapper getUploadDate() throws ParsingException {
113+
final String textualPublishedTime = getTextualUploadDate();
114+
if (isNullOrEmpty(textualPublishedTime)) {
115+
return null;
116+
}
117+
118+
return timeAgoParser.parse(textualPublishedTime);
119+
}
120+
121+
@Override
122+
public String getCommentId() throws ParsingException {
123+
String commentId = commentEntityPayload.getObject(PROPERTIES)
124+
.getString("commentId");
125+
if (isNullOrEmpty(commentId)) {
126+
commentId = commentViewModel.getString("commentId");
127+
if (isNullOrEmpty(commentId)) {
128+
throw new ParsingException("Could not get comment ID");
129+
}
130+
}
131+
return commentId;
132+
}
133+
134+
@Override
135+
public String getUploaderUrl() throws ParsingException {
136+
final JsonObject author = commentEntityPayload.getObject(AUTHOR);
137+
String channelId = author.getString("channelId");
138+
if (isNullOrEmpty(channelId)) {
139+
channelId = author.getObject("channelCommand")
140+
.getObject("innertubeCommand")
141+
.getObject("browseEndpoint")
142+
.getString("browseId");
143+
if (isNullOrEmpty(channelId)) {
144+
channelId = author.getObject("avatar")
145+
.getObject("endpoint")
146+
.getObject("innertubeCommand")
147+
.getObject("browseEndpoint")
148+
.getString("browseId");
149+
if (isNullOrEmpty(channelId)) {
150+
throw new ParsingException("Could not get channel ID");
151+
}
152+
}
153+
}
154+
return "https://www.youtube.com/channel/" + channelId;
155+
}
156+
157+
@Override
158+
public String getUploaderName() throws ParsingException {
159+
return commentEntityPayload.getObject(AUTHOR)
160+
.getString("displayName");
161+
}
162+
163+
@Nonnull
164+
@Override
165+
public List<Image> getUploaderAvatars() throws ParsingException {
166+
return getImagesFromThumbnailsArray(commentEntityPayload.getObject("avatar")
167+
.getObject("image")
168+
.getArray("sources"));
169+
}
170+
171+
@Override
172+
public boolean isHeartedByUploader() {
173+
return "TOOLBAR_HEART_STATE_HEARTED".equals(
174+
engagementToolbarStateEntityPayload.getString("heartState"));
175+
}
176+
177+
@Override
178+
public boolean isPinned() {
179+
return commentViewModel.has("pinnedText");
180+
}
181+
182+
@Override
183+
public boolean isUploaderVerified() throws ParsingException {
184+
final JsonObject author = commentEntityPayload.getObject(AUTHOR);
185+
return author.getBoolean("isVerified") || author.getBoolean("isArtist");
186+
}
187+
188+
@Override
189+
public int getReplyCount() throws ParsingException {
190+
// As YouTube allows replies up to 750 comments, we cannot check if the count returned is a
191+
// mixed number or a real number
192+
// Assume it is a mixed one, as it matches how numbers of most properties are returned
193+
final String replyCountString = commentEntityPayload.getObject("toolbar")
194+
.getString("replyCount");
195+
if (isNullOrEmpty(replyCountString)) {
196+
return 0;
197+
}
198+
return (int) Utils.mixedNumberWordToLong(replyCountString);
199+
}
200+
201+
@Nullable
202+
@Override
203+
public Page getReplies() throws ParsingException {
204+
if (isNullOrEmpty(commentRepliesRenderer)) {
205+
return null;
206+
}
207+
208+
final String continuation = commentRepliesRenderer.getArray("contents")
209+
.stream()
210+
.filter(JsonObject.class::isInstance)
211+
.map(JsonObject.class::cast)
212+
.map(content -> content.getObject("continuationItemRenderer", null))
213+
.filter(Objects::nonNull)
214+
.findFirst()
215+
.map(continuationItemRenderer ->
216+
continuationItemRenderer.getObject("continuationEndpoint")
217+
.getObject("continuationCommand")
218+
.getString("token"))
219+
.orElseThrow(() ->
220+
new ParsingException("Could not get comment replies continuation"));
221+
return new Page(videoUrl, continuation);
222+
}
223+
224+
@Override
225+
public boolean isChannelOwner() {
226+
return commentEntityPayload.getObject(AUTHOR)
227+
.getBoolean("isCreator");
228+
}
229+
230+
@Override
231+
public boolean hasCreatorReply() {
232+
return commentRepliesRenderer != null
233+
&& commentRepliesRenderer.has("viewRepliesCreatorThumbnail");
234+
}
235+
}

0 commit comments

Comments
 (0)