Skip to content

Commit a9d2144

Browse files
TobiGrTobiGr
authored
Merge pull request #703 from FireMasterK/comment-replies
Add support for extracting comment replies continuation
2 parents ce8cabb + 6aabdc6 commit a9d2144

10 files changed

Lines changed: 1275 additions & 30 deletions

File tree

‎extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItem.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package org.schabi.newpipe.extractor.comments;
22

33
import org.schabi.newpipe.extractor.InfoItem;
4+
import org.schabi.newpipe.extractor.Page;
45
import org.schabi.newpipe.extractor.localization.DateWrapper;
56

67
import javax.annotation.Nullable;
@@ -21,6 +22,8 @@ public class CommentsInfoItem extends InfoItem {
2122
private boolean heartedByUploader;
2223
private boolean pinned;
2324
private int streamPosition;
25+
@Nullable
26+
private Page replies;
2427

2528
public static final int NO_LIKE_COUNT = -1;
2629
public static final int NO_STREAM_POSITION = -1;
@@ -142,4 +145,8 @@ public void setStreamPosition(final int streamPosition) {
142145
public int getStreamPosition() {
143146
return streamPosition;
144147
}
148+
149+
public void setReplies(@Nullable Page replies) { this.replies = replies; }
150+
151+
public Page getReplies() { return this.replies; }
145152
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItemExtractor.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package org.schabi.newpipe.extractor.comments;
22

33
import org.schabi.newpipe.extractor.InfoItemExtractor;
4+
import org.schabi.newpipe.extractor.Page;
45
import org.schabi.newpipe.extractor.exceptions.ParsingException;
56
import org.schabi.newpipe.extractor.localization.DateWrapper;
67
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeCommentsInfoItemExtractor;
@@ -107,4 +108,13 @@ default boolean isUploaderVerified() throws ParsingException {
107108
default int getStreamPosition() throws ParsingException {
108109
return CommentsInfoItem.NO_STREAM_POSITION;
109110
}
111+
112+
/**
113+
* The continuation page which is used to get comment replies from.
114+
* @return the continuation Page for the replies, or null if replies are not supported
115+
*/
116+
@Nullable
117+
default Page getReplies() throws ParsingException {
118+
return null;
119+
}
110120
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItemsCollector.java‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,12 @@ public CommentsInfoItem extract(CommentsInfoItemExtractor extractor) throws Pars
9393
addError(e);
9494
}
9595

96+
try {
97+
resultItem.setReplies(extractor.getReplies());
98+
} catch (Exception e) {
99+
addError(e);
100+
}
101+
96102
return resultItem;
97103
}
98104

@@ -106,12 +112,6 @@ public void commit(CommentsInfoItemExtractor extractor) {
106112
}
107113

108114
public List<CommentsInfoItem> getCommentsInfoItemList() {
109-
List<CommentsInfoItem> siiList = new ArrayList<>();
110-
for (InfoItem ii : super.getItems()) {
111-
if (ii instanceof CommentsInfoItem) {
112-
siiList.add((CommentsInfoItem) ii);
113-
}
114-
}
115-
return siiList;
115+
return new ArrayList<>(super.getItems());
116116
}
117117
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java‎

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,7 @@ private String findInitialCommentsToken() throws ExtractionException {
101101

102102
if (itemSectionRenderer.isPresent()) {
103103
token = JsonUtils.getString(((JsonObject) itemSectionRenderer.get())
104-
.getObject("itemSectionRenderer").getArray("contents").getObject(0),
104+
.getObject("itemSectionRenderer").getArray("contents").getObject(0),
105105
"continuationItemRenderer.continuationEndpoint.continuationCommand.token");
106106
} else {
107107
token = null;
@@ -140,10 +140,13 @@ private Page getNextPage(@Nonnull final JsonObject ajaxJson) throws ExtractionEx
140140
return null;
141141
}
142142

143+
final JsonObject continuationItemRenderer = jsonArray.getObject(jsonArray.size() - 1).getObject("continuationItemRenderer");
144+
145+
final String jsonPath = continuationItemRenderer.has("button") ? "button.buttonRenderer.command.continuationCommand.token" : "continuationEndpoint.continuationCommand.token";
146+
143147
final String continuation;
144148
try {
145-
continuation = JsonUtils.getString(jsonArray.getObject(jsonArray.size() - 1),
146-
"continuationItemRenderer.continuationEndpoint.continuationCommand.token");
149+
continuation = JsonUtils.getString(continuationItemRenderer, jsonPath);
147150
} catch (final Exception e) {
148151
return null;
149152
}
@@ -167,7 +170,7 @@ public InfoItemsPage<CommentsInfoItem> getPage(final Page page)
167170

168171
final Localization localization = getExtractorLocalization();
169172
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(localization,
170-
getExtractorContentCountry())
173+
getExtractorContentCountry())
171174
.value("continuation", page.getId())
172175
.done())
173176
.getBytes(UTF_8);
@@ -212,10 +215,11 @@ private void collectCommentsFrom(final CommentsInfoItemsCollector collector,
212215
contents.remove(index);
213216
}
214217

218+
final String jsonKey = contents.getObject(0).has("commentThreadRenderer") ? "commentThreadRenderer" : "commentRenderer";
219+
215220
final List<Object> comments;
216221
try {
217-
comments = JsonUtils.getValues(contents,
218-
"commentThreadRenderer.comment.commentRenderer");
222+
comments = JsonUtils.getValues(contents, jsonKey);
219223
} catch (final Exception e) {
220224
throw new ParsingException("Unable to get parse youtube comments", e);
221225
}
@@ -234,7 +238,7 @@ public void onFetchPage(@Nonnull final Downloader downloader)
234238
throws IOException, ExtractionException {
235239
final Localization localization = getExtractorLocalization();
236240
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(localization,
237-
getExtractorContentCountry())
241+
getExtractorContentCountry())
238242
.value("videoId", getId())
239243
.done())
240244
.getBytes(UTF_8);

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java‎

Lines changed: 39 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@
33
import com.grack.nanojson.JsonArray;
44
import com.grack.nanojson.JsonObject;
55

6+
import com.grack.nanojson.JsonWriter;
7+
import org.schabi.newpipe.extractor.Page;
68
import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor;
79
import org.schabi.newpipe.extractor.exceptions.ParsingException;
810
import org.schabi.newpipe.extractor.localization.DateWrapper;
@@ -18,6 +20,7 @@
1820
public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtractor {
1921

2022
private final JsonObject json;
23+
private JsonObject commentRenderer;
2124
private final String url;
2225
private final TimeAgoParser timeAgoParser;
2326

@@ -29,6 +32,16 @@ public YoutubeCommentsInfoItemExtractor(final JsonObject json,
2932
this.timeAgoParser = timeAgoParser;
3033
}
3134

35+
private JsonObject getCommentRenderer() throws ParsingException {
36+
if(commentRenderer == null) {
37+
if(!json.has("comment"))
38+
commentRenderer = json;
39+
else
40+
commentRenderer = JsonUtils.getObject(json, "comment.commentRenderer");
41+
}
42+
return commentRenderer;
43+
}
44+
3245
@Override
3346
public String getUrl() throws ParsingException {
3447
return url;
@@ -37,7 +50,7 @@ public String getUrl() throws ParsingException {
3750
@Override
3851
public String getThumbnailUrl() throws ParsingException {
3952
try {
40-
final JsonArray arr = JsonUtils.getArray(json, "authorThumbnail.thumbnails");
53+
final JsonArray arr = JsonUtils.getArray(getCommentRenderer(), "authorThumbnail.thumbnails");
4154
return JsonUtils.getString(arr.getObject(2), "url");
4255
} catch (final Exception e) {
4356
throw new ParsingException("Could not get thumbnail url", e);
@@ -47,7 +60,7 @@ public String getThumbnailUrl() throws ParsingException {
4760
@Override
4861
public String getName() throws ParsingException {
4962
try {
50-
return getTextFromObject(JsonUtils.getObject(json, "authorText"));
63+
return getTextFromObject(JsonUtils.getObject(getCommentRenderer(), "authorText"));
5164
} catch (final Exception e) {
5265
return EMPTY_STRING;
5366
}
@@ -56,7 +69,7 @@ public String getName() throws ParsingException {
5669
@Override
5770
public String getTextualUploadDate() throws ParsingException {
5871
try {
59-
return getTextFromObject(JsonUtils.getObject(json, "publishedTimeText"));
72+
return getTextFromObject(JsonUtils.getObject(getCommentRenderer(), "publishedTimeText"));
6073
} catch (final Exception e) {
6174
throw new ParsingException("Could not get publishedTimeText", e);
6275
}
@@ -94,7 +107,7 @@ public int getLikeCount() throws ParsingException {
94107
// Try first to get the exact like count by using the accessibility data
95108
final String likeCount;
96109
try {
97-
likeCount = Utils.removeNonDigitCharacters(JsonUtils.getString(json,
110+
likeCount = Utils.removeNonDigitCharacters(JsonUtils.getString(getCommentRenderer(),
98111
"actionButtons.commentActionButtonsRenderer.likeButton.toggleButtonRenderer.accessibilityData.accessibilityData.label"));
99112
} catch (final Exception e) {
100113
// Use the approximate like count returned into the voteCount object
@@ -145,11 +158,11 @@ public String getTextualLikeCount() throws ParsingException {
145158
*/
146159
try {
147160
// If a comment has no likes voteCount is not set
148-
if (!json.has("voteCount")) {
161+
if (!getCommentRenderer().has("voteCount")) {
149162
return EMPTY_STRING;
150163
}
151164

152-
final JsonObject voteCountObj = JsonUtils.getObject(json, "voteCount");
165+
final JsonObject voteCountObj = JsonUtils.getObject(getCommentRenderer(), "voteCount");
153166
if (voteCountObj.isEmpty()) {
154167
return EMPTY_STRING;
155168
}
@@ -162,7 +175,7 @@ public String getTextualLikeCount() throws ParsingException {
162175
@Override
163176
public String getCommentText() throws ParsingException {
164177
try {
165-
final JsonObject contentText = JsonUtils.getObject(json, "contentText");
178+
final JsonObject contentText = JsonUtils.getObject(getCommentRenderer(), "contentText");
166179
if (contentText.isEmpty()) {
167180
// completely empty comments as described in
168181
// https://github.com/TeamNewPipe/NewPipeExtractor/issues/380#issuecomment-668808584
@@ -180,7 +193,7 @@ public String getCommentText() throws ParsingException {
180193
@Override
181194
public String getCommentId() throws ParsingException {
182195
try {
183-
return JsonUtils.getString(json, "commentId");
196+
return JsonUtils.getString(getCommentRenderer(), "commentId");
184197
} catch (final Exception e) {
185198
throw new ParsingException("Could not get comment id", e);
186199
}
@@ -189,7 +202,7 @@ public String getCommentId() throws ParsingException {
189202
@Override
190203
public String getUploaderAvatarUrl() throws ParsingException {
191204
try {
192-
JsonArray arr = JsonUtils.getArray(json, "authorThumbnail.thumbnails");
205+
JsonArray arr = JsonUtils.getArray(getCommentRenderer(), "authorThumbnail.thumbnails");
193206
return JsonUtils.getString(arr.getObject(2), "url");
194207
} catch (final Exception e) {
195208
throw new ParsingException("Could not get author thumbnail", e);
@@ -198,24 +211,24 @@ public String getUploaderAvatarUrl() throws ParsingException {
198211

199212
@Override
200213
public boolean isHeartedByUploader() throws ParsingException {
201-
final JsonObject commentActionButtonsRenderer = json.getObject("actionButtons")
214+
final JsonObject commentActionButtonsRenderer = getCommentRenderer().getObject("actionButtons")
202215
.getObject("commentActionButtonsRenderer");
203216
return commentActionButtonsRenderer.has("creatorHeart");
204217
}
205218

206219
@Override
207-
public boolean isPinned() {
208-
return json.has("pinnedCommentBadge");
220+
public boolean isPinned() throws ParsingException {
221+
return getCommentRenderer().has("pinnedCommentBadge");
209222
}
210223

211-
public boolean isUploaderVerified() {
212-
return json.has("authorCommentBadge");
224+
public boolean isUploaderVerified() throws ParsingException {
225+
return getCommentRenderer().has("authorCommentBadge");
213226
}
214227

215228
@Override
216229
public String getUploaderName() throws ParsingException {
217230
try {
218-
return getTextFromObject(JsonUtils.getObject(json, "authorText"));
231+
return getTextFromObject(JsonUtils.getObject(getCommentRenderer(), "authorText"));
219232
} catch (final Exception e) {
220233
return EMPTY_STRING;
221234
}
@@ -224,10 +237,20 @@ public String getUploaderName() throws ParsingException {
224237
@Override
225238
public String getUploaderUrl() throws ParsingException {
226239
try {
227-
return "https://www.youtube.com/channel/" + JsonUtils.getString(json,
240+
return "https://www.youtube.com/channel/" + JsonUtils.getString(getCommentRenderer(),
228241
"authorEndpoint.browseEndpoint.browseId");
229242
} catch (final Exception e) {
230243
return EMPTY_STRING;
231244
}
232245
}
246+
247+
@Override
248+
public Page getReplies() throws ParsingException {
249+
try {
250+
final String id = JsonUtils.getString(JsonUtils.getArray(json, "replies.commentRepliesRenderer.contents").getObject(0), "continuationItemRenderer.continuationEndpoint.continuationCommand.token");
251+
return new Page(url, id);
252+
} catch (final Exception e) {
253+
return null; // Would return null for Comment Replies, since YouTube does not support nested replies.
254+
}
255+
}
233256
}

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -306,4 +306,32 @@ public void testGetCommentsFirst() throws IOException, ExtractionException {
306306
assertTrue("The first pinned comment has no vote count", !Utils.isBlank(pinnedComment.getTextualLikeCount()));
307307
}
308308
}
309+
310+
public static class RepliesTest {
311+
private final static String url = "https://www.youtube.com/watch?v=--yeOvJGZQk";
312+
private static YoutubeCommentsExtractor extractor;
313+
314+
@BeforeClass
315+
public static void setUp() throws Exception {
316+
YoutubeParsingHelper.resetClientVersionAndKey();
317+
YoutubeParsingHelper.setNumberGenerator(new Random(1));
318+
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "replies"));
319+
extractor = (YoutubeCommentsExtractor) YouTube
320+
.getCommentsExtractor(url);
321+
extractor.fetchPage();
322+
}
323+
324+
@Test
325+
public void testGetCommentsFirstReplies() throws IOException, ExtractionException {
326+
final InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
327+
328+
DefaultTests.defaultTestListOfItems(YouTube, comments.getItems(), comments.getErrors());
329+
330+
CommentsInfoItem firstComment = comments.getItems().get(0);
331+
332+
InfoItemsPage<CommentsInfoItem> replies = extractor.getPage(firstComment.getReplies());
333+
334+
assertEquals("First reply comment did not match", "Lol", replies.getItems().get(0).getCommentText());
335+
}
336+
}
309337
}

0 commit comments

Comments
 (0)