Skip to content

Commit aff3e79

Browse files
TobiGrTobiGr
committed
[PeerTube] Fix multi level comment replies
1 parent a9ca5c4 commit aff3e79

4 files changed

Lines changed: 143 additions & 38 deletions

File tree

‎extractor/src/main/java/org/schabi/newpipe/extractor/Page.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,10 @@ public Page(final String url, final String id) {
3737
this(url, id, null, null, null);
3838
}
3939

40+
public Page(final String url, final String id, final byte[] body) {
41+
this(url, id, null, null, body);
42+
}
43+
4044
public Page(final String url, final byte[] body) {
4145
this(url, null, null, null, body);
4246
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java‎

Lines changed: 47 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import com.grack.nanojson.JsonArray;
44
import com.grack.nanojson.JsonObject;
55
import com.grack.nanojson.JsonParser;
6+
import com.grack.nanojson.JsonParserException;
67
import org.schabi.newpipe.extractor.Page;
78
import org.schabi.newpipe.extractor.StreamingService;
89
import org.schabi.newpipe.extractor.comments.CommentsExtractor;
@@ -17,6 +18,7 @@
1718
import org.schabi.newpipe.extractor.utils.Utils;
1819

1920
import java.io.IOException;
21+
import java.nio.charset.StandardCharsets;
2022

2123
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY;
2224
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE;
@@ -26,6 +28,9 @@
2628
import javax.annotation.Nonnull;
2729

2830
public class PeertubeCommentsExtractor extends CommentsExtractor {
31+
static final String CHILDREN = "children";
32+
private static final String IS_DELETED = "isDeleted";
33+
private static final String TOTAL = "total";
2934

3035
/**
3136
* Use {@link #isReply()} to access this variable.
@@ -49,7 +54,7 @@ public InfoItemsPage<CommentsInfoItem> getInitialPage()
4954
}
5055
}
5156

52-
private boolean isReply() throws ParsingException {
57+
boolean isReply() throws ParsingException {
5358
if (isReply == null) {
5459
if (getOriginalUrl().contains("/videos/watch/")) {
5560
isReply = false;
@@ -67,22 +72,24 @@ private void collectCommentsFrom(@Nonnull final CommentsInfoItemsCollector colle
6772
for (final Object c : contents) {
6873
if (c instanceof JsonObject) {
6974
final JsonObject item = (JsonObject) c;
70-
if (!item.getBoolean("isDeleted")) {
71-
collector.commit(new PeertubeCommentsInfoItemExtractor(item, this));
75+
if (!item.getBoolean(IS_DELETED)) {
76+
collector.commit(new PeertubeCommentsInfoItemExtractor(item, null, this));
7277
}
7378
}
7479
}
7580
}
7681

7782
private void collectRepliesFrom(@Nonnull final CommentsInfoItemsCollector collector,
7883
@Nonnull final JsonObject json) throws ParsingException {
79-
final JsonArray contents = json.getArray("children");
84+
final JsonArray contents = json.getArray(CHILDREN);
8085

8186
for (final Object c : contents) {
8287
if (c instanceof JsonObject) {
83-
final JsonObject item = ((JsonObject) c).getObject("comment");
84-
if (!item.getBoolean("isDeleted")) {
85-
collector.commit(new PeertubeCommentsInfoItemExtractor(item, this));
88+
final JsonObject content = (JsonObject) c;
89+
final JsonObject item = content.getObject("comment");
90+
final JsonArray children = content.getArray(CHILDREN);
91+
if (!item.getBoolean(IS_DELETED)) {
92+
collector.commit(new PeertubeCommentsInfoItemExtractor(item, children, this));
8693
}
8794
}
8895
}
@@ -95,36 +102,46 @@ public InfoItemsPage<CommentsInfoItem> getPage(final Page page)
95102
throw new IllegalArgumentException("Page doesn't contain an URL");
96103
}
97104

98-
final Response response = getDownloader().get(page.getUrl());
99-
100105
JsonObject json = null;
101-
if (response != null && !Utils.isBlank(response.responseBody())) {
106+
final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId());
107+
final long total;
108+
if (page.getBody() == null) {
109+
final Response response = getDownloader().get(page.getUrl());
110+
if (response != null && !Utils.isBlank(response.responseBody())) {
111+
try {
112+
json = JsonParser.object().from(response.responseBody());
113+
} catch (final Exception e) {
114+
throw new ParsingException("Could not parse json data for comments info", e);
115+
}
116+
}
117+
if (json != null) {
118+
PeertubeParsingHelper.validate(json);
119+
if (isReply() || json.has(CHILDREN)) {
120+
total = json.getArray(CHILDREN).size();
121+
collectRepliesFrom(collector, json);
122+
} else {
123+
total = json.getLong(TOTAL);
124+
collectCommentsFrom(collector, json);
125+
}
126+
} else {
127+
throw new ExtractionException("Unable to get PeerTube kiosk info");
128+
}
129+
} else {
102130
try {
103-
json = JsonParser.object().from(response.responseBody());
104-
} catch (final Exception e) {
105-
throw new ParsingException("Could not parse json data for comments info", e);
131+
json = JsonParser.object().from(new String(page.getBody(), StandardCharsets.UTF_8));
132+
isReply = true;
133+
total = json.getArray(CHILDREN).size();
134+
collectRepliesFrom(collector, json);
135+
} catch (final JsonParserException e) {
136+
throw new ParsingException(
137+
"Could not parse json data for nested comments info", e);
106138
}
107139
}
108140

109-
if (json != null) {
110-
PeertubeParsingHelper.validate(json);
111-
final long total;
112-
final CommentsInfoItemsCollector collector
113-
= new CommentsInfoItemsCollector(getServiceId());
141+
return new InfoItemsPage<>(collector,
142+
PeertubeParsingHelper.getNextPage(page.getUrl(), total));
114143

115-
if (isReply() || json.has("children")) {
116-
total = json.getArray("children").size();
117-
collectRepliesFrom(collector, json);
118-
} else {
119-
total = json.getLong("total");
120-
collectCommentsFrom(collector, json);
121-
}
122144

123-
return new InfoItemsPage<>(collector,
124-
PeertubeParsingHelper.getNextPage(page.getUrl(), total));
125-
} else {
126-
throw new ExtractionException("Unable to get PeerTube kiosk info");
127-
}
128145
}
129146

130147
@Override

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java‎

Lines changed: 42 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
package org.schabi.newpipe.extractor.services.peertube.extractors;
22

3+
import com.grack.nanojson.JsonArray;
34
import com.grack.nanojson.JsonObject;
45

6+
import com.grack.nanojson.JsonWriter;
57
import org.jsoup.Jsoup;
68
import org.jsoup.nodes.Document;
79
import org.schabi.newpipe.extractor.Page;
@@ -13,20 +15,36 @@
1315
import org.schabi.newpipe.extractor.stream.Description;
1416
import org.schabi.newpipe.extractor.utils.JsonUtils;
1517

18+
import javax.annotation.Nonnull;
1619
import javax.annotation.Nullable;
20+
import java.nio.charset.StandardCharsets;
1721
import java.util.Objects;
1822

23+
import static org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeCommentsExtractor.CHILDREN;
24+
1925
public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtractor {
26+
@Nonnull
2027
private final JsonObject item;
28+
@Nullable
29+
private final JsonArray children;
30+
@Nonnull
2131
private final String url;
32+
@Nonnull
2233
private final String baseUrl;
34+
@Nonnull
35+
private final PeertubeCommentsExtractor superCommentExtractor;
36+
37+
private Integer replyCount;
2338

24-
public PeertubeCommentsInfoItemExtractor(final JsonObject item,
25-
final PeertubeCommentsExtractor extractor)
39+
public PeertubeCommentsInfoItemExtractor(@Nonnull final JsonObject item,
40+
@Nullable final JsonArray children,
41+
@Nonnull final PeertubeCommentsExtractor extractor)
2642
throws ParsingException {
2743
this.item = item;
44+
this.children = children;
2845
this.url = extractor.getUrl();
2946
this.baseUrl = extractor.getBaseUrl();
47+
this.superCommentExtractor = extractor;
3048
}
3149

3250
@Override
@@ -107,15 +125,34 @@ public String getUploaderUrl() throws ParsingException {
107125
@Override
108126
@Nullable
109127
public Page getReplies() throws ParsingException {
110-
if (JsonUtils.getNumber(item, "totalReplies").intValue() == 0) {
128+
if (getReplyCount() == 0) {
111129
return null;
112130
}
113131
final String threadId = JsonUtils.getNumber(item, "threadId").toString();
114-
return new Page(url + "/" + threadId, threadId);
132+
final String repliesUrl = url + "/" + threadId;
133+
if (superCommentExtractor.isReply() && children != null && !children.isEmpty()) {
134+
// Nested replies are already included in the original thread's request.
135+
// Wrap the replies into a JsonObject, because the original thread's request body
136+
// is also structured like a JsonObject.
137+
final JsonObject pageContent = new JsonObject();
138+
pageContent.put(CHILDREN, children);
139+
return new Page(repliesUrl, threadId,
140+
JsonWriter.string(pageContent).getBytes(StandardCharsets.UTF_8));
141+
}
142+
return new Page(repliesUrl, threadId);
115143
}
116144

117145
@Override
118146
public int getReplyCount() throws ParsingException {
119-
return JsonUtils.getNumber(item, "totalReplies").intValue();
147+
if (replyCount == null) {
148+
if (children != null && !children.isEmpty()) {
149+
// The totalReplies field is inaccurate for nested replies and sometimes returns 0
150+
// although there are replies to that reply stored in children.
151+
replyCount = children.size();
152+
} else {
153+
replyCount = JsonUtils.getNumber(item, "totalReplies").intValue();
154+
}
155+
}
156+
return replyCount;
120157
}
121158
}

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java‎

Lines changed: 50 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,9 @@
1414

1515
import java.io.IOException;
1616
import java.util.List;
17+
import java.util.Optional;
1718

18-
import static org.junit.jupiter.api.Assertions.assertEquals;
19-
import static org.junit.jupiter.api.Assertions.assertFalse;
20-
import static org.junit.jupiter.api.Assertions.assertTrue;
19+
import static org.junit.jupiter.api.Assertions.*;
2120
import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
2221

2322
public class PeertubeCommentsExtractorTest {
@@ -121,4 +120,52 @@ void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException {
121120
assertTrue(commentsInfo.getErrors().isEmpty());
122121
}
123122
}
123+
124+
/**
125+
* Test a video that has comments with nested replies.
126+
*/
127+
public static class NestedComments {
128+
private static PeertubeCommentsExtractor extractor;
129+
130+
@BeforeAll
131+
public static void setUp() throws Exception {
132+
NewPipe.init(DownloaderTestImpl.getInstance());
133+
extractor = (PeertubeCommentsExtractor) PeerTube
134+
.getCommentsExtractor("https://share.tube/w/vxu4uTstUBAUromWwXGHrq");
135+
}
136+
137+
@Test
138+
void testGetComments() throws IOException, ExtractionException {
139+
final InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
140+
assertFalse(comments.getItems().isEmpty());
141+
final Optional<CommentsInfoItem> nestedCommentHeadOpt =
142+
comments.getItems()
143+
.stream()
144+
.filter(c -> c.getCommentId().equals("9770"))
145+
.findFirst();
146+
assertTrue(nestedCommentHeadOpt.isPresent());
147+
assertTrue(findNestedCommentWithId("9773", nestedCommentHeadOpt.get()), "The nested comment replies were not found");
148+
}
149+
}
150+
151+
private static boolean findNestedCommentWithId(final String id, final CommentsInfoItem comment)
152+
throws IOException, ExtractionException {
153+
if (comment.getCommentId().equals(id)) {
154+
return true;
155+
}
156+
return PeerTube
157+
.getCommentsExtractor(comment.getUrl())
158+
.getPage(comment.getReplies())
159+
.getItems()
160+
.stream()
161+
.map(c -> {
162+
try {
163+
return findNestedCommentWithId(id, c);
164+
} catch (final Exception ignored) {
165+
return false;
166+
}
167+
})
168+
.reduce((a, b) -> a || b)
169+
.orElse(false);
170+
}
124171
}

0 commit comments

Comments
 (0)