Skip to content

Commit 72d5ed3

Browse files
TobiGrTobiGr
authored
Merge pull request #987 from FireMasterK/comments-text-description
Use Description object for comments text.
2 parents 40f1ec4 + b566084 commit 72d5ed3

9 files changed

Lines changed: 41 additions & 30 deletions

File tree

‎extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItem.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,14 @@
33
import org.schabi.newpipe.extractor.InfoItem;
44
import org.schabi.newpipe.extractor.Page;
55
import org.schabi.newpipe.extractor.localization.DateWrapper;
6+
import org.schabi.newpipe.extractor.stream.Description;
67

78
import javax.annotation.Nullable;
89

910
public class CommentsInfoItem extends InfoItem {
1011

1112
private String commentId;
12-
private String commentText;
13+
private Description commentText;
1314
private String uploaderName;
1415
private String uploaderAvatarUrl;
1516
private String uploaderUrl;
@@ -43,11 +44,11 @@ public void setCommentId(final String commentId) {
4344
this.commentId = commentId;
4445
}
4546

46-
public String getCommentText() {
47+
public Description getCommentText() {
4748
return commentText;
4849
}
4950

50-
public void setCommentText(final String commentText) {
51+
public void setCommentText(final Description commentText) {
5152
this.commentText = commentText;
5253
}
5354

‎extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItemExtractor.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import org.schabi.newpipe.extractor.exceptions.ParsingException;
66
import org.schabi.newpipe.extractor.localization.DateWrapper;
77
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeCommentsInfoItemExtractor;
8+
import org.schabi.newpipe.extractor.stream.Description;
89
import org.schabi.newpipe.extractor.stream.StreamExtractor;
910

1011
import javax.annotation.Nullable;
@@ -41,8 +42,8 @@ default String getTextualLikeCount() throws ParsingException {
4142
/**
4243
* The text of the comment
4344
*/
44-
default String getCommentText() throws ParsingException {
45-
return "";
45+
default Description getCommentText() throws ParsingException {
46+
return Description.EMPTY_DESCRIPTION;
4647
}
4748

4849
/**

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampCommentsInfoItemExtractor.java‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import org.jsoup.nodes.Element;
44
import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor;
55
import org.schabi.newpipe.extractor.exceptions.ParsingException;
6+
import org.schabi.newpipe.extractor.stream.Description;
67

78
import java.util.Objects;
89

@@ -18,7 +19,7 @@ public BandcampCommentsInfoItemExtractor(final Element writing, final String url
1819

1920
@Override
2021
public String getName() throws ParsingException {
21-
return getCommentText();
22+
return getCommentText().getContent();
2223
}
2324

2425
@Override
@@ -32,12 +33,14 @@ public String getThumbnailUrl() throws ParsingException {
3233
}
3334

3435
@Override
35-
public String getCommentText() throws ParsingException {
36-
return writing.getElementsByClass("text").stream()
36+
public Description getCommentText() throws ParsingException {
37+
final var text = writing.getElementsByClass("text").stream()
3738
.filter(Objects::nonNull)
3839
.map(Element::ownText)
3940
.findFirst()
4041
.orElseThrow(() -> new ParsingException("Could not get comment text"));
42+
43+
return new Description(text, Description.PLAIN_TEXT);
4144
}
4245

4346
@Override

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
import org.schabi.newpipe.extractor.exceptions.ParsingException;
1010
import org.schabi.newpipe.extractor.localization.DateWrapper;
1111
import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
12+
import org.schabi.newpipe.extractor.stream.Description;
1213
import org.schabi.newpipe.extractor.utils.JsonUtils;
1314

1415
import java.util.Objects;
@@ -59,13 +60,15 @@ public DateWrapper getUploadDate() throws ParsingException {
5960
}
6061

6162
@Override
62-
public String getCommentText() throws ParsingException {
63+
public Description getCommentText() throws ParsingException {
6364
final String htmlText = JsonUtils.getString(item, "text");
6465
try {
6566
final Document doc = Jsoup.parse(htmlText);
66-
return doc.body().text();
67+
final var text = doc.body().text();
68+
return new Description(text, Description.PLAIN_TEXT);
6769
} catch (final Exception e) {
68-
return htmlText.replaceAll("(?s)<[^>]*>(\\s*<[^>]*>)*", "");
70+
final var text = htmlText.replaceAll("(?s)<[^>]*>(\\s*<[^>]*>)*", "");
71+
return new Description(text, Description.PLAIN_TEXT);
6972
}
7073
}
7174

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import org.schabi.newpipe.extractor.exceptions.ParsingException;
66
import org.schabi.newpipe.extractor.localization.DateWrapper;
77
import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper;
8+
import org.schabi.newpipe.extractor.stream.Description;
89

910
import javax.annotation.Nullable;
1011
import java.util.Objects;
@@ -24,8 +25,8 @@ public String getCommentId() {
2425
}
2526

2627
@Override
27-
public String getCommentText() {
28-
return json.getString("body");
28+
public Description getCommentText() {
29+
return new Description(json.getString("body"), Description.PLAIN_TEXT);
2930
}
3031

3132
@Override

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,21 @@
11
package org.schabi.newpipe.extractor.services.youtube.extractors;
22

3-
import static org.schabi.newpipe.extractor.comments.CommentsInfoItem.UNKNOWN_REPLY_COUNT;
4-
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
5-
63
import com.grack.nanojson.JsonArray;
74
import com.grack.nanojson.JsonObject;
8-
95
import org.schabi.newpipe.extractor.Page;
106
import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor;
117
import org.schabi.newpipe.extractor.exceptions.ParsingException;
128
import org.schabi.newpipe.extractor.localization.DateWrapper;
139
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
10+
import org.schabi.newpipe.extractor.stream.Description;
1411
import org.schabi.newpipe.extractor.utils.JsonUtils;
1512
import org.schabi.newpipe.extractor.utils.Utils;
1613

1714
import javax.annotation.Nullable;
1815

16+
import static org.schabi.newpipe.extractor.comments.CommentsInfoItem.UNKNOWN_REPLY_COUNT;
17+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
18+
1919
public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtractor {
2020

2121
private final JsonObject json;
@@ -176,18 +176,20 @@ public String getTextualLikeCount() throws ParsingException {
176176
}
177177

178178
@Override
179-
public String getCommentText() throws ParsingException {
179+
public Description getCommentText() throws ParsingException {
180180
try {
181181
final JsonObject contentText = JsonUtils.getObject(getCommentRenderer(), "contentText");
182182
if (contentText.isEmpty()) {
183183
// completely empty comments as described in
184184
// https://github.com/TeamNewPipe/NewPipeExtractor/issues/380#issuecomment-668808584
185-
return "";
185+
return Description.EMPTY_DESCRIPTION;
186186
}
187187
final String commentText = getTextFromObject(contentText, true);
188188
// YouTube adds U+FEFF in some comments.
189189
// eg. https://www.youtube.com/watch?v=Nj4F63E59io<feff>
190-
return Utils.removeUTF8BOM(commentText);
190+
final String commentTextBomRemoved = Utils.removeUTF8BOM(commentText);
191+
192+
return new Description(commentTextBomRemoved, Description.HTML);
191193
} catch (final Exception e) {
192194
throw new ParsingException("Could not get comment text", e);
193195
}

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampCommentsExtractorTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ public void testGetCommentsAllData() throws IOException, ExtractionException {
4242
for (CommentsInfoItem c : comments.getItems()) {
4343
assertFalse(Utils.isBlank(c.getUploaderName()));
4444
assertFalse(Utils.isBlank(c.getUploaderAvatarUrl()));
45-
assertFalse(Utils.isBlank(c.getCommentText()));
45+
assertFalse(Utils.isBlank(c.getCommentText().getContent()));
4646
assertFalse(Utils.isBlank(c.getName()));
4747
assertFalse(Utils.isBlank(c.getThumbnailUrl()));
4848
assertFalse(Utils.isBlank(c.getUrl()));

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ void testGetCommentsAllData() throws IOException, ExtractionException {
7575
assertFalse(Utils.isBlank(c.getUploaderName()));
7676
assertFalse(Utils.isBlank(c.getUploaderAvatarUrl()));
7777
assertFalse(Utils.isBlank(c.getCommentId()));
78-
assertFalse(Utils.isBlank(c.getCommentText()));
78+
assertFalse(Utils.isBlank(c.getCommentText().getContent()));
7979
assertFalse(Utils.isBlank(c.getName()));
8080
assertFalse(Utils.isBlank(c.getTextualUploadDate()));
8181
assertFalse(Utils.isBlank(c.getThumbnailUrl()));
@@ -91,7 +91,7 @@ private boolean findInComments(InfoItemsPage<CommentsInfoItem> comments, String
9191

9292
private boolean findInComments(List<CommentsInfoItem> comments, String comment) {
9393
for (CommentsInfoItem c : comments) {
94-
if (c.getCommentText().contains(comment)) {
94+
if (c.getCommentText().getContent().contains(comment)) {
9595
return true;
9696
}
9797
}

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ public void testGetCommentsAllData() throws IOException, ExtractionException {
9595
assertFalse(Utils.isBlank(c.getUploaderName()));
9696
assertFalse(Utils.isBlank(c.getUploaderAvatarUrl()));
9797
assertFalse(Utils.isBlank(c.getCommentId()));
98-
assertFalse(Utils.isBlank(c.getCommentText()));
98+
assertFalse(Utils.isBlank(c.getCommentText().getContent()));
9999
assertFalse(Utils.isBlank(c.getName()));
100100
assertFalse(Utils.isBlank(c.getTextualUploadDate()));
101101
assertNotNull(c.getUploadDate());
@@ -111,7 +111,7 @@ private boolean findInComments(InfoItemsPage<CommentsInfoItem> comments, String
111111

112112
private boolean findInComments(List<CommentsInfoItem> comments, String comment) {
113113
for (CommentsInfoItem c : comments) {
114-
if (c.getCommentText().contains(comment)) {
114+
if (c.getCommentText().getContent().contains(comment)) {
115115
return true;
116116
}
117117
}
@@ -152,9 +152,9 @@ public void testGetCommentsAllData() throws IOException, ExtractionException {
152152
assertFalse(Utils.isBlank(c.getUrl()));
153153
assertTrue(c.getLikeCount() >= 0);
154154
if (c.getCommentId().equals("Ugga_h1-EXdHB3gCoAEC")) { // comment without text
155-
assertTrue(Utils.isBlank(c.getCommentText()));
155+
assertTrue(Utils.isBlank(c.getCommentText().getContent()));
156156
} else {
157-
assertFalse(Utils.isBlank(c.getCommentText()));
157+
assertFalse(Utils.isBlank(c.getCommentText().getContent()));
158158
}
159159
}
160160
}
@@ -193,7 +193,7 @@ public void testGetCommentsAllData() throws IOException, ExtractionException {
193193
assertFalse(Utils.isBlank(c.getThumbnailUrl()));
194194
assertFalse(Utils.isBlank(c.getUrl()));
195195
assertTrue(c.getLikeCount() >= 0);
196-
assertFalse(Utils.isBlank(c.getCommentText()));
196+
assertFalse(Utils.isBlank(c.getCommentText().getContent()));
197197
if (c.isHeartedByUploader()) {
198198
heartedByUploader = true;
199199
}
@@ -233,7 +233,7 @@ public void testGetCommentsAllData() throws IOException, ExtractionException {
233233
assertFalse(Utils.isBlank(c.getThumbnailUrl()));
234234
assertFalse(Utils.isBlank(c.getUrl()));
235235
assertTrue(c.getLikeCount() >= 0);
236-
assertFalse(Utils.isBlank(c.getCommentText()));
236+
assertFalse(Utils.isBlank(c.getCommentText().getContent()));
237237
}
238238

239239
assertTrue(comments.getItems().get(0).isPinned(), "First comment isn't pinned");
@@ -328,7 +328,7 @@ public void testGetCommentsFirstReplies() throws IOException, ExtractionExceptio
328328

329329
InfoItemsPage<CommentsInfoItem> replies = extractor.getPage(firstComment.getReplies());
330330

331-
assertEquals("First", replies.getItems().get(0).getCommentText(),
331+
assertEquals("First", replies.getItems().get(0).getCommentText().getContent(),
332332
"First reply comment did not match");
333333
}
334334

0 commit comments

Comments
 (0)