Skip to content

Commit de8edbe

Browse files
TobiGrTobiGr
authored
Merge pull request #382 from TobiGr/fix/empty_comment
[YouTube] Fix crash on empty comment
2 parents 5ac8062 + 0fb7330 commit de8edbe

3 files changed

Lines changed: 134 additions & 79 deletions

File tree

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -47,13 +47,13 @@ public YoutubeCommentsExtractor(StreamingService service, ListLinkHandler uiHand
4747

4848
@Override
4949
public InfoItemsPage<CommentsInfoItem> getInitialPage() throws IOException, ExtractionException {
50-
String commentsTokenInside = findValue(responseBody, "commentSectionRenderer", "}");
51-
String commentsToken = findValue(commentsTokenInside, "continuation\":\"", "\"");
50+
final String commentsTokenInside = findValue(responseBody, "commentSectionRenderer", "}");
51+
final String commentsToken = findValue(commentsTokenInside, "continuation\":\"", "\"");
5252
return getPage(getNextPage(commentsToken));
5353
}
5454

5555
private Page getNextPage(JsonObject ajaxJson) throws ParsingException {
56-
JsonArray arr;
56+
final JsonArray arr;
5757
try {
5858
arr = JsonUtils.getArray(ajaxJson, "response.continuationContents.commentSectionContinuation.continuations");
5959
} catch (Exception e) {
@@ -89,14 +89,14 @@ public InfoItemsPage<CommentsInfoItem> getPage(final Page page) throws IOExcepti
8989
throw new IllegalArgumentException("Page doesn't contain an URL");
9090
}
9191

92-
String ajaxResponse = makeAjaxRequest(page.getUrl());
93-
JsonObject ajaxJson;
92+
final String ajaxResponse = makeAjaxRequest(page.getUrl());
93+
final JsonObject ajaxJson;
9494
try {
9595
ajaxJson = JsonParser.array().from(ajaxResponse).getObject(1);
9696
} catch (Exception e) {
9797
throw new ParsingException("Could not parse json data for comments", e);
9898
}
99-
CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId());
99+
final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId());
100100
collectCommentsFrom(collector, ajaxJson);
101101
return new InfoItemsPage<>(collector, getNextPage(ajaxJson));
102102
}
@@ -160,8 +160,8 @@ private String getDataString(Map<String, String> params) throws UnsupportedEncod
160160
}
161161

162162
private String findValue(String doc, String start, String end) {
163-
int beginIndex = doc.indexOf(start) + start.length();
164-
int endIndex = doc.indexOf(end, beginIndex);
163+
final int beginIndex = doc.indexOf(start) + start.length();
164+
final int endIndex = doc.indexOf(end, beginIndex);
165165
return doc.substring(beginIndex, endIndex);
166166
}
167167
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ public String getUrl() throws ParsingException {
3434
@Override
3535
public String getThumbnailUrl() throws ParsingException {
3636
try {
37-
JsonArray arr = JsonUtils.getArray(json, "authorThumbnail.thumbnails");
37+
final JsonArray arr = JsonUtils.getArray(json, "authorThumbnail.thumbnails");
3838
return JsonUtils.getString(arr.getObject(2), "url");
3939
} catch (Exception e) {
4040
throw new ParsingException("Could not get thumbnail url", e);
@@ -82,7 +82,13 @@ public int getLikeCount() throws ParsingException {
8282
@Override
8383
public String getCommentText() throws ParsingException {
8484
try {
85-
String commentText = getTextFromObject(JsonUtils.getObject(json, "contentText"));
85+
final JsonObject contentText = JsonUtils.getObject(json, "contentText");
86+
if (contentText.isEmpty()) {
87+
// completely empty comments as described in
88+
// https://github.com/TeamNewPipe/NewPipeExtractor/issues/380#issuecomment-668808584
89+
return "";
90+
}
91+
final String commentText = getTextFromObject(contentText);
8692
// youtube adds U+FEFF in some comments. eg. https://www.youtube.com/watch?v=Nj4F63E59io<feff>
8793
return Utils.removeUTF8BOM(commentText);
8894
} catch (Exception e) {

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java‎

Lines changed: 118 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -23,91 +23,140 @@
2323
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
2424

2525
public class YoutubeCommentsExtractorTest {
26-
private static final String urlYT = "https://www.youtube.com/watch?v=D00Au7k3i6o";
27-
private static final String urlInvidious = "https://invidio.us/watch?v=D00Au7k3i6o";
28-
private static YoutubeCommentsExtractor extractorYT;
29-
private static YoutubeCommentsExtractor extractorInvidious;
30-
31-
@BeforeClass
32-
public static void setUp() throws Exception {
33-
NewPipe.init(DownloaderTestImpl.getInstance());
34-
extractorYT = (YoutubeCommentsExtractor) YouTube
35-
.getCommentsExtractor(urlYT);
36-
extractorYT.fetchPage();
37-
extractorInvidious = (YoutubeCommentsExtractor) YouTube
38-
.getCommentsExtractor(urlInvidious);
39-
}
26+
/**
27+
* Test a "normal" YouTube and Invidious page
28+
*/
29+
public static class Thomas {
30+
private static final String urlYT = "https://www.youtube.com/watch?v=D00Au7k3i6o";
31+
private static final String urlInvidious = "https://invidio.us/watch?v=D00Au7k3i6o";
32+
private static YoutubeCommentsExtractor extractorYT;
33+
private static YoutubeCommentsExtractor extractorInvidious;
34+
35+
private static final String commentContent = "sub 4 sub";
36+
37+
@BeforeClass
38+
public static void setUp() throws Exception {
39+
NewPipe.init(DownloaderTestImpl.getInstance());
40+
extractorYT = (YoutubeCommentsExtractor) YouTube
41+
.getCommentsExtractor(urlYT);
42+
extractorYT.fetchPage();
43+
extractorInvidious = (YoutubeCommentsExtractor) YouTube
44+
.getCommentsExtractor(urlInvidious);
45+
extractorInvidious.fetchPage();
46+
}
4047

41-
@Test
42-
public void testGetComments() throws IOException, ExtractionException {
43-
assertTrue(getCommentsHelper(extractorYT));
44-
assertTrue(getCommentsHelper(extractorInvidious));
45-
}
48+
@Test
49+
public void testGetComments() throws IOException, ExtractionException {
50+
assertTrue(getCommentsHelper(extractorYT));
51+
assertTrue(getCommentsHelper(extractorInvidious));
52+
}
53+
54+
private boolean getCommentsHelper(YoutubeCommentsExtractor extractor) throws IOException, ExtractionException {
55+
InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
56+
boolean result = findInComments(comments, commentContent);
4657

47-
private boolean getCommentsHelper(YoutubeCommentsExtractor extractor) throws IOException, ExtractionException {
48-
InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
49-
boolean result = findInComments(comments, "s1ck m3m3");
58+
while (comments.hasNextPage() && !result) {
59+
comments = extractor.getPage(comments.getNextPage());
60+
result = findInComments(comments, commentContent);
61+
}
5062

51-
while (comments.hasNextPage() && !result) {
52-
comments = extractor.getPage(comments.getNextPage());
53-
result = findInComments(comments, "s1ck m3m3");
63+
return result;
5464
}
5565

56-
return result;
57-
}
66+
@Test
67+
public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException {
68+
assertTrue(getCommentsFromCommentsInfoHelper(urlYT));
69+
assertTrue(getCommentsFromCommentsInfoHelper(urlInvidious));
70+
}
5871

59-
@Test
60-
public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException {
61-
assertTrue(getCommentsFromCommentsInfoHelper(urlYT));
62-
assertTrue(getCommentsFromCommentsInfoHelper(urlInvidious));
63-
}
72+
private boolean getCommentsFromCommentsInfoHelper(String url) throws IOException, ExtractionException {
73+
final CommentsInfo commentsInfo = CommentsInfo.getInfo(url);
6474

65-
private boolean getCommentsFromCommentsInfoHelper(String url) throws IOException, ExtractionException {
66-
CommentsInfo commentsInfo = CommentsInfo.getInfo(url);
75+
assertEquals("Comments", commentsInfo.getName());
76+
boolean result = findInComments(commentsInfo.getRelatedItems(), commentContent);
6777

68-
assertEquals("Comments", commentsInfo.getName());
69-
boolean result = findInComments(commentsInfo.getRelatedItems(), "s1ck m3m3");
78+
Page nextPage = commentsInfo.getNextPage();
79+
InfoItemsPage<CommentsInfoItem> moreItems = new InfoItemsPage<>(null, nextPage, null);
80+
while (moreItems.hasNextPage() && !result) {
81+
moreItems = CommentsInfo.getMoreItems(YouTube, commentsInfo, nextPage);
82+
result = findInComments(moreItems.getItems(), commentContent);
83+
nextPage = moreItems.getNextPage();
84+
}
85+
return result;
86+
}
7087

71-
Page nextPage = commentsInfo.getNextPage();
72-
InfoItemsPage<CommentsInfoItem> moreItems = new InfoItemsPage<>(null, nextPage, null);
73-
while (moreItems.hasNextPage() && !result) {
74-
moreItems = CommentsInfo.getMoreItems(YouTube, commentsInfo, nextPage);
75-
result = findInComments(moreItems.getItems(), "s1ck m3m3");
76-
nextPage = moreItems.getNextPage();
88+
@Test
89+
public void testGetCommentsAllData() throws IOException, ExtractionException {
90+
InfoItemsPage<CommentsInfoItem> comments = extractorYT.getInitialPage();
91+
92+
DefaultTests.defaultTestListOfItems(YouTube, comments.getItems(), comments.getErrors());
93+
for (CommentsInfoItem c : comments.getItems()) {
94+
assertFalse(Utils.isBlank(c.getUploaderUrl()));
95+
assertFalse(Utils.isBlank(c.getUploaderName()));
96+
assertFalse(Utils.isBlank(c.getUploaderAvatarUrl()));
97+
assertFalse(Utils.isBlank(c.getCommentId()));
98+
assertFalse(Utils.isBlank(c.getCommentText()));
99+
assertFalse(Utils.isBlank(c.getName()));
100+
assertFalse(Utils.isBlank(c.getTextualUploadDate()));
101+
assertNotNull(c.getUploadDate());
102+
assertFalse(Utils.isBlank(c.getThumbnailUrl()));
103+
assertFalse(Utils.isBlank(c.getUrl()));
104+
assertFalse(c.getLikeCount() < 0);
105+
}
77106
}
78-
return result;
79-
}
80107

81-
@Test
82-
public void testGetCommentsAllData() throws IOException, ExtractionException {
83-
InfoItemsPage<CommentsInfoItem> comments = extractorYT.getInitialPage();
84-
85-
DefaultTests.defaultTestListOfItems(YouTube, comments.getItems(), comments.getErrors());
86-
for (CommentsInfoItem c : comments.getItems()) {
87-
assertFalse(Utils.isBlank(c.getUploaderUrl()));
88-
assertFalse(Utils.isBlank(c.getUploaderName()));
89-
assertFalse(Utils.isBlank(c.getUploaderAvatarUrl()));
90-
assertFalse(Utils.isBlank(c.getCommentId()));
91-
assertFalse(Utils.isBlank(c.getCommentText()));
92-
assertFalse(Utils.isBlank(c.getName()));
93-
assertFalse(Utils.isBlank(c.getTextualUploadDate()));
94-
assertNotNull(c.getUploadDate());
95-
assertFalse(Utils.isBlank(c.getThumbnailUrl()));
96-
assertFalse(Utils.isBlank(c.getUrl()));
97-
assertFalse(c.getLikeCount() < 0);
108+
private boolean findInComments(InfoItemsPage<CommentsInfoItem> comments, String comment) {
109+
return findInComments(comments.getItems(), comment);
98110
}
99-
}
100111

101-
private boolean findInComments(InfoItemsPage<CommentsInfoItem> comments, String comment) {
102-
return findInComments(comments.getItems(), comment);
112+
private boolean findInComments(List<CommentsInfoItem> comments, String comment) {
113+
for (CommentsInfoItem c : comments) {
114+
if (c.getCommentText().contains(comment)) {
115+
return true;
116+
}
117+
}
118+
return false;
119+
}
103120
}
104121

105-
private boolean findInComments(List<CommentsInfoItem> comments, String comment) {
106-
for (CommentsInfoItem c : comments) {
107-
if (c.getCommentText().contains(comment)) {
108-
return true;
122+
/**
123+
* Test a video with an empty comment
124+
*/
125+
public static class EmptyComment {
126+
private static YoutubeCommentsExtractor extractor;
127+
private final static String url = "https://www.youtube.com/watch?v=VM_6n762j6M";
128+
129+
@BeforeClass
130+
public static void setUp() throws Exception {
131+
NewPipe.init(DownloaderTestImpl.getInstance());
132+
extractor = (YoutubeCommentsExtractor) YouTube
133+
.getCommentsExtractor(url);
134+
extractor.fetchPage();
135+
}
136+
137+
@Test
138+
public void testGetCommentsAllData() throws IOException, ExtractionException {
139+
final InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
140+
141+
DefaultTests.defaultTestListOfItems(YouTube, comments.getItems(), comments.getErrors());
142+
for (CommentsInfoItem c : comments.getItems()) {
143+
assertFalse(Utils.isBlank(c.getUploaderUrl()));
144+
assertFalse(Utils.isBlank(c.getUploaderName()));
145+
assertFalse(Utils.isBlank(c.getUploaderAvatarUrl()));
146+
assertFalse(Utils.isBlank(c.getCommentId()));
147+
assertFalse(Utils.isBlank(c.getName()));
148+
assertFalse(Utils.isBlank(c.getTextualUploadDate()));
149+
assertNotNull(c.getUploadDate());
150+
assertFalse(Utils.isBlank(c.getThumbnailUrl()));
151+
assertFalse(Utils.isBlank(c.getUrl()));
152+
assertFalse(c.getLikeCount() < 0);
153+
if (c.getCommentId().equals("Ugga_h1-EXdHB3gCoAEC")) { // comment without text
154+
assertTrue(Utils.isBlank(c.getCommentText()));
155+
} else {
156+
assertFalse(Utils.isBlank(c.getCommentText()));
157+
}
109158
}
110159
}
111-
return false;
160+
112161
}
113162
}

0 commit comments

Comments
 (0)