Skip to content

Commit 8e27801

Browse files
yausernameyausername
committed
removed jackson and java 8
1 parent c1199c8 commit 8e27801

8 files changed

Lines changed: 231 additions & 123 deletions

File tree

‎build.gradle‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
allprojects {
22
apply plugin: 'java-library'
3-
sourceCompatibility = 1.8
4-
targetCompatibility = 1.8
3+
sourceCompatibility = 1.7
4+
targetCompatibility = 1.7
55

66
version 'v0.13.0'
77

‎extractor/build.gradle‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@ dependencies {
66
implementation 'org.mozilla:rhino:1.7.7.1'
77
implementation 'com.github.spotbugs:spotbugs-annotations:3.1.0'
88
implementation 'org.nibor.autolink:autolink:0.8.0'
9-
implementation 'com.fasterxml.jackson.core:jackson-databind:2.9.5'
109

1110
testImplementation 'junit:junit:4.12'
1211
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java‎

Lines changed: 37 additions & 115 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@
77
import java.util.HashMap;
88
import java.util.List;
99
import java.util.Map;
10-
import java.util.Optional;
1110

1211
import org.schabi.newpipe.extractor.DownloadResponse;
1312
import org.schabi.newpipe.extractor.Downloader;
@@ -21,9 +20,12 @@
2120
import org.schabi.newpipe.extractor.exceptions.ParsingException;
2221
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
2322
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
23+
import org.schabi.newpipe.extractor.utils.JsonUtils;
2424

25-
import com.fasterxml.jackson.databind.JsonNode;
26-
import com.fasterxml.jackson.databind.ObjectMapper;
25+
import com.grack.nanojson.JsonArray;
26+
import com.grack.nanojson.JsonObject;
27+
import com.grack.nanojson.JsonParser;
28+
import com.grack.nanojson.JsonParserException;
2729

2830
public class YoutubeCommentsExtractor extends CommentsExtractor {
2931

@@ -34,8 +36,6 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
3436
private String title;
3537
private InfoItemsPage<CommentsInfoItem> initPage;
3638

37-
private ObjectMapper mapper = new ObjectMapper();
38-
3939
public YoutubeCommentsExtractor(StreamingService service, ListLinkHandler uiHandler) {
4040
super(service, uiHandler);
4141
}
@@ -57,16 +57,24 @@ public String getNextPageUrl() throws IOException, ExtractionException {
5757
return initPage.getNextPageUrl();
5858
}
5959

60-
private String getNextPageUrl(JsonNode ajaxJson) throws IOException, ExtractionException {
61-
Optional<JsonNode> element = Optional.ofNullable(ajaxJson.findValue("itemSectionContinuation"))
62-
.map(e -> e.get("continuations")).map(e -> e.findValue("continuation"));
63-
64-
if (element.isPresent()) {
65-
return getNextPageUrl(element.get().asText());
66-
} else {
67-
// no more comments
60+
private String getNextPageUrl(JsonObject ajaxJson) throws IOException, ParsingException {
61+
62+
JsonArray arr;
63+
try {
64+
arr = JsonUtils.getValue(ajaxJson, "response.continuationContents.itemSectionContinuation.continuations");
65+
} catch (ParsingException e) {
66+
return "";
67+
}
68+
if(null == arr || arr.isEmpty()) {
6869
return "";
6970
}
71+
String continuation;
72+
try {
73+
continuation = JsonUtils.getValue(arr.getObject(0), "nextContinuationData.continuation");
74+
} catch (ParsingException e) {
75+
return "";
76+
}
77+
return getNextPageUrl(continuation);
7078
}
7179

7280
private String getNextPageUrl(String continuation) throws ParsingException {
@@ -88,121 +96,35 @@ public InfoItemsPage<CommentsInfoItem> getPage(String pageUrl) throws IOExceptio
8896
throw new ExtractionException(new IllegalArgumentException("Page url is empty or null"));
8997
}
9098
String ajaxResponse = makeAjaxRequest(pageUrl);
91-
JsonNode ajaxJson = mapper.readTree(ajaxResponse);
99+
JsonObject ajaxJson;
100+
try {
101+
ajaxJson = JsonParser.object().from(ajaxResponse);
102+
} catch (JsonParserException e) {
103+
throw new ParsingException("Could not parse json data for comments", e);
104+
}
92105
CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId());
93106
collectCommentsFrom(collector, ajaxJson, pageUrl);
94107
return new InfoItemsPage<>(collector, getNextPageUrl(ajaxJson));
95108
}
96109

97-
private void collectCommentsFrom(CommentsInfoItemsCollector collector, JsonNode ajaxJson, String pageUrl) {
110+
private void collectCommentsFrom(CommentsInfoItemsCollector collector, JsonObject ajaxJson, String pageUrl) throws ParsingException {
98111

99-
fetchTitle(ajaxJson);
100112

101-
List<JsonNode> comments = ajaxJson.findValues("commentRenderer");
102-
comments.stream().forEach(c -> {
103-
CommentsInfoItemExtractor extractor = new CommentsInfoItemExtractor() {
104-
105-
@Override
106-
public String getUrl() throws ParsingException {
107-
return pageUrl;
108-
}
109-
110-
@Override
111-
public String getThumbnailUrl() throws ParsingException {
112-
try {
113-
return c.get("authorThumbnail").get("thumbnails").get(2).get("url").asText();
114-
} catch (Exception e) {
115-
throw new ParsingException("Could not get thumbnail url", e);
116-
}
117-
}
118-
119-
@Override
120-
public String getName() throws ParsingException {
121-
try {
122-
return c.get("authorText").get("simpleText").asText();
123-
} catch (Exception e) {
124-
throw new ParsingException("Could not get author name", e);
125-
}
126-
}
127-
128-
@Override
129-
public String getPublishedTime() throws ParsingException {
130-
try {
131-
return c.get("publishedTimeText").get("runs").get(0).get("text").asText();
132-
} catch (Exception e) {
133-
throw new ParsingException("Could not get publishedTimeText", e);
134-
}
135-
}
136-
137-
@Override
138-
public Integer getLikeCount() throws ParsingException {
139-
try {
140-
return c.get("likeCount").intValue();
141-
} catch (Exception e) {
142-
throw new ParsingException("Could not get like count", e);
143-
}
144-
}
145-
146-
@Override
147-
public String getCommentText() throws ParsingException {
148-
try {
149-
if (null != c.get("contentText").get("simpleText")) {
150-
return c.get("contentText").get("simpleText").asText();
151-
} else {
152-
return c.get("contentText").get("runs").get(0).get("text").asText();
153-
}
154-
} catch (Exception e) {
155-
throw new ParsingException("Could not get comment text", e);
156-
}
157-
}
158-
159-
@Override
160-
public String getCommentId() throws ParsingException {
161-
try {
162-
return c.get("commentId").asText();
163-
} catch (Exception e) {
164-
throw new ParsingException("Could not get comment id", e);
165-
}
166-
}
167-
168-
@Override
169-
public String getAuthorThumbnail() throws ParsingException {
170-
try {
171-
return c.get("authorThumbnail").get("thumbnails").get(2).get("url").asText();
172-
} catch (Exception e) {
173-
throw new ParsingException("Could not get author thumbnail", e);
174-
}
175-
}
176-
177-
@Override
178-
public String getAuthorName() throws ParsingException {
179-
try {
180-
return c.get("authorText").get("simpleText").asText();
181-
} catch (Exception e) {
182-
throw new ParsingException("Could not get author name", e);
183-
}
184-
}
185-
186-
@Override
187-
public String getAuthorEndpoint() throws ParsingException {
188-
try {
189-
return "https://youtube.com"
190-
+ c.get("authorEndpoint").get("browseEndpoint").get("canonicalBaseUrl").asText();
191-
} catch (Exception e) {
192-
throw new ParsingException("Could not get author endpoint", e);
193-
}
194-
}
195-
};
196-
113+
JsonArray contents = JsonUtils.getValue(ajaxJson, "response.continuationContents.itemSectionContinuation.contents");
114+
fetchTitle(contents);
115+
List<JsonObject> comments = JsonUtils.getValues(contents, "commentThreadRenderer.comment.commentRenderer");
116+
117+
for(JsonObject c: comments) {
118+
CommentsInfoItemExtractor extractor = new YoutubeCommentsInfoItemExtractor(c, pageUrl);
197119
collector.commit(extractor);
198-
});
120+
}
199121

200122
}
201123

202-
private void fetchTitle(JsonNode ajaxJson) {
124+
private void fetchTitle(JsonArray contents) {
203125
if(null == title) {
204126
try {
205-
title = ajaxJson.findValue("commentTargetTitle").get("simpleText").asText();
127+
title = JsonUtils.getValue(contents.getObject(0), "commentThreadRenderer.commentTargetTitle.simpleText");
206128
} catch (Exception e) {
207129
title = "Youtube Comments";
208130
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
package org.schabi.newpipe.extractor.services.youtube.extractors;
2+
3+
import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor;
4+
import org.schabi.newpipe.extractor.exceptions.ParsingException;
5+
import org.schabi.newpipe.extractor.utils.JsonUtils;
6+
7+
import com.grack.nanojson.JsonArray;
8+
import com.grack.nanojson.JsonObject;
9+
10+
public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtractor{
11+
12+
private final JsonObject json;
13+
private final String url;
14+
15+
public YoutubeCommentsInfoItemExtractor(JsonObject json, String url) {
16+
this.json = json;
17+
this.url = url;
18+
}
19+
20+
@Override
21+
public String getUrl() throws ParsingException {
22+
return url;
23+
}
24+
25+
@Override
26+
public String getThumbnailUrl() throws ParsingException {
27+
JsonArray arr = JsonUtils.getValue(json, "authorThumbnail.thumbnails");
28+
return JsonUtils.getValue(arr.getObject(2), "url");
29+
}
30+
31+
@Override
32+
public String getName() throws ParsingException {
33+
return JsonUtils.getValue(json, "authorText.simpleText");
34+
}
35+
36+
@Override
37+
public String getPublishedTime() throws ParsingException {
38+
JsonArray arr = JsonUtils.getValue(json, "publishedTimeText.runs");
39+
return JsonUtils.getValue(arr.getObject(0), "text");
40+
}
41+
42+
@Override
43+
public Integer getLikeCount() throws ParsingException {
44+
return JsonUtils.getValue(json, "likeCount");
45+
}
46+
47+
@Override
48+
public String getCommentText() throws ParsingException {
49+
try {
50+
return JsonUtils.getValue(json, "contentText.simpleText");
51+
} catch (Exception e) {
52+
JsonArray arr = JsonUtils.getValue(json, "contentText.runs");
53+
return JsonUtils.getValue(arr.getObject(0), "text");
54+
}
55+
}
56+
57+
@Override
58+
public String getCommentId() throws ParsingException {
59+
return JsonUtils.getValue(json, "commentId");
60+
}
61+
62+
@Override
63+
public String getAuthorThumbnail() throws ParsingException {
64+
JsonArray arr = JsonUtils.getValue(json, "authorThumbnail.thumbnails");
65+
return JsonUtils.getValue(arr.getObject(2), "url");
66+
}
67+
68+
@Override
69+
public String getAuthorName() throws ParsingException {
70+
return JsonUtils.getValue(json, "authorText.simpleText");
71+
}
72+
73+
@Override
74+
public String getAuthorEndpoint() throws ParsingException {
75+
return "https://youtube.com" + JsonUtils.getValue(json, "authorEndpoint.browseEndpoint.canonicalBaseUrl");
76+
}
77+
78+
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
package org.schabi.newpipe.extractor.utils;
2+
3+
import java.util.ArrayList;
4+
import java.util.Arrays;
5+
import java.util.List;
6+
7+
import javax.annotation.Nonnull;
8+
import javax.annotation.Nullable;
9+
10+
import org.schabi.newpipe.extractor.exceptions.ParsingException;
11+
12+
import com.grack.nanojson.JsonArray;
13+
import com.grack.nanojson.JsonObject;
14+
15+
public class JsonUtils {
16+
17+
private JsonUtils() {
18+
}
19+
20+
@Nonnull
21+
public static <T> T getValue(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{
22+
23+
List<String> keys = Arrays.asList(path.split("\\."));
24+
object = getObject(object, keys.subList(0, keys.size() - 1));
25+
if (null == object) throw new ParsingException("Unable to get " + path);
26+
T result = (T) object.get(keys.get(keys.size() - 1));
27+
if(null == result) throw new ParsingException("Unable to get " + path);
28+
return result;
29+
}
30+
31+
32+
@Nonnull
33+
public static <T> List<T> getValues(@Nonnull JsonArray array, @Nonnull String path) throws ParsingException {
34+
35+
List<T> result = new ArrayList<>();
36+
for (int i = 0; i < array.size(); i++) {
37+
JsonObject obj = array.getObject(i);
38+
result.add((T)getValue(obj, path));
39+
}
40+
return result;
41+
}
42+
43+
@Nullable
44+
private static JsonObject getObject(@Nonnull JsonObject object, @Nonnull List<String> keys) {
45+
JsonObject result = object;
46+
for (String key : keys) {
47+
result = result.getObject(key);
48+
if (null == result) break;
49+
}
50+
return result;
51+
}
52+
53+
}

‎extractor/src/test/java/org/schabi/newpipe/Downloader.java‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -177,15 +177,17 @@ public DownloadResponse get(String siteUrl, Map<String, List<String>> requestHea
177177
URL url = new URL(siteUrl);
178178
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
179179
for (Map.Entry<String, List<String>> pair : requestHeaders.entrySet()) {
180-
pair.getValue().stream().forEach(value -> con.addRequestProperty(pair.getKey(), value));
180+
for(String value: pair.getValue()) {
181+
con.addRequestProperty(pair.getKey(), value);
182+
}
181183
}
182184
String responseBody = dl(con);
183185
return new DownloadResponse(responseBody, con.getHeaderFields());
184186
}
185187

186188
@Override
187189
public DownloadResponse get(String siteUrl) throws IOException, ReCaptchaException {
188-
return get(siteUrl, Collections.emptyMap());
190+
return get(siteUrl, Collections.EMPTY_MAP);
189191
}
190192

191193
@Override
@@ -195,7 +197,9 @@ public DownloadResponse post(String siteUrl, String requestBody, Map<String, Lis
195197
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
196198
con.setRequestMethod("POST");
197199
for (Map.Entry<String, List<String>> pair : requestHeaders.entrySet()) {
198-
pair.getValue().stream().forEach(value -> con.addRequestProperty(pair.getKey(), value));
200+
for(String value: pair.getValue()) {
201+
con.addRequestProperty(pair.getKey(), value);
202+
}
199203
}
200204
// set fields to default if not set already
201205
setDefaults(con);

0 commit comments

Comments
 (0)