Skip to content

Commit 8de5311

Browse files
theScrabitheScrabi
authored
Merge pull request #101 from yausername/master
added comments extractor
2 parents 04460bb + 021e4e0 commit 8de5311

24 files changed

Lines changed: 1311 additions & 105 deletions

‎extractor/src/main/java/org/schabi/newpipe/extractor/DownloadRequest.java‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
package org.schabi.newpipe.extractor;
2+
3+
import java.util.Collections;
4+
import java.util.List;
5+
import java.util.Map;
6+
7+
public class DownloadRequest {
8+
9+
private final String requestBody;
10+
private final Map<String, List<String>> requestHeaders;
11+
public static final DownloadRequest emptyRequest = new DownloadRequest(null, null);
12+
13+
public DownloadRequest(String requestBody, Map<String, List<String>> headers) {
14+
super();
15+
this.requestBody = requestBody;
16+
if(null != headers) {
17+
this.requestHeaders = headers;
18+
}else {
19+
this.requestHeaders = Collections.emptyMap();
20+
}
21+
}
22+
23+
public String getRequestBody() {
24+
return requestBody;
25+
}
26+
27+
public Map<String, List<String>> getRequestHeaders() {
28+
return requestHeaders;
29+
}
30+
31+
public void setRequestCookies(List<String> cookies){
32+
requestHeaders.put("Cookie", cookies);
33+
}
34+
35+
public List<String> getRequestCookies(){
36+
if(null == requestHeaders) return Collections.emptyList();
37+
List<String> cookies = requestHeaders.get("Cookie");
38+
if(null == cookies)
39+
return Collections.emptyList();
40+
else
41+
return cookies;
42+
}
43+
44+
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/DownloadResponse.java‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
package org.schabi.newpipe.extractor;
2+
3+
import java.util.Collections;
4+
import java.util.List;
5+
import java.util.Map;
6+
7+
import javax.annotation.Nonnull;
8+
9+
public class DownloadResponse {
10+
private final String responseBody;
11+
private final Map<String, List<String>> responseHeaders;
12+
13+
public DownloadResponse(String responseBody, Map<String, List<String>> headers) {
14+
super();
15+
this.responseBody = responseBody;
16+
this.responseHeaders = headers;
17+
}
18+
19+
public String getResponseBody() {
20+
return responseBody;
21+
}
22+
23+
public Map<String, List<String>> getResponseHeaders() {
24+
return responseHeaders;
25+
}
26+
27+
@Nonnull
28+
public List<String> getResponseCookies(){
29+
if(null == responseHeaders) return Collections.emptyList();
30+
List<String> cookies = responseHeaders.get("Set-Cookie");
31+
if(null == cookies)
32+
return Collections.emptyList();
33+
else
34+
return cookies;
35+
}
36+
37+
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/Downloader.java‎

Lines changed: 17 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
11
package org.schabi.newpipe.extractor;
22

3-
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
4-
import org.schabi.newpipe.extractor.utils.Localization;
5-
63
import java.io.IOException;
74
import java.util.Map;
85

6+
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
7+
import org.schabi.newpipe.extractor.utils.Localization;
8+
99
/*
1010
* Created by Christian Schabesberger on 28.01.16.
1111
*
@@ -29,8 +29,8 @@
2929
public interface Downloader {
3030

3131
/**
32-
* Download the text file at the supplied URL as in download(String),
33-
* but set the HTTP header field "Accept-Language" to the supplied string.
32+
* Download the text file at the supplied URL as in download(String), but set
33+
* the HTTP header field "Accept-Language" to the supplied string.
3434
*
3535
* @param siteUrl the URL of the text file to return the contents of
3636
* @param localization the language and country (usually a 2-character code for each)
@@ -40,8 +40,8 @@ public interface Downloader {
4040
String download(String siteUrl, Localization localization) throws IOException, ReCaptchaException;
4141

4242
/**
43-
* Download the text file at the supplied URL as in download(String),
44-
* but set the HTTP header field "Accept-Language" to the supplied string.
43+
* Download the text file at the supplied URL as in download(String), but set
44+
* the HTTP header field "Accept-Language" to the supplied string.
4545
*
4646
* @param siteUrl the URL of the text file to return the contents of
4747
* @param customProperties set request header properties
@@ -51,12 +51,20 @@ public interface Downloader {
5151
String download(String siteUrl, Map<String, String> customProperties) throws IOException, ReCaptchaException;
5252

5353
/**
54-
* Download (via HTTP) the text file located at the supplied URL, and return its contents.
55-
* Primarily intended for downloading web pages.
54+
* Download (via HTTP) the text file located at the supplied URL, and return its
55+
* contents. Primarily intended for downloading web pages.
5656
*
5757
* @param siteUrl the URL of the text file to download
5858
* @return the contents of the specified text file
5959
* @throws IOException
6060
*/
6161
String download(String siteUrl) throws IOException, ReCaptchaException;
62+
63+
DownloadResponse get(String siteUrl, DownloadRequest request)
64+
throws IOException, ReCaptchaException;
65+
66+
DownloadResponse get(String siteUrl) throws IOException, ReCaptchaException;
67+
68+
DownloadResponse post(String siteUrl, DownloadRequest request)
69+
throws IOException, ReCaptchaException;
6270
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,9 @@
88
import javax.annotation.Nonnull;
99
import javax.annotation.Nullable;
1010
import java.io.IOException;
11+
import java.io.Serializable;
1112

12-
public abstract class Extractor {
13+
public abstract class Extractor{
1314
/**
1415
* {@link StreamingService} currently related to this extractor.<br>
1516
* Useful for getting other things from a service (like the url handlers for cleaning/accepting/get id from urls).

‎extractor/src/main/java/org/schabi/newpipe/extractor/InfoItem.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@ public String toString() {
6868
public enum InfoType {
6969
STREAM,
7070
PLAYLIST,
71-
CHANNEL
71+
CHANNEL,
72+
COMMENT
7273
}
7374
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java‎

Lines changed: 28 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,25 @@
11
package org.schabi.newpipe.extractor;
22

3+
import java.util.Collections;
4+
import java.util.List;
5+
36
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
7+
import org.schabi.newpipe.extractor.comments.CommentsExtractor;
48
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
59
import org.schabi.newpipe.extractor.exceptions.ParsingException;
610
import org.schabi.newpipe.extractor.kiosk.KioskList;
11+
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
12+
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
13+
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
14+
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
15+
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
16+
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
717
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
818
import org.schabi.newpipe.extractor.search.SearchExtractor;
9-
import org.schabi.newpipe.extractor.linkhandler.*;
1019
import org.schabi.newpipe.extractor.stream.StreamExtractor;
1120
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
1221
import org.schabi.newpipe.extractor.utils.Localization;
1322

14-
import java.util.Collections;
15-
import java.util.List;
16-
1723
/*
1824
* Copyright (C) Christian Schabesberger 2018 <chris.schabesberger@mailbox.org>
1925
* StreamingService.java is part of NewPipe.
@@ -60,7 +66,7 @@ public List<MediaCapability> getMediaCapabilities() {
6066
}
6167

6268
public enum MediaCapability {
63-
AUDIO, VIDEO, LIVE
69+
AUDIO, VIDEO, LIVE, COMMENTS
6470
}
6571
}
6672

@@ -135,6 +141,7 @@ public String toString() {
135141
* @return an instance of a SearchQueryHandlerFactory
136142
*/
137143
public abstract SearchQueryHandlerFactory getSearchQHFactory();
144+
public abstract ListLinkHandlerFactory getCommentsLHFactory();
138145

139146

140147
////////////////////////////////////////////
@@ -198,6 +205,8 @@ public abstract PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandl
198205
*/
199206
public abstract StreamExtractor getStreamExtractor(LinkHandler linkHandler,
200207
Localization localization) throws ExtractionException;
208+
public abstract CommentsExtractor getCommentsExtractor(ListLinkHandler linkHandler,
209+
Localization localization) throws ExtractionException;
201210
////////////////////////////////////////////
202211
// Extractor with default localization
203212
////////////////////////////////////////////
@@ -213,14 +222,18 @@ public SuggestionExtractor getSuggestionExtractor() {
213222
public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler) throws ExtractionException {
214223
return getChannelExtractor(linkHandler, NewPipe.getPreferredLocalization());
215224
}
216-
225+
217226
public PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler) throws ExtractionException {
218227
return getPlaylistExtractor(linkHandler, NewPipe.getPreferredLocalization());
219228
}
220229

221230
public StreamExtractor getStreamExtractor(LinkHandler linkHandler) throws ExtractionException {
222231
return getStreamExtractor(linkHandler, NewPipe.getPreferredLocalization());
223232
}
233+
234+
public CommentsExtractor getCommentsExtractor(ListLinkHandler urlIdHandler) throws ExtractionException {
235+
return getCommentsExtractor(urlIdHandler, NewPipe.getPreferredLocalization());
236+
}
224237

225238
////////////////////////////////////////////
226239
// Extractor without link handler
@@ -274,6 +287,15 @@ public PlaylistExtractor getPlaylistExtractor(String url) throws ExtractionExcep
274287
public StreamExtractor getStreamExtractor(String url) throws ExtractionException {
275288
return getStreamExtractor(getStreamLHFactory().fromUrl(url), NewPipe.getPreferredLocalization());
276289
}
290+
291+
public CommentsExtractor getCommentsExtractor(String url) throws ExtractionException {
292+
ListLinkHandlerFactory llhf = getCommentsLHFactory();
293+
if(null == llhf) {
294+
return null;
295+
}
296+
return getCommentsExtractor(llhf.fromUrl(url), NewPipe.getPreferredLocalization());
297+
}
298+
277299

278300
/**
279301
* Figures out where the link is pointing to (a channel, a video, a playlist, etc.)

‎extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsExtractor.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
package org.schabi.newpipe.extractor.comments;
2+
3+
import org.schabi.newpipe.extractor.ListExtractor;
4+
import org.schabi.newpipe.extractor.StreamingService;
5+
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
6+
import org.schabi.newpipe.extractor.utils.Localization;
7+
8+
public abstract class CommentsExtractor extends ListExtractor<CommentsInfoItem> {
9+
10+
public CommentsExtractor(StreamingService service, ListLinkHandler uiHandler, Localization localization) {
11+
super(service, uiHandler, localization);
12+
// TODO Auto-generated constructor stub
13+
}
14+
15+
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfo.java‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
package org.schabi.newpipe.extractor.comments;
2+
3+
import java.io.IOException;
4+
5+
import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage;
6+
import org.schabi.newpipe.extractor.ListInfo;
7+
import org.schabi.newpipe.extractor.NewPipe;
8+
import org.schabi.newpipe.extractor.StreamingService;
9+
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
10+
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
11+
import org.schabi.newpipe.extractor.utils.ExtractorHelper;
12+
13+
public class CommentsInfo extends ListInfo<CommentsInfoItem>{
14+
15+
private CommentsInfo(int serviceId, ListLinkHandler listUrlIdHandler, String name) {
16+
super(serviceId, listUrlIdHandler, name);
17+
// TODO Auto-generated constructor stub
18+
}
19+
20+
public static CommentsInfo getInfo(String url) throws IOException, ExtractionException {
21+
return getInfo(NewPipe.getServiceByUrl(url), url);
22+
}
23+
24+
public static CommentsInfo getInfo(StreamingService serviceByUrl, String url) throws ExtractionException, IOException {
25+
return getInfo(serviceByUrl.getCommentsExtractor(url));
26+
}
27+
28+
private static CommentsInfo getInfo(CommentsExtractor commentsExtractor) throws IOException, ExtractionException {
29+
// for services which do not have a comments extractor
30+
if (null == commentsExtractor) {
31+
return null;
32+
}
33+
34+
commentsExtractor.fetchPage();
35+
String name = commentsExtractor.getName();
36+
int serviceId = commentsExtractor.getServiceId();
37+
ListLinkHandler listUrlIdHandler = commentsExtractor.getLinkHandler();
38+
CommentsInfo commentsInfo = new CommentsInfo(serviceId, listUrlIdHandler, name);
39+
commentsInfo.setCommentsExtractor(commentsExtractor);
40+
InfoItemsPage<CommentsInfoItem> initialCommentsPage = ExtractorHelper.getItemsPageOrLogError(commentsInfo,
41+
commentsExtractor);
42+
commentsInfo.setRelatedItems(initialCommentsPage.getItems());
43+
commentsInfo.setNextPageUrl(initialCommentsPage.getNextPageUrl());
44+
45+
return commentsInfo;
46+
}
47+
48+
public static InfoItemsPage<CommentsInfoItem> getMoreItems(CommentsInfo commentsInfo, String pageUrl)
49+
throws ExtractionException, IOException {
50+
return getMoreItems(NewPipe.getService(commentsInfo.getServiceId()), commentsInfo, pageUrl);
51+
}
52+
53+
public static InfoItemsPage<CommentsInfoItem> getMoreItems(StreamingService service, CommentsInfo commentsInfo,
54+
String pageUrl) throws IOException, ExtractionException {
55+
if (null == commentsInfo.getCommentsExtractor()) {
56+
commentsInfo.setCommentsExtractor(service.getCommentsExtractor(commentsInfo.getUrl()));
57+
commentsInfo.getCommentsExtractor().fetchPage();
58+
}
59+
return commentsInfo.getCommentsExtractor().getPage(pageUrl);
60+
}
61+
62+
private transient CommentsExtractor commentsExtractor;
63+
64+
public CommentsExtractor getCommentsExtractor() {
65+
return commentsExtractor;
66+
}
67+
68+
public void setCommentsExtractor(CommentsExtractor commentsExtractor) {
69+
this.commentsExtractor = commentsExtractor;
70+
}
71+
72+
}

0 commit comments

Comments
 (0)