Skip to content

Commit 1b52230

Browse files
wb9688wb9688
authored
Merge pull request #308 from Royosef/PeerTubeChannelExtractor
[PeerTube] channels support
2 parents a5155fb + 4afe657 commit 1b52230

10 files changed

Lines changed: 444 additions & 30 deletions

File tree

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,12 @@ public SubscriptionExtractor getSubscriptionExtractor() {
7777
@Override
7878
public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler)
7979
throws ExtractionException {
80-
return new PeertubeChannelExtractor(this, linkHandler);
80+
81+
if (linkHandler.getUrl().contains("/video-channels/")) {
82+
return new PeertubeChannelExtractor(this, linkHandler);
83+
} else {
84+
return new PeertubeAccountExtractor(this, linkHandler);
85+
}
8186
}
8287

8388
@Override

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java‎

Lines changed: 187 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,187 @@
1+
package org.schabi.newpipe.extractor.services.peertube.extractors;
2+
3+
import com.grack.nanojson.JsonArray;
4+
import com.grack.nanojson.JsonObject;
5+
import com.grack.nanojson.JsonParser;
6+
import com.grack.nanojson.JsonParserException;
7+
import org.jsoup.helper.StringUtil;
8+
import org.schabi.newpipe.extractor.StreamingService;
9+
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
10+
import org.schabi.newpipe.extractor.downloader.Downloader;
11+
import org.schabi.newpipe.extractor.downloader.Response;
12+
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
13+
import org.schabi.newpipe.extractor.exceptions.ParsingException;
14+
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
15+
import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
16+
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
17+
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
18+
import org.schabi.newpipe.extractor.utils.JsonUtils;
19+
import org.schabi.newpipe.extractor.utils.Parser;
20+
import org.schabi.newpipe.extractor.utils.Parser.RegexException;
21+
22+
import java.io.IOException;
23+
24+
public class PeertubeAccountExtractor extends ChannelExtractor {
25+
26+
private static final String START_KEY = "start";
27+
private static final String COUNT_KEY = "count";
28+
private static final int ITEMS_PER_PAGE = 12;
29+
private static final String START_PATTERN = "start=(\\d*)";
30+
31+
private InfoItemsPage<StreamInfoItem> initPage;
32+
private long total;
33+
34+
private JsonObject json;
35+
private final String baseUrl;
36+
37+
public PeertubeAccountExtractor(StreamingService service, ListLinkHandler linkHandler) throws ParsingException {
38+
super(service, linkHandler);
39+
this.baseUrl = getBaseUrl();
40+
}
41+
42+
@Override
43+
public String getAvatarUrl() throws ParsingException {
44+
String value;
45+
try {
46+
value = JsonUtils.getString(json, "avatar.path");
47+
} catch (Exception e) {
48+
value = "/client/assets/images/default-avatar.png";
49+
}
50+
return baseUrl + value;
51+
}
52+
53+
@Override
54+
public String getBannerUrl() throws ParsingException {
55+
return null;
56+
}
57+
58+
@Override
59+
public String getFeedUrl() throws ParsingException {
60+
return getBaseUrl() + "/feeds/videos.xml?accountId=" + json.get("id");
61+
}
62+
63+
@Override
64+
public long getSubscriberCount() throws ParsingException {
65+
Number number = JsonUtils.getNumber(json, "followersCount");
66+
return number.longValue();
67+
}
68+
69+
@Override
70+
public String getDescription() throws ParsingException {
71+
try {
72+
return JsonUtils.getString(json, "description");
73+
} catch (ParsingException e) {
74+
return "No description";
75+
}
76+
}
77+
78+
@Override
79+
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
80+
super.fetchPage();
81+
return initPage;
82+
}
83+
84+
private void collectStreamsFrom(StreamInfoItemsCollector collector, JsonObject json, String pageUrl) throws ParsingException {
85+
JsonArray contents;
86+
try {
87+
contents = (JsonArray) JsonUtils.getValue(json, "data");
88+
} catch (Exception e) {
89+
throw new ParsingException("unable to extract channel streams", e);
90+
}
91+
92+
for (Object c : contents) {
93+
if (c instanceof JsonObject) {
94+
final JsonObject item = (JsonObject) c;
95+
PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item, baseUrl);
96+
collector.commit(extractor);
97+
}
98+
}
99+
100+
}
101+
102+
@Override
103+
public String getNextPageUrl() throws IOException, ExtractionException {
104+
super.fetchPage();
105+
return initPage.getNextPageUrl();
106+
}
107+
108+
@Override
109+
public InfoItemsPage<StreamInfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
110+
Response response = getDownloader().get(pageUrl);
111+
JsonObject json = null;
112+
if (null != response && !StringUtil.isBlank(response.responseBody())) {
113+
try {
114+
json = JsonParser.object().from(response.responseBody());
115+
} catch (Exception e) {
116+
throw new ParsingException("Could not parse json data for kiosk info", e);
117+
}
118+
}
119+
120+
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
121+
if (json != null) {
122+
PeertubeParsingHelper.validate(json);
123+
Number number = JsonUtils.getNumber(json, "total");
124+
if (number != null) this.total = number.longValue();
125+
collectStreamsFrom(collector, json, pageUrl);
126+
} else {
127+
throw new ExtractionException("Unable to get PeerTube kiosk info");
128+
}
129+
return new InfoItemsPage<>(collector, getNextPageUrl(pageUrl));
130+
}
131+
132+
133+
private String getNextPageUrl(String prevPageUrl) {
134+
String prevStart;
135+
try {
136+
prevStart = Parser.matchGroup1(START_PATTERN, prevPageUrl);
137+
} catch (RegexException e) {
138+
return "";
139+
}
140+
if (StringUtil.isBlank(prevStart)) return "";
141+
long nextStart = 0;
142+
try {
143+
nextStart = Long.valueOf(prevStart) + ITEMS_PER_PAGE;
144+
} catch (NumberFormatException e) {
145+
return "";
146+
}
147+
148+
if (nextStart >= total) {
149+
return "";
150+
} else {
151+
return prevPageUrl.replace(START_KEY + "=" + prevStart, START_KEY + "=" + String.valueOf(nextStart));
152+
}
153+
}
154+
155+
@Override
156+
public void onFetchPage(Downloader downloader) throws IOException, ExtractionException {
157+
Response response = downloader.get(getUrl());
158+
if (null != response && null != response.responseBody()) {
159+
setInitialData(response.responseBody());
160+
} else {
161+
throw new ExtractionException("Unable to extract PeerTube channel data");
162+
}
163+
164+
String pageUrl = getUrl() + "/videos?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE;
165+
this.initPage = getPage(pageUrl);
166+
}
167+
168+
private void setInitialData(String responseBody) throws ExtractionException {
169+
try {
170+
json = JsonParser.object().from(responseBody);
171+
} catch (JsonParserException e) {
172+
throw new ExtractionException("Unable to extract peertube channel data", e);
173+
}
174+
if (json == null) throw new ExtractionException("Unable to extract PeerTube channel data");
175+
}
176+
177+
@Override
178+
public String getName() throws ParsingException {
179+
return JsonUtils.getString(json, "displayName");
180+
}
181+
182+
@Override
183+
public String getOriginalUrl() throws ParsingException {
184+
return baseUrl + "/" + getId();
185+
}
186+
187+
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ public String getBannerUrl() throws ParsingException {
5757

5858
@Override
5959
public String getFeedUrl() throws ParsingException {
60-
return getBaseUrl() + "/feeds/videos.xml?accountId=" + json.get("id");
60+
return getBaseUrl() + "/feeds/videos.xml?videoChannelId=" + json.get("id");
6161
}
6262

6363
@Override
@@ -181,7 +181,7 @@ public String getName() throws ParsingException {
181181

182182
@Override
183183
public String getOriginalUrl() throws ParsingException {
184-
return baseUrl + "/accounts/" + getId();
184+
return baseUrl + "/" + getId();
185185
}
186186

187187
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ public String getAuthorName() throws ParsingException {
9797
public String getAuthorEndpoint() throws ParsingException {
9898
String name = JsonUtils.getString(item, "account.name");
9999
String host = JsonUtils.getString(item, "account.host");
100-
return ServiceList.PeerTube.getChannelLHFactory().fromId(name + "@" + host, baseUrl).getUrl();
100+
return ServiceList.PeerTube.getChannelLHFactory().fromId("accounts/" + name + "@" + host, baseUrl).getUrl();
101101
}
102102

103103
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -128,7 +128,7 @@ public long getDislikeCount() throws ParsingException {
128128
public String getUploaderUrl() throws ParsingException {
129129
String name = JsonUtils.getString(json, "account.name");
130130
String host = JsonUtils.getString(json, "account.host");
131-
return getService().getChannelLHFactory().fromId(name + "@" + host, baseUrl).getUrl();
131+
return getService().getChannelLHFactory().fromId("accounts/" + name + "@" + host, baseUrl).getUrl();
132132
}
133133

134134
@Override

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,8 @@ public long getViewCount() throws ParsingException {
5151
public String getUploaderUrl() throws ParsingException {
5252
String name = JsonUtils.getString(item, "account.name");
5353
String host = JsonUtils.getString(item, "account.host");
54-
return ServiceList.PeerTube.getChannelLHFactory().fromId(name + "@" + host, baseUrl).getUrl();
54+
55+
return ServiceList.PeerTube.getChannelLHFactory().fromId("accounts/" + name + "@" + host, baseUrl).getUrl();
5556
}
5657

5758
@Override

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeChannelLinkHandlerFactory.java‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,16 +10,16 @@
1010
public class PeertubeChannelLinkHandlerFactory extends ListLinkHandlerFactory {
1111

1212
private static final PeertubeChannelLinkHandlerFactory instance = new PeertubeChannelLinkHandlerFactory();
13-
private static final String ID_PATTERN = "/accounts/([^/?&#]*)";
14-
private static final String ACCOUNTS_ENDPOINT = "/api/v1/accounts/";
13+
private static final String ID_PATTERN = "(accounts|video-channels)/([^/?&#]*)";
14+
private static final String API_ENDPOINT = "/api/v1/";
1515

1616
public static PeertubeChannelLinkHandlerFactory getInstance() {
1717
return instance;
1818
}
1919

2020
@Override
2121
public String getId(String url) throws ParsingException {
22-
return Parser.matchGroup1(ID_PATTERN, url);
22+
return Parser.matchGroup(ID_PATTERN, url, 0);
2323
}
2424

2525
@Override
@@ -31,11 +31,17 @@ public String getUrl(String id, List<String> contentFilters, String searchFilter
3131
@Override
3232
public String getUrl(String id, List<String> contentFilter, String sortFilter, String baseUrl)
3333
throws ParsingException {
34-
return baseUrl + ACCOUNTS_ENDPOINT + id;
34+
35+
if (id.matches(ID_PATTERN)) {
36+
return baseUrl + API_ENDPOINT + id;
37+
} else {
38+
// This is needed for compatibility with older versions were we didn't support video channels yet
39+
return baseUrl + API_ENDPOINT + "accounts/" + id;
40+
}
3541
}
3642

3743
@Override
3844
public boolean onAcceptUrl(String url) {
39-
return url.contains("/accounts/");
45+
return url.contains("/accounts/") || url.contains("/video-channels/");
4046
}
4147
}

0 commit comments

Comments
 (0)