Skip to content

Commit 14c0c37

Browse files
TobiGrTobiGr
authored
Merge pull request #309 from Stypox/improve-stream-tests
Improve stream extractor tests & various fixes
2 parents df26f61 + 57e7994 commit 14c0c37

38 files changed

Lines changed: 1396 additions & 1417 deletions

File tree

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceExtractor.java‎

Lines changed: 6 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
import org.schabi.newpipe.extractor.exceptions.ParsingException;
1414
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
1515
import org.schabi.newpipe.extractor.services.media_ccc.extractors.infoItems.MediaCCCStreamInfoItemExtractor;
16+
import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferenceLinkHandlerFactory;
1617
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
1718
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
1819

@@ -71,8 +72,8 @@ public String getParentChannelAvatarUrl() throws ParsingException {
7172
@Nonnull
7273
@Override
7374
public InfoItemsPage<StreamInfoItem> getInitialPage() {
74-
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
75-
JsonArray events = conferenceData.getArray("events");
75+
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
76+
final JsonArray events = conferenceData.getArray("events");
7677
for (int i = 0; i < events.size(); i++) {
7778
collector.commit(new MediaCCCStreamInfoItemExtractor(events.getObject(i)));
7879
}
@@ -87,10 +88,11 @@ public InfoItemsPage<StreamInfoItem> getPage(final Page page) {
8788
@Override
8889
public void onFetchPage(@Nonnull final Downloader downloader)
8990
throws IOException, ExtractionException {
91+
final String conferenceUrl = MediaCCCConferenceLinkHandlerFactory.CONFERENCE_API_ENDPOINT + getId();
9092
try {
91-
conferenceData = JsonParser.object().from(downloader.get(getUrl()).responseBody());
93+
conferenceData = JsonParser.object().from(downloader.get(conferenceUrl).responseBody());
9294
} catch (JsonParserException jpe) {
93-
throw new ExtractionException("Could not parse json returnd by url: " + getUrl());
95+
throw new ExtractionException("Could not parse json returnd by url: " + conferenceUrl);
9496
}
9597
}
9698

@@ -99,10 +101,4 @@ public void onFetchPage(@Nonnull final Downloader downloader)
99101
public String getName() throws ParsingException {
100102
return conferenceData.getString("title");
101103
}
102-
103-
@Nonnull
104-
@Override
105-
public String getOriginalUrl() {
106-
return "https://media.ccc.de/c/" + conferenceData.getString("acronym");
107-
}
108104
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCParsingHelper.java‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,17 @@
66
import java.text.SimpleDateFormat;
77
import java.util.Calendar;
88
import java.util.Date;
9+
import java.util.TimeZone;
910

1011
public final class MediaCCCParsingHelper {
1112
private MediaCCCParsingHelper() { }
1213

1314
public static Calendar parseDateFrom(final String textualUploadDate) throws ParsingException {
14-
Date date;
15+
final Date date;
1516
try {
16-
date = new SimpleDateFormat("yyyy-MM-dd").parse(textualUploadDate);
17+
final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
18+
sdf.setTimeZone(TimeZone.getTimeZone("GMT"));
19+
date = sdf.parse(textualUploadDate);
1720
} catch (ParseException e) {
1821
throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e);
1922
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java‎

Lines changed: 21 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -19,14 +19,18 @@
1919
import org.schabi.newpipe.extractor.stream.StreamType;
2020
import org.schabi.newpipe.extractor.stream.SubtitlesStream;
2121
import org.schabi.newpipe.extractor.stream.VideoStream;
22+
import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferenceLinkHandlerFactory;
23+
import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCStreamLinkHandlerFactory;
2224

2325
import java.io.IOException;
2426
import java.util.ArrayList;
27+
import java.util.Arrays;
2528
import java.util.Collections;
2629
import java.util.List;
2730
import java.util.Locale;
2831

2932
import javax.annotation.Nonnull;
33+
import javax.annotation.Nullable;
3034

3135
public class MediaCCCStreamExtractor extends StreamExtractor {
3236
private JsonObject data;
@@ -93,7 +97,7 @@ public long getDislikeCount() {
9397
@Nonnull
9498
@Override
9599
public String getUploaderUrl() {
96-
return data.getString("conference_url");
100+
return MediaCCCConferenceLinkHandlerFactory.CONFERENCE_PATH + getUploaderName();
97101
}
98102

99103
@Nonnull
@@ -111,25 +115,25 @@ public String getUploaderAvatarUrl() {
111115

112116
@Nonnull
113117
@Override
114-
public String getSubChannelUrl() throws ParsingException {
118+
public String getSubChannelUrl() {
115119
return "";
116120
}
117121

118122
@Nonnull
119123
@Override
120-
public String getSubChannelName() throws ParsingException {
124+
public String getSubChannelName() {
121125
return "";
122126
}
123127

124128
@Nonnull
125129
@Override
126-
public String getSubChannelAvatarUrl() throws ParsingException {
130+
public String getSubChannelAvatarUrl() {
127131
return "";
128132
}
129133

130134
@Nonnull
131135
@Override
132-
public String getDashMpdUrl() throws ParsingException {
136+
public String getDashMpdUrl() {
133137
return "";
134138
}
135139

@@ -194,7 +198,7 @@ public List<VideoStream> getVideoStreams() throws ExtractionException {
194198

195199
@Override
196200
public List<VideoStream> getVideoOnlyStreams() {
197-
return null;
201+
return Collections.emptyList();
198202
}
199203

200204
@Nonnull
@@ -214,9 +218,10 @@ public StreamType getStreamType() {
214218
return StreamType.VIDEO_STREAM;
215219
}
216220

221+
@Nullable
217222
@Override
218223
public StreamInfoItemsCollector getRelatedStreams() {
219-
return new StreamInfoItemsCollector(getServiceId());
224+
return null;
220225
}
221226

222227
@Override
@@ -227,14 +232,13 @@ public String getErrorMessage() {
227232
@Override
228233
public void onFetchPage(@Nonnull final Downloader downloader)
229234
throws IOException, ExtractionException {
235+
final String videoUrl = MediaCCCStreamLinkHandlerFactory.VIDEO_API_ENDPOINT + getId();
230236
try {
231-
data = JsonParser.object().from(
232-
downloader.get(getLinkHandler().getUrl()).responseBody());
237+
data = JsonParser.object().from(downloader.get(videoUrl).responseBody());
233238
conferenceData = JsonParser.object()
234-
.from(downloader.get(getUploaderUrl()).responseBody());
239+
.from(downloader.get(data.getString("conference_url")).responseBody());
235240
} catch (JsonParserException jpe) {
236-
throw new ExtractionException("Could not parse json returned by url: "
237-
+ getLinkHandler().getUrl(), jpe);
241+
throw new ExtractionException("Could not parse json returned by url: " + videoUrl, jpe);
238242
}
239243
}
240244

@@ -250,21 +254,25 @@ public String getOriginalUrl() {
250254
return data.getString("frontend_link");
251255
}
252256

257+
@Nonnull
253258
@Override
254259
public String getHost() {
255260
return "";
256261
}
257262

263+
@Nonnull
258264
@Override
259265
public String getPrivacy() {
260266
return "";
261267
}
262268

269+
@Nonnull
263270
@Override
264271
public String getCategory() {
265272
return "";
266273
}
267274

275+
@Nonnull
268276
@Override
269277
public String getLicence() {
270278
return "";
@@ -278,7 +286,7 @@ public Locale getLanguageInfo() {
278286
@Nonnull
279287
@Override
280288
public List<String> getTags() {
281-
return new ArrayList<>();
289+
return Arrays.asList(data.getArray("tags").toArray(new String[0]));
282290
}
283291

284292
@Nonnull

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferenceLinkHandlerFactory.java‎

Lines changed: 10 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -7,30 +7,26 @@
77
import java.util.List;
88

99
public class MediaCCCConferenceLinkHandlerFactory extends ListLinkHandlerFactory {
10+
public static final String CONFERENCE_API_ENDPOINT = "https://api.media.ccc.de/public/conferences/";
11+
public static final String CONFERENCE_PATH = "https://media.ccc.de/c/";
12+
private static final String ID_PATTERN = "(?:(?:(?:api\\.)?media\\.ccc\\.de/public/conferences/)|(?:media\\.ccc\\.de/[bc]/))([^/?&#]*)";
13+
1014
@Override
11-
public String getUrl(final String id, final List<String> contentFilter, final String sortFilter)
12-
throws ParsingException {
13-
return "https://media.ccc.de/public/conferences/" + id;
15+
public String getUrl(final String id,
16+
final List<String> contentFilter,
17+
final String sortFilter) throws ParsingException {
18+
return CONFERENCE_PATH + id;
1419
}
1520

1621
@Override
1722
public String getId(final String url) throws ParsingException {
18-
if (url.startsWith("https://media.ccc.de/public/conferences/")
19-
|| url.startsWith("https://api.media.ccc.de/public/conferences/")) {
20-
return url.replaceFirst("https://(api\\.)?media\\.ccc\\.de/public/conferences/", "");
21-
} else if (url.startsWith("https://media.ccc.de/c/")) {
22-
return Parser.matchGroup1("https://media.ccc.de/c/([^?#]*)", url);
23-
} else if (url.startsWith("https://media.ccc.de/b/")) {
24-
return Parser.matchGroup1("https://media.ccc.de/b/([^?#]*)", url);
25-
}
26-
throw new ParsingException("Could not get id from url: " + url);
23+
return Parser.matchGroup1(ID_PATTERN, url);
2724
}
2825

2926
@Override
3027
public boolean onAcceptUrl(final String url) {
3128
try {
32-
getId(url);
33-
return true;
29+
return getId(url) != null;
3430
} catch (ParsingException e) {
3531
return false;
3632
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java‎

Lines changed: 9 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -2,54 +2,27 @@
22

33
import org.schabi.newpipe.extractor.exceptions.ParsingException;
44
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
5-
import org.schabi.newpipe.extractor.utils.Utils;
6-
7-
import java.net.MalformedURLException;
8-
import java.net.URL;
5+
import org.schabi.newpipe.extractor.utils.Parser;
96

107
public class MediaCCCStreamLinkHandlerFactory extends LinkHandlerFactory {
11-
@Override
12-
public String getId(final String urlString) throws ParsingException {
13-
if (urlString.startsWith("https://media.ccc.de/public/events/")
14-
&& !urlString.contains("?q=")) {
15-
return urlString.substring(35); //remove …/public/events part
16-
}
8+
public static final String VIDEO_API_ENDPOINT = "https://api.media.ccc.de/public/events/";
9+
private static final String VIDEO_PATH = "https://media.ccc.de/v/";
10+
private static final String ID_PATTERN = "(?:(?:(?:api\\.)?media\\.ccc\\.de/public/events/)|(?:media\\.ccc\\.de/v/))([^/?&#]*)";
1711

18-
if (urlString.startsWith("https://api.media.ccc.de/public/events/")
19-
&& !urlString.contains("?q=")) {
20-
return urlString.substring(39); //remove api…/public/events part
21-
}
22-
23-
URL url;
24-
try {
25-
url = Utils.stringToURL(urlString);
26-
} catch (MalformedURLException e) {
27-
throw new IllegalArgumentException("The given URL is not valid");
28-
}
29-
30-
String path = url.getPath();
31-
// remove leading "/" of URL-path if URL-path is given
32-
if (!path.isEmpty()) {
33-
path = path.substring(1);
34-
}
35-
36-
if (path.startsWith("v/")) {
37-
return path.substring(2);
38-
}
39-
40-
throw new ParsingException("Could not get id from url: " + url);
12+
@Override
13+
public String getId(final String url) throws ParsingException {
14+
return Parser.matchGroup1(ID_PATTERN, url);
4115
}
4216

4317
@Override
4418
public String getUrl(final String id) throws ParsingException {
45-
return "https://media.ccc.de/public/events/" + id;
19+
return VIDEO_PATH + id;
4620
}
4721

4822
@Override
4923
public boolean onAcceptUrl(final String url) {
5024
try {
51-
getId(url);
52-
return true;
25+
return getId(url) != null;
5326
} catch (ParsingException e) {
5427
return false;
5528
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java‎

Lines changed: 19 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -13,13 +13,16 @@
1313
import org.schabi.newpipe.extractor.exceptions.ParsingException;
1414
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
1515
import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
16+
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory;
1617
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
1718
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
1819
import org.schabi.newpipe.extractor.utils.JsonUtils;
1920
import org.schabi.newpipe.extractor.utils.Utils;
2021

2122
import java.io.IOException;
2223

24+
import javax.annotation.Nonnull;
25+
2326
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY;
2427
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE;
2528
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY;
@@ -85,14 +88,16 @@ public String getParentChannelAvatarUrl() {
8588
return "";
8689
}
8790

91+
@Nonnull
8892
@Override
8993
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
90-
final String pageUrl = getUrl() + "/videos?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE;
91-
return getPage(new Page(pageUrl));
94+
return getPage(new Page(
95+
getUrl() + "/videos?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE));
9296
}
9397

9498
@Override
95-
public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException, ExtractionException {
99+
public InfoItemsPage<StreamInfoItem> getPage(final Page page)
100+
throws IOException, ExtractionException {
96101
if (page == null || isNullOrEmpty(page.getUrl())) {
97102
throw new IllegalArgumentException("Page doesn't contain an URL");
98103
}
@@ -122,8 +127,16 @@ public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException
122127
}
123128

124129
@Override
125-
public void onFetchPage(final Downloader downloader) throws IOException, ExtractionException {
126-
final Response response = downloader.get(getUrl());
130+
public void onFetchPage(@Nonnull final Downloader downloader)
131+
throws IOException, ExtractionException {
132+
String accountUrl = baseUrl + PeertubeChannelLinkHandlerFactory.API_ENDPOINT;
133+
if (getId().contains("accounts/")) {
134+
accountUrl += getId();
135+
} else {
136+
accountUrl += "accounts/" + getId();
137+
}
138+
139+
final Response response = downloader.get(accountUrl);
127140
if (response != null && response.responseBody() != null) {
128141
setInitialData(response.responseBody());
129142
} else {
@@ -140,13 +153,9 @@ private void setInitialData(final String responseBody) throws ExtractionExceptio
140153
if (json == null) throw new ExtractionException("Unable to extract PeerTube account data");
141154
}
142155

156+
@Nonnull
143157
@Override
144158
public String getName() throws ParsingException {
145159
return JsonUtils.getString(json, "displayName");
146160
}
147-
148-
@Override
149-
public String getOriginalUrl() throws ParsingException {
150-
return baseUrl + "/" + getId();
151-
}
152161
}

0 commit comments

Comments
 (0)