Skip to content

Commit 73bd33d

Browse files
litetexlitetex
committed
Rewrote extractors part 1
1 parent 8b94206 commit 73bd33d

6 files changed

Lines changed: 447 additions & 545 deletions

File tree

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java‎

Lines changed: 15 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -11,19 +11,20 @@
1111
import org.jsoup.Jsoup;
1212
import org.jsoup.nodes.Document;
1313
import org.jsoup.nodes.Element;
14-
import org.schabi.newpipe.extractor.MediaFormat;
1514
import org.schabi.newpipe.extractor.StreamingService;
1615
import org.schabi.newpipe.extractor.downloader.Downloader;
1716
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
1817
import org.schabi.newpipe.extractor.exceptions.ParsingException;
1918
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
2019
import org.schabi.newpipe.extractor.localization.DateWrapper;
2120
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemsCollector;
22-
import org.schabi.newpipe.extractor.stream.AudioStream;
2321
import org.schabi.newpipe.extractor.stream.Description;
2422
import org.schabi.newpipe.extractor.stream.StreamExtractor;
2523
import org.schabi.newpipe.extractor.stream.StreamType;
26-
import org.schabi.newpipe.extractor.stream.VideoStream;
24+
import org.schabi.newpipe.extractor.streamdata.delivery.simpleimpl.SimpleProgressiveHTTPDeliveryDataImpl;
25+
import org.schabi.newpipe.extractor.streamdata.format.registry.AudioFormatRegistry;
26+
import org.schabi.newpipe.extractor.streamdata.stream.AudioStream;
27+
import org.schabi.newpipe.extractor.streamdata.stream.simpleimpl.SimpleAudioStreamImpl;
2728
import org.schabi.newpipe.extractor.utils.JsonUtils;
2829
import org.schabi.newpipe.extractor.utils.Utils;
2930

@@ -143,15 +144,17 @@ public Description getDescription() {
143144

144145
@Override
145146
public List<AudioStream> getAudioStreams() {
146-
return Collections.singletonList(new AudioStream.Builder()
147-
.setId("mp3-128")
148-
.setContent(albumJson.getArray("trackinfo")
149-
.getObject(0)
150-
.getObject("file")
151-
.getString("mp3-128"), true)
152-
.setMediaFormat(MediaFormat.MP3)
153-
.setAverageBitrate(128)
154-
.build());
147+
return Collections.singletonList(
148+
new SimpleAudioStreamImpl(
149+
new SimpleProgressiveHTTPDeliveryDataImpl(albumJson
150+
.getArray("trackinfo")
151+
.getObject(0)
152+
.getObject("file")
153+
.getString("mp3-128")),
154+
AudioFormatRegistry.MP3,
155+
128
156+
)
157+
);
155158
}
156159

157160
@Override
@@ -160,16 +163,6 @@ public long getLength() throws ParsingException {
160163
.getDouble("duration");
161164
}
162165

163-
@Override
164-
public List<VideoStream> getVideoStreams() {
165-
return Collections.emptyList();
166-
}
167-
168-
@Override
169-
public List<VideoStream> getVideoOnlyStreams() {
170-
return Collections.emptyList();
171-
}
172-
173166
@Override
174167
public StreamType getStreamType() {
175168
return StreamType.AUDIO_STREAM;

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamExtractor.java‎

Lines changed: 57 additions & 80 deletions
Original file line numberDiff line numberDiff line change
@@ -6,25 +6,29 @@
66
import com.grack.nanojson.JsonArray;
77
import com.grack.nanojson.JsonObject;
88

9-
import org.schabi.newpipe.extractor.MediaFormat;
109
import org.schabi.newpipe.extractor.StreamingService;
1110
import org.schabi.newpipe.extractor.downloader.Downloader;
1211
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
1312
import org.schabi.newpipe.extractor.exceptions.ParsingException;
1413
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
15-
import org.schabi.newpipe.extractor.stream.AudioStream;
16-
import org.schabi.newpipe.extractor.stream.DeliveryMethod;
1714
import org.schabi.newpipe.extractor.stream.Description;
18-
import org.schabi.newpipe.extractor.stream.Stream;
1915
import org.schabi.newpipe.extractor.stream.StreamExtractor;
2016
import org.schabi.newpipe.extractor.stream.StreamType;
21-
import org.schabi.newpipe.extractor.stream.VideoStream;
17+
import org.schabi.newpipe.extractor.streamdata.delivery.DeliveryData;
18+
import org.schabi.newpipe.extractor.streamdata.delivery.simpleimpl.SimpleDASHUrlDeliveryDataImpl;
19+
import org.schabi.newpipe.extractor.streamdata.delivery.simpleimpl.SimpleHLSDeliveryDataImpl;
20+
import org.schabi.newpipe.extractor.streamdata.delivery.simpleimpl.SimpleProgressiveHTTPDeliveryDataImpl;
21+
import org.schabi.newpipe.extractor.streamdata.format.registry.AudioFormatRegistry;
22+
import org.schabi.newpipe.extractor.streamdata.format.registry.VideoAudioFormatRegistry;
23+
import org.schabi.newpipe.extractor.streamdata.stream.AudioStream;
24+
import org.schabi.newpipe.extractor.streamdata.stream.VideoAudioStream;
25+
import org.schabi.newpipe.extractor.streamdata.stream.simpleimpl.SimpleAudioStreamImpl;
26+
import org.schabi.newpipe.extractor.streamdata.stream.simpleimpl.SimpleVideoAudioStreamImpl;
2227

2328
import java.io.IOException;
24-
import java.util.Collections;
2529
import java.util.List;
26-
import java.util.function.Function;
2730
import java.util.stream.Collectors;
31+
import java.util.stream.Stream;
2832

2933
import javax.annotation.Nonnull;
3034

@@ -45,7 +49,8 @@ public MediaCCCLiveStreamExtractor(final StreamingService service,
4549
@Override
4650
public void onFetchPage(@Nonnull final Downloader downloader)
4751
throws IOException, ExtractionException {
48-
final JsonArray doc = MediaCCCParsingHelper.getLiveStreams(downloader,
52+
final JsonArray doc = MediaCCCParsingHelper.getLiveStreams(
53+
downloader,
4954
getExtractorLocalization());
5055
// Find the correct room
5156
for (int c = 0; c < doc.size(); c++) {
@@ -137,7 +142,7 @@ public String getDashMpdUrl() throws ParsingException {
137142
*/
138143
@Nonnull
139144
@Override
140-
public String getHlsUrl() {
145+
public String getHlsMasterPlaylistUrl() {
141146
return getManifestOfDeliveryMethodWanted("hls");
142147
}
143148

@@ -155,77 +160,54 @@ private String getManifestOfDeliveryMethodWanted(@Nonnull final String deliveryM
155160

156161
@Override
157162
public List<AudioStream> getAudioStreams() throws IOException, ExtractionException {
158-
return getStreams("audio",
159-
dto -> {
160-
final AudioStream.Builder builder = new AudioStream.Builder()
161-
.setId(dto.urlValue.getString("tech", ID_UNKNOWN))
162-
.setContent(dto.urlValue.getString(URL), true)
163-
.setAverageBitrate(UNKNOWN_BITRATE);
164-
165-
if ("hls".equals(dto.urlKey)) {
166-
// We don't know with the type string what media format will
167-
// have HLS streams.
168-
// However, the tech string may contain some information
169-
// about the media format used.
170-
return builder.setDeliveryMethod(DeliveryMethod.HLS)
171-
.build();
163+
return getStreamDTOs("audio")
164+
.map(dto -> {
165+
final String url = dto.getUrlValue().getString(URL);
166+
final DeliveryData deliveryData;
167+
if ("hls".equals(dto.getUrlKey())) {
168+
deliveryData = new SimpleHLSDeliveryDataImpl(url);
169+
} else if ("dash".equals(dto.getUrlKey())) {
170+
deliveryData = new SimpleDASHUrlDeliveryDataImpl(url);
171+
} else {
172+
deliveryData = new SimpleProgressiveHTTPDeliveryDataImpl(url);
172173
}
173174

174-
return builder.setMediaFormat(MediaFormat.getFromSuffix(dto.urlKey))
175-
.build();
176-
});
175+
return new SimpleAudioStreamImpl(
176+
deliveryData,
177+
// TODO: This looks wrong
178+
new AudioFormatRegistry().getFromSuffix(dto.getUrlKey())
179+
);
180+
})
181+
.collect(Collectors.toList());
177182
}
178183

179184
@Override
180-
public List<VideoStream> getVideoStreams() throws IOException, ExtractionException {
181-
return getStreams("video",
182-
dto -> {
183-
final JsonArray videoSize = dto.streamJsonObj.getArray("videoSize");
184-
185-
final VideoStream.Builder builder = new VideoStream.Builder()
186-
.setId(dto.urlValue.getString("tech", ID_UNKNOWN))
187-
.setContent(dto.urlValue.getString(URL), true)
188-
.setIsVideoOnly(false)
189-
.setResolution(videoSize.getInt(0) + "x" + videoSize.getInt(1));
190-
191-
if ("hls".equals(dto.urlKey)) {
192-
// We don't know with the type string what media format will
193-
// have HLS streams.
194-
// However, the tech string may contain some information
195-
// about the media format used.
196-
return builder.setDeliveryMethod(DeliveryMethod.HLS)
197-
.build();
185+
public List<VideoAudioStream> getVideoStreams() throws IOException, ExtractionException {
186+
return getStreamDTOs("video")
187+
.map(dto -> {
188+
final String url = dto.getUrlValue().getString(URL);
189+
final DeliveryData deliveryData;
190+
if ("hls".equals(dto.getUrlKey())) {
191+
deliveryData = new SimpleHLSDeliveryDataImpl(url);
192+
} else if ("dash".equals(dto.getUrlKey())) {
193+
deliveryData = new SimpleDASHUrlDeliveryDataImpl(url);
194+
} else {
195+
deliveryData = new SimpleProgressiveHTTPDeliveryDataImpl(url);
198196
}
199197

200-
return builder.setMediaFormat(MediaFormat.getFromSuffix(dto.urlKey))
201-
.build();
202-
});
203-
}
204-
205-
206-
/**
207-
* This is just an internal class used in {@link #getStreams(String, Function)} to tie together
208-
* the stream json object, its URL key and its URL value. An object of this class would be
209-
* temporary and the three values it holds would be <b>convert</b>ed to a proper {@link Stream}
210-
* object based on the wanted stream type.
211-
*/
212-
private static final class MediaCCCLiveStreamMapperDTO {
213-
final JsonObject streamJsonObj;
214-
final String urlKey;
215-
final JsonObject urlValue;
198+
final JsonArray videoSize = dto.getStreamJsonObj().getArray("videoSize");
216199

217-
MediaCCCLiveStreamMapperDTO(final JsonObject streamJsonObj,
218-
final String urlKey,
219-
final JsonObject urlValue) {
220-
this.streamJsonObj = streamJsonObj;
221-
this.urlKey = urlKey;
222-
this.urlValue = urlValue;
223-
}
200+
return new SimpleVideoAudioStreamImpl(
201+
deliveryData,
202+
// TODO: This looks wrong
203+
new VideoAudioFormatRegistry().getFromSuffix(dto.getUrlKey()),
204+
videoSize.getInt(0) + "x" + videoSize.getInt(1)
205+
);
206+
})
207+
.collect(Collectors.toList());
224208
}
225209

226-
private <T extends Stream> List<T> getStreams(
227-
@Nonnull final String streamType,
228-
@Nonnull final Function<MediaCCCLiveStreamMapperDTO, T> converter) {
210+
private Stream<MediaCCCLiveStreamMapperDTO> getStreamDTOs(@Nonnull final String streamType) {
229211
return room.getArray(STREAMS).stream()
230212
// Ensure that we use only process JsonObjects
231213
.filter(JsonObject.class::isInstance)
@@ -238,22 +220,17 @@ private <T extends Stream> List<T> getStreams(
238220
.map(e -> new MediaCCCLiveStreamMapperDTO(
239221
streamJsonObj,
240222
e.getKey(),
241-
(JsonObject) e.getValue())))
242-
// The DASH manifest will be extracted with getDashMpdUrl
243-
.filter(dto -> !"dash".equals(dto.urlKey))
244-
// Convert
245-
.map(converter)
246-
.collect(Collectors.toList());
223+
(JsonObject) e.getValue())));
247224
}
248225

249226
@Override
250-
public List<VideoStream> getVideoOnlyStreams() {
251-
return Collections.emptyList();
227+
public StreamType getStreamType() throws ParsingException {
228+
return StreamType.LIVE_STREAM;
252229
}
253230

254231
@Override
255-
public StreamType getStreamType() throws ParsingException {
256-
return StreamType.LIVE_STREAM;
232+
public boolean isLive() {
233+
return true;
257234
}
258235

259236
@Nonnull

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCParsingHelper.java‎

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import com.grack.nanojson.JsonArray;
44
import com.grack.nanojson.JsonParser;
55
import com.grack.nanojson.JsonParserException;
6+
67
import org.schabi.newpipe.extractor.downloader.Downloader;
78
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
89
import org.schabi.newpipe.extractor.exceptions.ParsingException;
@@ -43,13 +44,21 @@ public static boolean isLiveStreamId(final String id) {
4344
/**
4445
* Get currently available live streams from
4546
* <a href="https://streaming.media.ccc.de/streams/v2.json">
46-
* https://streaming.media.ccc.de/streams/v2.json</a>.
47+
* https://streaming.media.ccc.de/streams/v2.json</a>.
48+
* <p>
4749
* Use this method to cache requests, because they can get quite big.
50+
* </p>
51+
*
52+
* <p>
53+
* For more information see also: <a href="https://github.com/voc/streaming-website#json-api">
54+
* https://github.com/voc/streaming-website#json-api</a>.
55+
* </p>
4856
* TODO: implement better caching policy (max-age: 3 min)
49-
* @param downloader The downloader to use for making the request
57+
*
58+
* @param downloader The downloader to use for making the request
5059
* @param localization The localization to be used. Will most likely be ignored.
5160
* @return {@link JsonArray} containing current conferences and info about their rooms and
52-
* streams.
61+
* streams.
5362
* @throws ExtractionException if the data could not be fetched or the retrieved data could not
5463
* be parsed to a {@link JsonArray}
5564
*/
@@ -58,13 +67,14 @@ public static JsonArray getLiveStreams(final Downloader downloader,
5867
throws ExtractionException {
5968
if (liveStreams == null) {
6069
try {
61-
final String site = downloader.get("https://streaming.media.ccc.de/streams/v2.json",
62-
localization).responseBody();
70+
final String site = downloader
71+
.get("https://streaming.media.ccc.de/streams/v2.json", localization)
72+
.responseBody();
6373
liveStreams = JsonParser.array().from(site);
6474
} catch (final IOException | ReCaptchaException e) {
65-
throw new ExtractionException("Could not get live stream JSON.", e);
75+
throw new ExtractionException("Could not get live stream JSON", e);
6676
} catch (final JsonParserException e) {
67-
throw new ExtractionException("Could not parse JSON.", e);
77+
throw new ExtractionException("Could not parse JSON", e);
6878
}
6979
}
7080
return liveStreams;

0 commit comments

Comments
 (0)