Skip to content

Commit 4de99ae

Browse files
theScrabitheScrabi
committed
add support for segmented streams
1 parent 32d3163 commit 4de99ae

4 files changed

Lines changed: 115 additions & 20 deletions

File tree

‎extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,9 @@ private static StreamInfo extractStreams(StreamInfo streamInfo, StreamExtractor
152152
streamInfo.getVideoOnlyStreams().addAll(result.getVideoOnlyStreams());
153153
streamInfo.getAudioStreams().addAll(result.getAudioStreams());
154154
streamInfo.getVideoStreams().addAll(result.getVideoStreams());
155+
streamInfo.segmentedVideoOnlyStreams = result.getSegmentedVideoOnlyStreams();
156+
streamInfo.segmentedAudioStreams = result.getSegmentedAudioStreams();
157+
streamInfo.segmentedVideoStreams = result.getSegmentedVideoStreams();
155158
} catch (Exception e) {
156159
// Sometimes we receive 403 (forbidden) error when trying to download the manifest (similar to what happens with youtube-dl),
157160
// just skip the exception (but store it somewhere), as we later check if we have streams anyway.
@@ -271,6 +274,11 @@ private static StreamInfo extractOptionalData(StreamInfo streamInfo, StreamExtra
271274
private List<VideoStream> videoOnlyStreams;
272275

273276
private String dashMpdUrl;
277+
private List<VideoStream> segmentedVideoStreams;
278+
private List<AudioStream> segmentedAudioStreams;
279+
private List<VideoStream> segmentedVideoOnlyStreams;
280+
281+
274282
private String hlsUrl;
275283
private StreamInfoItem nextVideo;
276284
private List<InfoItem> relatedStreams;
@@ -431,6 +439,30 @@ public void setDashMpdUrl(String dashMpdUrl) {
431439
this.dashMpdUrl = dashMpdUrl;
432440
}
433441

442+
public List<VideoStream> getSegmentedVideoStreams() {
443+
return segmentedVideoStreams;
444+
}
445+
446+
public void setSegmentedVideoStreams(List<VideoStream> segmentedVideoStreams) {
447+
this.segmentedVideoStreams = segmentedVideoStreams;
448+
}
449+
450+
public List<AudioStream> getSegmentedAudioStreams() {
451+
return segmentedAudioStreams;
452+
}
453+
454+
public void setSegmentedAudioStreams(List<AudioStream> segmentedAudioStreams) {
455+
this.segmentedAudioStreams = segmentedAudioStreams;
456+
}
457+
458+
public List<VideoStream> getSegmentedVideoOnlyStreams() {
459+
return segmentedVideoOnlyStreams;
460+
}
461+
462+
public void setSegmentedVideoOnlyStreams(List<VideoStream> segmentedVideoOnlyStreams) {
463+
this.segmentedVideoOnlyStreams = segmentedVideoOnlyStreams;
464+
}
465+
434466
public String getHlsUrl() {
435467
return hlsUrl;
436468
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/utils/DashMpdParser.java‎

Lines changed: 74 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -59,10 +59,23 @@ public static class ParserResult {
5959
private final List<AudioStream> audioStreams;
6060
private final List<VideoStream> videoOnlyStreams;
6161

62-
public ParserResult(List<VideoStream> videoStreams, List<AudioStream> audioStreams, List<VideoStream> videoOnlyStreams) {
62+
private final List<VideoStream> segmentedVideoStreams;
63+
private final List<AudioStream> segmentedAudioStreams;
64+
private final List<VideoStream> segmentedVideoOnlyStreams;
65+
66+
67+
public ParserResult(List<VideoStream> videoStreams,
68+
List<AudioStream> audioStreams,
69+
List<VideoStream> videoOnlyStreams,
70+
List<VideoStream> segmentedVideoStreams,
71+
List<AudioStream> segmentedAudioStreams,
72+
List<VideoStream> segmentedVideoOnlyStreams) {
6373
this.videoStreams = videoStreams;
6474
this.audioStreams = audioStreams;
6575
this.videoOnlyStreams = videoOnlyStreams;
76+
this.segmentedVideoStreams = segmentedVideoStreams;
77+
this.segmentedAudioStreams = segmentedAudioStreams;
78+
this.segmentedVideoOnlyStreams = segmentedVideoOnlyStreams;
6679
}
6780

6881
public List<VideoStream> getVideoStreams() {
@@ -76,10 +89,22 @@ public List<AudioStream> getAudioStreams() {
7689
public List<VideoStream> getVideoOnlyStreams() {
7790
return videoOnlyStreams;
7891
}
92+
93+
public List<VideoStream> getSegmentedVideoStreams() {
94+
return segmentedVideoStreams;
95+
}
96+
97+
public List<AudioStream> getSegmentedAudioStreams() {
98+
return segmentedAudioStreams;
99+
}
100+
101+
public List<VideoStream> getSegmentedVideoOnlyStreams() {
102+
return segmentedVideoOnlyStreams;
103+
}
79104
}
80105

81106
/**
82-
* Will try to download (using {@link StreamInfo#dashMpdUrl}) and parse the dash manifest,
107+
* Will try to download (using {@link StreamInfo#getDashMpdUrl()}) and parse the dash manifest,
83108
* then it will search for any stream that the ItagItem has (by the id).
84109
* <p>
85110
* It has video, video only and audio streams and will only add to the list if it don't
@@ -90,7 +115,8 @@ public List<VideoStream> getVideoOnlyStreams() {
90115
*
91116
* @param streamInfo where the parsed streams will be added
92117
*/
93-
public static ParserResult getStreams(final StreamInfo streamInfo) throws DashMpdParsingException, ReCaptchaException {
118+
public static ParserResult getStreams(final StreamInfo streamInfo)
119+
throws DashMpdParsingException, ReCaptchaException {
94120
String dashDoc;
95121
Downloader downloader = NewPipe.getDownloader();
96122
try {
@@ -113,6 +139,10 @@ public static ParserResult getStreams(final StreamInfo streamInfo) throws DashMp
113139
final List<AudioStream> audioStreams = new ArrayList<>();
114140
final List<VideoStream> videoOnlyStreams = new ArrayList<>();
115141

142+
final List<VideoStream> segmentedVideoStreams = new ArrayList<>();
143+
final List<AudioStream> segmentedAudioStreams = new ArrayList<>();
144+
final List<VideoStream> segmentedVideoOnlyStreams = new ArrayList<>();
145+
116146
for (int i = 0; i < representationList.getLength(); i++) {
117147
final Element representation = (Element) representationList.item(i);
118148
try {
@@ -126,34 +156,61 @@ public static ParserResult getStreams(final StreamInfo streamInfo) throws DashMp
126156
// instead we need to add the "media=" value from the <SegementURL/> tags inside the <SegmentList/>
127157
// tag in order to get a full working url. However each of these is just pointing to a part of the
128158
// video, so we can not return a URL with a working stream here.
129-
// We decided not to ignore such streams for the moment.
130-
if (itag != null && segmentationList == null) {
159+
// Instead of putting those streams into the list of regular stream urls wie put them in a
160+
// for example "segmentedVideoStreams" list.
161+
if (itag != null) {
131162
final MediaFormat mediaFormat = MediaFormat.getFromMimeType(mimeType);
132163

133164
if (itag.itagType.equals(ItagItem.ItagType.AUDIO)) {
134-
final AudioStream audioStream = new AudioStream(url, mediaFormat, itag.avgBitrate);
135-
136-
if (!Stream.containSimilarStream(audioStream, streamInfo.getAudioStreams())) {
137-
audioStreams.add(audioStream);
165+
if(segmentationList == null) {
166+
final AudioStream audioStream = new AudioStream(url, mediaFormat, itag.avgBitrate);
167+
if (!Stream.containSimilarStream(audioStream, streamInfo.getAudioStreams())) {
168+
audioStreams.add(audioStream);
169+
}
170+
} else {
171+
segmentedAudioStreams.add(
172+
new AudioStream(id, mediaFormat, itag.avgBitrate));
138173
}
139174
} else {
140175
boolean isVideoOnly = itag.itagType.equals(ItagItem.ItagType.VIDEO_ONLY);
141-
final VideoStream videoStream = new VideoStream(url, mediaFormat, itag.resolutionString, isVideoOnly);
142176

143-
if (isVideoOnly) {
144-
if (!Stream.containSimilarStream(videoStream, streamInfo.getVideoOnlyStreams())) {
145-
streamInfo.getVideoOnlyStreams().add(videoStream);
146-
videoOnlyStreams.add(videoStream);
177+
if(segmentationList == null) {
178+
final VideoStream videoStream = new VideoStream(url,
179+
mediaFormat,
180+
itag.resolutionString,
181+
isVideoOnly);
182+
183+
if (isVideoOnly) {
184+
if (!Stream.containSimilarStream(videoStream, streamInfo.getVideoOnlyStreams())) {
185+
videoOnlyStreams.add(videoStream);
186+
}
187+
} else if (!Stream.containSimilarStream(videoStream, streamInfo.getVideoStreams())) {
188+
videoStreams.add(videoStream);
189+
}
190+
} else {
191+
final VideoStream videoStream = new VideoStream(id,
192+
mediaFormat,
193+
itag.resolutionString,
194+
isVideoOnly);
195+
196+
if(isVideoOnly) {
197+
segmentedVideoOnlyStreams.add(videoStream);
198+
} else {
199+
segmentedVideoStreams.add(videoStream);
147200
}
148-
} else if (!Stream.containSimilarStream(videoStream, streamInfo.getVideoStreams())) {
149-
videoStreams.add(videoStream);
150201
}
151202
}
152203
}
153204
} catch (Exception ignored) {
154205
}
155206
}
156-
return new ParserResult(videoStreams, audioStreams, videoOnlyStreams);
207+
return new ParserResult(
208+
videoStreams,
209+
audioStreams,
210+
videoOnlyStreams,
211+
segmentedVideoStreams,
212+
segmentedAudioStreams,
213+
segmentedVideoOnlyStreams);
157214
} catch (Exception e) {
158215
throw new DashMpdParsingException("Could not parse Dash mpd", e);
159216
}

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDASHTest.java‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,15 +47,21 @@ public static void setUp() throws Exception {
4747

4848
@Test
4949
public void testGetDashMpd() {
50-
System.out.println(info.getDashMpdUrl());
5150
assertTrue(info.getDashMpdUrl(),
5251
info.getDashMpdUrl() != null && !info.getDashMpdUrl().isEmpty());
5352
}
5453

5554
@Test
56-
public void testDashMpdParser() {
55+
public void testRegularStreams() {
5756
assertEquals(0, info.getAudioStreams().size());
5857
assertEquals(0, info.getVideoOnlyStreams().size());
5958
assertEquals(4, info.getVideoStreams().size());
6059
}
60+
61+
@Test
62+
public void testSegmentedStreams() {
63+
assertEquals(2, info.getSegmentedAudioStreams().size());
64+
assertEquals(3, info.getSegmentedVideoOnlyStreams().size());
65+
assertEquals(0, info.getSegmentedVideoStreams().size());
66+
}
6167
}

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchCountTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ public static void setUpClass() throws Exception {
2828
public void testViewCount() {
2929
ChannelInfoItem ci = (ChannelInfoItem) itemsPage.getItems().get(0);
3030
assertTrue("Count does not fit: " + Long.toString(ci.getSubscriberCount()),
31-
65043316 < ci.getSubscriberCount() && ci.getSubscriberCount() < 68043316);
31+
69043316 < ci.getSubscriberCount() && ci.getSubscriberCount() < 73043316);
3232
}
3333
}
3434
}

0 commit comments

Comments
 (0)