Skip to content

Commit a77865b

Browse files
litetexlitetex
committed
More refactoring + started implementing ItagInfo
1 parent f632fc9 commit a77865b

38 files changed

Lines changed: 669 additions & 301 deletions

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -146,12 +146,12 @@ public Description getDescription() {
146146
public List<AudioStream> getAudioStreams() {
147147
return Collections.singletonList(
148148
new SimpleAudioStreamImpl(
149+
AudioFormatRegistry.MP3,
149150
new SimpleProgressiveHTTPDeliveryDataImpl(albumJson
150151
.getArray("trackinfo")
151152
.getObject(0)
152153
.getObject("file")
153154
.getString("mp3-128")),
154-
AudioFormatRegistry.MP3,
155155
128
156156
)
157157
);

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamExtractor.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -174,9 +174,9 @@ public List<AudioStream> getAudioStreams() throws IOException, ExtractionExcepti
174174
}
175175

176176
return new SimpleAudioStreamImpl(
177-
deliveryData,
178177
// TODO: This looks wrong
179-
new AudioFormatRegistry().getFromSuffix(dto.getUrlKey())
178+
new AudioFormatRegistry().getFromSuffix(dto.getUrlKey()),
179+
deliveryData
180180
);
181181
})
182182
.collect(Collectors.toList());
@@ -199,9 +199,9 @@ public List<VideoAudioStream> getVideoStreams() throws IOException, ExtractionEx
199199
final JsonArray videoSize = dto.getStreamJsonObj().getArray("videoSize");
200200

201201
return new SimpleVideoAudioStreamImpl(
202-
deliveryData,
203202
// TODO: This looks wrong
204203
new VideoAudioFormatRegistry().getFromSuffix(dto.getUrlKey()),
204+
deliveryData,
205205
VideoQualityData.fromHeightWidth(
206206
/*height=*/videoSize.getInt(1, VideoQualityData.UNKNOWN),
207207
/*width=*/videoSize.getInt(0, VideoQualityData.UNKNOWN))

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -99,8 +99,8 @@ public String getUploaderAvatarUrl() {
9999
public List<AudioStream> getAudioStreams() throws ExtractionException {
100100
return getRecordingsByMimeType("audio")
101101
.map(o -> new SimpleAudioStreamImpl(
102-
new SimpleProgressiveHTTPDeliveryDataImpl(o.getString("recording_url")),
103-
new AudioFormatRegistry().getFromMimeType(o.getString("mime_type"))
102+
new AudioFormatRegistry().getFromMimeType(o.getString("mime_type")),
103+
new SimpleProgressiveHTTPDeliveryDataImpl(o.getString("recording_url"))
104104
))
105105
.collect(Collectors.toList());
106106
}
@@ -109,10 +109,8 @@ public List<AudioStream> getAudioStreams() throws ExtractionException {
109109
public List<VideoAudioStream> getVideoStreams() throws ExtractionException {
110110
return getRecordingsByMimeType("video")
111111
.map(o -> new SimpleVideoAudioStreamImpl(
112-
new SimpleProgressiveHTTPDeliveryDataImpl(o.getString("recording_url")),
113-
null,
114-
AudioStream.UNKNOWN_BITRATE,
115112
new VideoAudioFormatRegistry().getFromMimeType(o.getString("mime_type")),
113+
new SimpleProgressiveHTTPDeliveryDataImpl(o.getString("recording_url")),
116114
VideoQualityData.fromHeightWidth(
117115
o.getInt("height", VideoQualityData.UNKNOWN),
118116
o.getInt("width", VideoQualityData.UNKNOWN))

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -402,14 +402,15 @@ private void tryExtractSubtitles() {
402402
.map(JsonObject.class::cast)
403403
.map(caption -> {
404404
try {
405-
final String url = baseUrl + JsonUtils.getString(caption,
406-
"captionPath");
405+
final String url =
406+
baseUrl + JsonUtils.getString(caption, "captionPath");
407407

408408
return new SimpleSubtitleStreamImpl(
409-
new SimpleProgressiveHTTPDeliveryDataImpl(url),
409+
// TODO: Check for null
410410
new SubtitleFormatRegistry()
411411
.getFromSuffix(
412412
url.substring(url.lastIndexOf(".") + 1)),
413+
new SimpleProgressiveHTTPDeliveryDataImpl(url),
413414
false,
414415
JsonUtils.getString(caption, "language.id")
415416
);
@@ -467,9 +468,9 @@ private void extractLiveVideoStreams() throws ParsingException {
467468
.filter(JsonObject.class::isInstance)
468469
.map(JsonObject.class::cast)
469470
// TODO Check! This is the master playlist!
470-
.map(stream -> new SimpleVideoAudioStreamImpl(
471-
new SimpleHLSDeliveryDataImpl(stream.getString(PLAYLIST_URL, "")),
472-
VideoAudioFormatRegistry.MPEG_4)
471+
.map(s -> new SimpleVideoAudioStreamImpl(
472+
VideoAudioFormatRegistry.MPEG_4,
473+
new SimpleHLSDeliveryDataImpl(s.getString(PLAYLIST_URL, "")))
473474
)
474475
// Don't use the containsSimilarStream method because it will always
475476
// return
@@ -501,9 +502,9 @@ private void addStreamsFromArray(
501502
stream,
502503
playlistUrl,
503504
(s, dd) -> new SimpleAudioStreamImpl(
504-
dd,
505505
new AudioFormatRegistry()
506-
.getFromSuffix(getExtensionFromStream(s))
506+
.getFromSuffix(getExtensionFromStream(s)),
507+
dd
507508
)
508509
);
509510

@@ -514,9 +515,9 @@ private void addStreamsFromArray(
514515
stream,
515516
playlistUrl,
516517
(s, dd) -> new SimpleVideoAudioStreamImpl(
517-
dd,
518518
new VideoAudioFormatRegistry()
519519
.getFromSuffix(getExtensionFromStream(s)),
520+
dd,
520521
VideoQualityData.fromHeightFps(
521522
resJson.getInt("id", VideoQualityData.UNKNOWN),
522523
stream.getInt("fps", VideoQualityData.UNKNOWN))

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java‎

Lines changed: 47 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
import org.schabi.newpipe.extractor.NewPipe;
1515
import org.schabi.newpipe.extractor.StreamingService;
1616
import org.schabi.newpipe.extractor.downloader.Downloader;
17+
import org.schabi.newpipe.extractor.downloader.Response;
1718
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
1819
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
1920
import org.schabi.newpipe.extractor.exceptions.GeographicRestrictionException;
@@ -258,8 +259,8 @@ private List<AudioStream> extractAudioStreams() {
258259

259260
final String preset = transcoding.getString("preset", "");
260261

261-
AudioMediaFormat mediaFormat = null;
262-
int averageBitrate = AudioStream.UNKNOWN_BITRATE;
262+
final AudioMediaFormat mediaFormat;
263+
final int averageBitrate;
263264
if (preset.contains("mp3")) {
264265
// Don't add the MP3 HLS stream if there is a progressive stream present
265266
// because the two have the same bitrate
@@ -271,13 +272,15 @@ private List<AudioStream> extractAudioStreams() {
271272
} else if (preset.contains("opus")) {
272273
mediaFormat = AudioFormatRegistry.OPUS;
273274
averageBitrate = 64;
275+
} else {
276+
return null;
274277
}
275278

276279
return (AudioStream) new SimpleAudioStreamImpl(
280+
mediaFormat,
277281
protocol.equals("hls")
278282
? new SimpleHLSDeliveryDataImpl(mediaUrl)
279283
: new SimpleProgressiveHTTPDeliveryDataImpl(mediaUrl),
280-
mediaFormat,
281284
averageBitrate
282285
);
283286
})
@@ -311,7 +314,17 @@ private Optional<AudioStream> extractDownloadableFileIfAvailable() {
311314
if (isNullOrEmpty(downloadUrl)) {
312315
return Optional.empty();
313316
}
317+
318+
// Find out what type of file is served
319+
final String fileType = determineFileTypeFromDownloadUrl(downloadUrl);
320+
321+
// No fileType found -> ignore it
322+
if (isNullOrEmpty(fileType)) {
323+
return Optional.empty();
324+
}
325+
314326
return Optional.of(new SimpleAudioStreamImpl(
327+
new AudioFormatRegistry().getFromSuffix(fileType),
315328
new SimpleProgressiveHTTPDeliveryDataImpl(downloadUrl)
316329
));
317330
} catch (final Exception ignored) {
@@ -321,6 +334,37 @@ private Optional<AudioStream> extractDownloadableFileIfAvailable() {
321334
}
322335
}
323336

337+
/**
338+
* Determines the file type/extension of the download url.
339+
* <p>
340+
* Note: Uses HTTP FETCH for inspection.
341+
* </p>
342+
*/
343+
@Nullable
344+
private String determineFileTypeFromDownloadUrl(final String downloadUrl)
345+
throws IOException, ReCaptchaException {
346+
347+
final Response response = NewPipe.getDownloader().head(downloadUrl);
348+
349+
// As of 2022-06 Soundcloud uses AWS S3
350+
// Use the AWS header to identify the filetype first because it's simpler
351+
final String amzMetaFileType = response.getHeader("x-amz-meta-file-type");
352+
if (!isNullOrEmpty(amzMetaFileType)) {
353+
return amzMetaFileType;
354+
}
355+
356+
// If the AWS header was not present try extract the filetype
357+
// by inspecting the download file name
358+
// Example-Value:
359+
// attachment;filename="SoundCloud%20Download"; filename*=utf-8''song.mp3
360+
final String contentDisp = response.getHeader("Content-Disposition");
361+
if (!isNullOrEmpty(contentDisp) && contentDisp.contains(".")) {
362+
return contentDisp.substring(contentDisp.lastIndexOf(".") + 1);
363+
}
364+
365+
return null;
366+
}
367+
324368
/**
325369
* Parses a SoundCloud HLS manifest to get a single URL of HLS streams.
326370
*

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/AudioItagFormat.java‎

Lines changed: 0 additions & 12 deletions
This file was deleted.

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/ItagFormat.java‎

Lines changed: 0 additions & 9 deletions
This file was deleted.

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/VideoAudioItagFormat.java‎

Lines changed: 0 additions & 5 deletions
This file was deleted.

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/format/AudioItagFormat.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
package org.schabi.newpipe.extractor.services.youtube.itag.format;
2+
3+
import org.schabi.newpipe.extractor.streamdata.format.AudioMediaFormat;
4+
5+
public interface AudioItagFormat extends ItagFormat<AudioMediaFormat>, BaseAudioItagFormat {
6+
// Nothing additional to implement
7+
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/itag/format/BaseAudioItagFormat.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
package org.schabi.newpipe.extractor.services.youtube.itag.format;
2+
3+
public interface BaseAudioItagFormat {
4+
/**
5+
* Average audio bitrate in KBit/s
6+
*/
7+
int averageBitrate();
8+
}

0 commit comments

Comments
 (0)