Skip to content

Commit c2bdead

Browse files
absurdlylongusernameabsurdlylongusername
committed
[SoundCloud] Refactor Soundcloud audio stream extraction code to separate building Hls and Progressive streams
Add HlsAudioStream to facilitate refreshing expired Hls playlists Implement refreshing expired hls playlists in RefreshableHlsHttpDataSource
1 parent 3e3aa9e commit c2bdead

3 files changed

Lines changed: 252 additions & 5 deletions

File tree

‎app/src/main/java/org/schabi/newpipe/player/datasource/RefreshableHlsHttpDataSource.java‎

Lines changed: 233 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,233 @@
1+
package org.schabi.newpipe.player.datasource;
2+
3+
import android.net.Uri;
4+
import android.util.Log;
5+
6+
import androidx.annotation.NonNull;
7+
import androidx.annotation.Nullable;
8+
9+
import com.google.android.exoplayer2.source.hls.playlist.HlsMediaPlaylist;
10+
import com.google.android.exoplayer2.source.hls.playlist.HlsPlaylistParser;
11+
import com.google.android.exoplayer2.upstream.DataSourceInputStream;
12+
import com.google.android.exoplayer2.upstream.DataSpec;
13+
import com.google.android.exoplayer2.upstream.HttpDataSource;
14+
15+
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
16+
import org.schabi.newpipe.extractor.stream.RefreshableStream;
17+
18+
import java.io.IOException;
19+
import java.util.ArrayList;
20+
import java.util.LinkedHashMap;
21+
import java.util.List;
22+
import java.util.Map;
23+
import java.util.Objects;
24+
25+
public class RefreshableHlsHttpDataSource extends LoggingHttpDataSource {
26+
27+
private final String TAG =
28+
RefreshableHlsHttpDataSource.class.getSimpleName() + "@" + hashCode();
29+
private final RefreshableStream refreshableStream;
30+
private final String originalPlaylistUrl;
31+
private final Map<String, String> chunkUrlMap = new LinkedHashMap<>();
32+
private int readsCalled;
33+
private boolean isError;
34+
35+
public RefreshableHlsHttpDataSource(final RefreshableStream refreshableStream) {
36+
this.refreshableStream = refreshableStream;
37+
originalPlaylistUrl = refreshableStream.initialUrl();
38+
}
39+
40+
@SuppressWarnings("checkstyle:LineLength")
41+
public RefreshableHlsHttpDataSource(final RefreshableStream refreshableStream,
42+
@Nullable final String userAgent,
43+
final int connectTimeoutMillis,
44+
final int readTimeoutMillis,
45+
final boolean allowCrossProtocolRedirects,
46+
@Nullable final RequestProperties defaultRequestProperties) {
47+
super(userAgent,
48+
connectTimeoutMillis,
49+
readTimeoutMillis,
50+
allowCrossProtocolRedirects,
51+
defaultRequestProperties);
52+
this.refreshableStream = refreshableStream;
53+
originalPlaylistUrl = refreshableStream.initialUrl();
54+
}
55+
56+
@Override
57+
public int read(@NonNull final byte[] buffer, final int offset, final int length)
58+
throws HttpDataSourceException {
59+
return super.read(buffer, offset, length);
60+
}
61+
62+
@Override
63+
public long open(final DataSpec dataSpec) throws HttpDataSourceException {
64+
final var url = dataSpec.uri.toString();
65+
Log.d(TAG, "called open(" + url + ")");
66+
if (!url.contains(refreshableStream.playlistId())) {
67+
// TODO: throw error or no?
68+
Log.e(TAG, "Playlist id does not match");
69+
}
70+
return chunkUrlMap.isEmpty()
71+
? openInternal(dataSpec)
72+
: openInternal(getUpdatedDataSpec(dataSpec));
73+
}
74+
75+
private long openInternal(final DataSpec dataSpec) throws HttpDataSourceException {
76+
try {
77+
final var bytesToRead = super.open(dataSpec);
78+
Log.d(TAG, "Bytes to read: " + bytesToRead);
79+
isError = false; // if we got to this line there was no error
80+
return bytesToRead;
81+
} catch (final InvalidResponseCodeException e) {
82+
// TODO: This assumes SoundCloud returning 403 when playlist expires
83+
// If we need to refresh playlists for other services at a later date then
84+
// need to generalize this class
85+
if (isError || e.responseCode != 403) {
86+
// Use isError to prevent infinite loop if playlist expires, we replace signature
87+
// but then that one gives an error, and then we replace signature again, and so on
88+
// The expectation is that no error will be thrown on the first recursion
89+
throw e;
90+
}
91+
92+
try {
93+
refreshPlaylist();
94+
} catch (final ExtractionException | IOException ex) {
95+
// TODO: better error here
96+
// TODO: so what happens when we throw exception here
97+
// and this method gets called again in this class?
98+
// if we want to prevent open being called again we need to throw a runtime
99+
throw new HttpDataSourceException("Error refreshing Hls playlist: "
100+
+ originalPlaylistUrl,
101+
new IOException(ex), dataSpec,
102+
HttpDataSourceException.TYPE_OPEN);
103+
}
104+
isError = true;
105+
// Use recursion to reuse error handling without code duplication
106+
return openInternal(getUpdatedDataSpec(dataSpec));
107+
}
108+
}
109+
110+
private void refreshPlaylist() throws ExtractionException, IOException {
111+
Log.d(TAG, "refreshPlaylist() - originalPlaylistUrl " + originalPlaylistUrl);
112+
final var newPlaylistUrl = refreshableStream.fetchLatestUrl();
113+
Log.d(TAG, "New playlist url " + newPlaylistUrl);
114+
Log.d(TAG, "Extracting new playlist Chunks");
115+
final var newChunks = extractChunksFromPlaylist(newPlaylistUrl);
116+
117+
if (!chunkUrlMap.isEmpty()) {
118+
updateChunkMap(chunkUrlMap, newChunks);
119+
}
120+
initializeChunkMappings(chunkUrlMap, newChunks);
121+
}
122+
123+
private static void initializeChunkMappings(final Map<String, String> chunkMap,
124+
final List<String> newChunks) {
125+
for (int i = 0; i < newChunks.size(); ++i) {
126+
final var newUrl = newChunks.get(i);
127+
chunkMap.put(getBaseUrl(newUrl), newUrl);
128+
}
129+
}
130+
131+
private static void updateChunkMap(final Map<String, String> chunkMap,
132+
final List<String> newChunks) throws IOException {
133+
if (chunkMap.size() != newChunks.size()) {
134+
throw new IOException("Error extracting chunks: chunks are not same size\n"
135+
+ "Expected " + chunkMap.size()
136+
+ " and got " + newChunks.size());
137+
}
138+
139+
final var baseUrlIt = chunkMap.keySet().iterator();
140+
final var newChunkUrlIt = newChunks.iterator();
141+
while (baseUrlIt.hasNext()) {
142+
chunkMap.put(baseUrlIt.next(), newChunkUrlIt.next());
143+
}
144+
}
145+
146+
private static String getBaseUrl(final String url) {
147+
final int idx = url.indexOf('?');
148+
return idx == -1 ? url : url.substring(0, idx);
149+
}
150+
151+
// TODO: better name
152+
private DataSpec getUpdatedDataSpec(final DataSpec dataSpec) {
153+
final var currentUrl = dataSpec.uri.toString();
154+
Log.d(TAG, "getUpdatedDataSpec(" + currentUrl + ')');
155+
// Playlist has expired, so get mapping for new url
156+
final var baseUrl = getBaseUrl(currentUrl);
157+
158+
if (baseUrl.equals(currentUrl)) {
159+
Log.e(TAG, "Url has no query parameters");
160+
}
161+
162+
final var updatedUrl = chunkUrlMap.get(baseUrl);
163+
if (updatedUrl == null) {
164+
throw new IllegalStateException("baseUrl not found in mappings: " + baseUrl);
165+
// TODO: problemo
166+
}
167+
Log.d(TAG, "updated url:" + updatedUrl);
168+
return dataSpec.buildUpon()
169+
.setUri(Uri.parse(updatedUrl))
170+
.build();
171+
}
172+
173+
/**
174+
* Extracts the chunks/segments from an m3u8 playlist using
175+
* ExoPlayer's {@link HlsPlaylistParser}.
176+
* @param playlistUrl url of m3u8 playlist to extract
177+
* @return Urls for all the chunks/segments in the playlist
178+
* @throws IOException If error extracting the chunks
179+
*/
180+
private List<String> extractChunksFromPlaylist(final String playlistUrl)
181+
throws IOException {
182+
Log.d(TAG, "extractChunksFromPlaylist(" + playlistUrl + ')');
183+
final var chunks = new ArrayList<String>();
184+
final var parser = new HlsPlaylistParser();
185+
final var dataSpec = new DataSpec(Uri.parse(playlistUrl));
186+
final var httpDataSource = new LoggingHttpDataSource.Factory().createDataSource();
187+
188+
// Adapted from ParsingLoadable.load()
189+
// DataSourceInputStream opens the data source internally on open()
190+
// It passes dataSpec to data source
191+
// httpDataSource is a DefaultHttpDataSource, and getUri will return dataSpec's uri
192+
// which == playlistUrl
193+
try (@SuppressWarnings("LocalCanBeFinal")
194+
var inputStream = new DataSourceInputStream(httpDataSource, dataSpec)) {
195+
inputStream.open();
196+
197+
final var playlist =
198+
parser.parse(Objects.requireNonNull(httpDataSource.getUri()), inputStream);
199+
200+
if (!(playlist instanceof final HlsMediaPlaylist hlsMediaPlaylist)) {
201+
throw new IOException("Expected Hls playlist to be an HlsMediaPlaylist, but was a "
202+
+ playlist.getClass().getSimpleName());
203+
}
204+
for (final var segment : hlsMediaPlaylist.segments) {
205+
chunks.add(segment.url);
206+
}
207+
Log.d(TAG, "Extracted " + chunks.size() + " chunks");
208+
chunks.stream().forEach(m -> Log.d(TAG, "Chunk " + m));
209+
return chunks;
210+
} finally {
211+
httpDataSource.close();
212+
}
213+
}
214+
215+
public static class Factory extends LoggingHttpDataSource.Factory {
216+
private final RefreshableStream refreshableStream;
217+
218+
public Factory(final RefreshableStream refreshableStream) {
219+
this.refreshableStream = refreshableStream;
220+
}
221+
222+
@NonNull
223+
@Override
224+
public HttpDataSource createDataSource() {
225+
return new RefreshableHlsHttpDataSource(refreshableStream,
226+
userAgent,
227+
connectTimeoutMs,
228+
readTimeoutMs,
229+
allowCrossProtocolRedirects,
230+
defaultRequestProperties);
231+
}
232+
}
233+
}

‎app/src/main/java/org/schabi/newpipe/player/helper/PlayerDataSource.java‎

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,11 @@
2626
import org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeOtfDashManifestCreator;
2727
import org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubePostLiveStreamDvrDashManifestCreator;
2828
import org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeProgressiveDashManifestCreator;
29+
import org.schabi.newpipe.extractor.stream.RefreshableStream;
30+
import org.schabi.newpipe.extractor.stream.Stream;
2931
import org.schabi.newpipe.player.datasource.LoggingHttpDataSource;
3032
import org.schabi.newpipe.player.datasource.NonUriHlsDataSourceFactory;
33+
import org.schabi.newpipe.player.datasource.RefreshableHlsHttpDataSource;
3134
import org.schabi.newpipe.player.datasource.YoutubeHttpDataSource;
3235

3336
import java.io.File;
@@ -144,12 +147,23 @@ public DashMediaSource.Factory getLiveDashMediaSourceFactory() {
144147

145148
//region Generic media source factories
146149
public HlsMediaSource.Factory getHlsMediaSourceFactory(
147-
@Nullable final NonUriHlsDataSourceFactory.Builder hlsDataSourceFactoryBuilder) {
148-
if (hlsDataSourceFactoryBuilder != null) {
149-
hlsDataSourceFactoryBuilder.setDataSourceFactory(cacheDataSourceFactory);
150-
return new HlsMediaSource.Factory(hlsDataSourceFactoryBuilder.build());
150+
@NonNull final NonUriHlsDataSourceFactory.Builder hlsDataSourceFactoryBuilder) {
151+
hlsDataSourceFactoryBuilder.setDataSourceFactory(cacheDataSourceFactory);
152+
return new HlsMediaSource.Factory(hlsDataSourceFactoryBuilder.build());
153+
}
154+
155+
public HlsMediaSource.Factory getHlsMediaSourceFactory(final Stream stream) {
156+
if (stream instanceof final RefreshableStream refreshableStream) {
157+
return new HlsMediaSource.Factory(
158+
createCacheDataSourceFactory(
159+
new RefreshableHlsHttpDataSource.Factory(refreshableStream)
160+
)
161+
);
151162
}
163+
return getHlsMediaSourceFactory();
164+
}
152165

166+
public HlsMediaSource.Factory getHlsMediaSourceFactory() {
153167
return new HlsMediaSource.Factory(cacheDataSourceFactory);
154168
}
155169

‎app/src/main/java/org/schabi/newpipe/player/resolver/PlaybackResolver.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -335,7 +335,7 @@ private static HlsMediaSource buildHlsMediaSource(final PlayerDataSource dataSou
335335
throws ResolverException {
336336
if (stream.isUrl()) {
337337
throwResolverExceptionIfUrlNullOrEmpty(stream.getContent());
338-
return dataSource.getHlsMediaSourceFactory(null).createMediaSource(
338+
return dataSource.getHlsMediaSourceFactory(stream).createMediaSource(
339339
new MediaItem.Builder()
340340
.setTag(metadata)
341341
.setUri(Uri.parse(stream.getContent()))

0 commit comments

Comments
 (0)