Skip to content

Commit 43b46bd

Browse files
committed
Use new youtube continuations api for playlists
Requires sending a POST request instead of GET. clientName and clientVersion, which were required as headers previously now need to be part of the request payload. continuation id also needs to be part of request body. quick and dirty solution.
1 parent beb05bd commit 43b46bd

19 files changed

Lines changed: 1635 additions & 233 deletions

File tree

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,11 @@
11
package org.schabi.newpipe.extractor.services.youtube;
22

3-
import com.grack.nanojson.*;
3+
import com.grack.nanojson.JsonArray;
4+
import com.grack.nanojson.JsonObject;
5+
import com.grack.nanojson.JsonParser;
6+
import com.grack.nanojson.JsonParserException;
7+
import com.grack.nanojson.JsonWriter;
8+
49
import org.schabi.newpipe.extractor.MetaInfo;
510
import org.schabi.newpipe.extractor.Page;
611
import org.schabi.newpipe.extractor.downloader.Response;
@@ -33,7 +38,12 @@
3338
import javax.annotation.Nullable;
3439

3540
import static org.schabi.newpipe.extractor.NewPipe.getDownloader;
36-
import static org.schabi.newpipe.extractor.utils.Utils.*;
41+
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
42+
import static org.schabi.newpipe.extractor.utils.Utils.HTTP;
43+
import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
44+
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
45+
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
46+
import static org.schabi.newpipe.extractor.utils.Utils.join;
3747

3848
/*
3949
* Created by Christian Schabesberger on 02.03.16.
@@ -663,6 +673,14 @@ public static JsonArray toJsonArray(final String responseBody) throws ParsingExc
663673
}
664674
}
665675

676+
public static JsonObject toJsonObject(final String responseBody) throws ParsingException {
677+
try {
678+
return JsonParser.object().from(responseBody);
679+
} catch (JsonParserException e) {
680+
throw new ParsingException("Could not parse JSON", e);
681+
}
682+
}
683+
666684
/**
667685
* Shared alert detection function, multiple endpoints return the error similarly structured.
668686
* <p>

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java

Lines changed: 35 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,12 @@
22

33
import com.grack.nanojson.JsonArray;
44
import com.grack.nanojson.JsonObject;
5+
import com.grack.nanojson.JsonWriter;
6+
57
import org.schabi.newpipe.extractor.Page;
68
import org.schabi.newpipe.extractor.StreamingService;
79
import org.schabi.newpipe.extractor.downloader.Downloader;
10+
import org.schabi.newpipe.extractor.downloader.Response;
811
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
912
import org.schabi.newpipe.extractor.exceptions.ParsingException;
1013
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
@@ -19,11 +22,19 @@
1922
import org.schabi.newpipe.extractor.stream.StreamType;
2023
import org.schabi.newpipe.extractor.utils.Utils;
2124

25+
import java.io.IOException;
26+
2227
import javax.annotation.Nonnull;
2328
import javax.annotation.Nullable;
24-
import java.io.IOException;
2529

26-
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
30+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
31+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getClientVersion;
32+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
33+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
34+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
35+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
36+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.toJsonObject;
37+
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
2738
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
2839

2940
@SuppressWarnings("WeakerAccess")
@@ -205,12 +216,27 @@ public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException
205216
throw new IllegalArgumentException("Page doesn't contain an URL");
206217
}
207218

219+
// @formatter:off
220+
byte[] json = JsonWriter.string()
221+
.object()
222+
.object("context")
223+
.object("client")
224+
.value("clientName", "1")
225+
.value("clientVersion", getClientVersion())
226+
.end()
227+
.end()
228+
.value("continuation", page.getId())
229+
.end()
230+
.done()
231+
.getBytes(UTF_8);
232+
// @formatter:on
233+
208234
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
209-
final JsonArray ajaxJson = getJsonResponse(page.getUrl(), getExtractorLocalization());
235+
final Response response = getDownloader().post(page.getUrl(), null, json, getExtractorLocalization());
236+
237+
final JsonObject ajaxJson = toJsonObject(getValidJsonResponseBody(response));
210238

211-
final JsonArray continuation = ajaxJson.getObject(1)
212-
.getObject("response")
213-
.getArray("onResponseReceivedActions")
239+
final JsonArray continuation = ajaxJson.getArray("onResponseReceivedActions")
214240
.getObject(0)
215241
.getObject("appendContinuationItemsAction")
216242
.getArray("continuationItems");
@@ -232,7 +258,9 @@ private Page getNextPageFrom(final JsonArray contents) {
232258
.getObject("continuationEndpoint")
233259
.getObject("continuationCommand")
234260
.getString("token");
235-
return new Page("https://www.youtube.com/browse_ajax?continuation=" + continuation);
261+
return new Page(
262+
"https://www.youtube.com/youtubei/v1/browse?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8",
263+
continuation);
236264
} else {
237265
return null;
238266
}

extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/playlist/TimelessPopHits/generated_mock_0.json

Lines changed: 6 additions & 6 deletions
Large diffs are not rendered by default.

extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/playlist/TimelessPopHits/generated_mock_1.json

Lines changed: 6 additions & 6 deletions
Large diffs are not rendered by default.

0 commit comments

Comments
 (0)