Skip to content

Commit 90afd61

Browse files
XiangRongLinXiangRongLin
authored
Merge pull request #599 from XiangRongLin/post_body
Add body field to Page and use it
2 parents 4e0be60 + 7b06c69 commit 90afd61

40 files changed

Lines changed: 640 additions & 627 deletions

‎extractor/src/main/java/org/schabi/newpipe/extractor/Page.java‎

Lines changed: 22 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44
import java.util.List;
55
import java.util.Map;
66

7+
import javax.annotation.Nullable;
8+
79
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
810

911
public class Page implements Serializable {
@@ -12,31 +14,40 @@ public class Page implements Serializable {
1214
private final List<String> ids;
1315
private final Map<String, String> cookies;
1416

15-
public Page(final String url, final String id, final List<String> ids, final Map<String, String> cookies) {
17+
@Nullable
18+
private final byte[] body;
19+
20+
public Page(final String url, final String id, final List<String> ids,
21+
final Map<String, String> cookies, @Nullable final byte[] body) {
1622
this.url = url;
1723
this.id = id;
1824
this.ids = ids;
1925
this.cookies = cookies;
26+
this.body = body;
2027
}
2128

2229
public Page(final String url) {
23-
this(url, null, null, null);
30+
this(url, null, null, null, null);
2431
}
2532

2633
public Page(final String url, final String id) {
27-
this(url, id, null, null);
34+
this(url, id, null, null, null);
35+
}
36+
37+
public Page(final String url, final byte[] body) {
38+
this(url, null, null, null, body);
2839
}
2940

3041
public Page(final String url, final Map<String, String> cookies) {
31-
this(url, null, null, cookies);
42+
this(url, null, null, cookies, null);
3243
}
3344

3445
public Page(final List<String> ids) {
35-
this(null, null, ids, null);
46+
this(null, null, ids, null, null);
3647
}
3748

3849
public Page(final List<String> ids, final Map<String, String> cookies) {
39-
this(null, null, ids, cookies);
50+
this(null, null, ids, cookies, null);
4051
}
4152

4253
public String getUrl() {
@@ -59,4 +70,9 @@ public static boolean isValid(final Page page) {
5970
return page != null && (!isNullOrEmpty(page.getUrl())
6071
|| !isNullOrEmpty(page.getIds()));
6172
}
73+
74+
@Nullable
75+
public byte[] getBody() {
76+
return body;
77+
}
6278
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package org.schabi.newpipe.extractor.services.youtube;
22

33
import com.grack.nanojson.JsonArray;
4+
import com.grack.nanojson.JsonBuilder;
45
import com.grack.nanojson.JsonObject;
56
import com.grack.nanojson.JsonParser;
67
import com.grack.nanojson.JsonParserException;
@@ -676,6 +677,19 @@ public static JsonArray getJsonResponse(final Page page, final Localization loca
676677
return JsonUtils.toJsonArray(getValidJsonResponseBody(response));
677678
}
678679

680+
public static JsonBuilder<JsonObject> prepareJsonBuilder()
681+
throws IOException, ExtractionException {
682+
// @formatter:off
683+
return JsonObject.builder()
684+
.object("context")
685+
.object("client")
686+
.value("clientName", "1")
687+
.value("clientVersion", getClientVersion())
688+
.end()
689+
.end();
690+
// @formatter:on
691+
}
692+
679693
/**
680694
* Shared alert detection function, multiple endpoints return the error similarly structured.
681695
* <p>

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java‎

Lines changed: 13 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -21,15 +21,16 @@
2121
import org.schabi.newpipe.extractor.utils.JsonUtils;
2222
import org.schabi.newpipe.extractor.utils.Utils;
2323

24-
import javax.annotation.Nonnull;
2524
import java.io.IOException;
2625

26+
import javax.annotation.Nonnull;
27+
2728
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
28-
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getClientVersion;
2929
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
30-
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
3130
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
31+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
3232
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
33+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareJsonBuilder;
3334
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
3435
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
3536
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
@@ -264,23 +265,9 @@ public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException
264265
// as they don't deliver enough information on their own (the channel name, for example).
265266
fetchPage();
266267

267-
// @formatter:off
268-
byte[] json = JsonWriter.string()
269-
.object()
270-
.object("context")
271-
.object("client")
272-
.value("clientName", "1")
273-
.value("clientVersion", getClientVersion())
274-
.end()
275-
.end()
276-
.value("continuation", page.getId())
277-
.end()
278-
.done()
279-
.getBytes(UTF_8);
280-
// @formatter:on
281-
282268
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
283-
final Response response = getDownloader().post(page.getUrl(), null, json, getExtractorLocalization());
269+
final Response response = getDownloader().post(page.getUrl(), null, page.getBody(),
270+
getExtractorLocalization());
284271

285272
final JsonObject ajaxJson = JsonUtils.toJsonObject(getValidJsonResponseBody(response));
286273

@@ -300,8 +287,14 @@ private Page getNextPageFrom(final JsonObject continuations) throws IOException,
300287

301288
final JsonObject continuationEndpoint = continuations.getObject("continuationEndpoint");
302289
final String continuation = continuationEndpoint.getObject("continuationCommand").getString("token");
290+
291+
final byte[] body = JsonWriter.string(prepareJsonBuilder()
292+
.value("continuation", continuation)
293+
.done())
294+
.getBytes(UTF_8);
295+
303296
return new Page("https://www.youtube.com/youtubei/v1/browse?key=" + getKey(),
304-
continuation);
297+
body);
305298
}
306299

307300
/**

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java‎

Lines changed: 12 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
2121
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
2222
import org.schabi.newpipe.extractor.stream.StreamType;
23+
import org.schabi.newpipe.extractor.utils.JsonUtils;
2324
import org.schabi.newpipe.extractor.utils.Utils;
2425

2526
import java.io.IOException;
@@ -28,13 +29,12 @@
2829
import javax.annotation.Nullable;
2930

3031
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
31-
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getClientVersion;
3232
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
3333
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
3434
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
3535
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
3636
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
37-
import static org.schabi.newpipe.extractor.utils.JsonUtils.toJsonObject;
37+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareJsonBuilder;
3838
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
3939
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
4040

@@ -217,25 +217,11 @@ public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException
217217
throw new IllegalArgumentException("Page doesn't contain an URL");
218218
}
219219

220-
// @formatter:off
221-
byte[] json = JsonWriter.string()
222-
.object()
223-
.object("context")
224-
.object("client")
225-
.value("clientName", "1")
226-
.value("clientVersion", getClientVersion())
227-
.end()
228-
.end()
229-
.value("continuation", page.getId())
230-
.end()
231-
.done()
232-
.getBytes(UTF_8);
233-
// @formatter:on
234-
235220
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
236-
final Response response = getDownloader().post(page.getUrl(), null, json, getExtractorLocalization());
237221

238-
final JsonObject ajaxJson = toJsonObject(getValidJsonResponseBody(response));
222+
final Response response = getDownloader().post(page.getUrl(), null, page.getBody(),
223+
getExtractorLocalization());
224+
final JsonObject ajaxJson = JsonUtils.toJsonObject(getValidJsonResponseBody(response));
239225

240226
final JsonArray continuation = ajaxJson.getArray("onResponseReceivedActions")
241227
.getObject(0)
@@ -259,9 +245,15 @@ private Page getNextPageFrom(final JsonArray contents) throws IOException, Extra
259245
.getObject("continuationEndpoint")
260246
.getObject("continuationCommand")
261247
.getString("token");
248+
249+
final byte[] body = JsonWriter.string(prepareJsonBuilder()
250+
.value("continuation", continuation)
251+
.done())
252+
.getBytes(UTF_8);
253+
262254
return new Page(
263255
"https://www.youtube.com/youtubei/v1/browse?key=" + getKey(),
264-
continuation);
256+
body);
265257
} else {
266258
return null;
267259
}

‎extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/VSauce/generated_mock_0.json‎

Lines changed: 4 additions & 6 deletions
Large diffs are not rendered by default.

‎extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/VSauce/generated_mock_1.json‎

Lines changed: 4 additions & 6 deletions
Large diffs are not rendered by default.

‎extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/VSauce/generated_mock_2.json‎

Lines changed: 5 additions & 7 deletions
Large diffs are not rendered by default.

0 commit comments

Comments
 (0)