Skip to content

Commit 7898ba2

Browse files
wb9688wb9688
committed
Use pbj in YoutubeStreamExtractor
1 parent 612ec06 commit 7898ba2

1 file changed

Lines changed: 30 additions & 63 deletions

File tree

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java‎

Lines changed: 30 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -3,18 +3,15 @@
33
import com.grack.nanojson.JsonArray;
44
import com.grack.nanojson.JsonObject;
55
import com.grack.nanojson.JsonParser;
6+
import com.grack.nanojson.JsonParserException;
67

7-
import org.jsoup.nodes.Document;
8-
import org.jsoup.nodes.Element;
98
import org.mozilla.javascript.Context;
109
import org.mozilla.javascript.Function;
1110
import org.mozilla.javascript.ScriptableObject;
1211
import org.schabi.newpipe.extractor.MediaFormat;
1312
import org.schabi.newpipe.extractor.NewPipe;
1413
import org.schabi.newpipe.extractor.StreamingService;
1514
import org.schabi.newpipe.extractor.downloader.Downloader;
16-
import org.schabi.newpipe.extractor.downloader.Response;
17-
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
1815
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
1916
import org.schabi.newpipe.extractor.exceptions.ParsingException;
2017
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
@@ -91,7 +88,7 @@ public class DecryptException extends ParsingException {
9188

9289
/*//////////////////////////////////////////////////////////////////////////*/
9390

94-
private Document doc;
91+
private JsonArray initialAjaxJson;
9592
@Nullable
9693
private JsonObject playerArgs;
9794
@Nonnull
@@ -554,23 +551,8 @@ public StreamInfoItemsCollector getRelatedStreams() throws ExtractionException {
554551
*/
555552
@Override
556553
public String getErrorMessage() {
557-
StringBuilder errorReason;
558-
Element errorElement = doc.select("h1[id=\"unavailable-message\"]").first();
559-
560-
if (errorElement == null) {
561-
errorReason = null;
562-
} else {
563-
String errorMessage = errorElement.text();
564-
if (errorMessage == null || errorMessage.isEmpty()) {
565-
errorReason = null;
566-
} else {
567-
errorReason = new StringBuilder(errorMessage);
568-
errorReason.append(" ");
569-
errorReason.append(doc.select("[id=\"unavailable-submessage\"]").first().text());
570-
}
571-
}
572-
573-
return errorReason != null ? errorReason.toString() : "";
554+
return getTextFromObject(initialAjaxJson.getObject(2).getObject("playerResponse").getObject("playabilityStatus")
555+
.getObject("errorScreen").getObject("playerErrorMessageRenderer").getObject("reason"));
574556
}
575557

576558
/*//////////////////////////////////////////////////////////////////////////
@@ -580,11 +562,8 @@ public String getErrorMessage() {
580562
private static final String FORMATS = "formats";
581563
private static final String ADAPTIVE_FORMATS = "adaptiveFormats";
582564
private static final String HTTPS = "https:";
583-
private static final String CONTENT = "content";
584565
private static final String DECRYPTION_FUNC_NAME = "decrypt";
585566

586-
private static final String VERIFIED_URL_PARAMS = "&has_verified=1&bpctr=9999999999";
587-
588567
private final static String DECRYPTION_SIGNATURE_FUNCTION_REGEX =
589568
"([\\w$]+)\\s*=\\s*function\\((\\w+)\\)\\{\\s*\\2=\\s*\\2\\.split\\(\"\"\\)\\s*;";
590569
private final static String DECRYPTION_SIGNATURE_FUNCTION_REGEX_2 =
@@ -596,29 +575,42 @@ public String getErrorMessage() {
596575

597576
private volatile String decryptionCode = "";
598577

599-
private String pageHtml = null;
600-
601578
@Override
602579
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
603-
final String verifiedUrl = getUrl() + VERIFIED_URL_PARAMS;
604-
final Response response = downloader.get(verifiedUrl, getExtractorLocalization());
605-
pageHtml = response.responseBody();
606-
doc = YoutubeParsingHelper.parseAndCheckPage(verifiedUrl, response);
580+
final String url = getUrl() + "&pbj=1";
581+
582+
Map<String, List<String>> headers = new HashMap<>();
583+
headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
584+
headers.put("X-YouTube-Client-Version",
585+
Collections.singletonList(YoutubeParsingHelper.getClientVersion()));
586+
final String response = getDownloader().get(url, headers, getExtractorLocalization()).responseBody();
587+
if (response.length() < 50) { // ensure to have a valid response
588+
throw new ParsingException("Could not parse json data for next streams");
589+
}
590+
591+
try {
592+
initialAjaxJson = JsonParser.array().from(response);
593+
} catch (JsonParserException e) {
594+
throw new ParsingException("Could not parse json data for next streams", e);
595+
}
607596

608597
final String playerUrl;
609-
initialData = YoutubeParsingHelper.getInitialData(pageHtml);
610-
// Check if the video is age restricted
611-
if (getAgeLimit() == 18) {
598+
599+
if (initialAjaxJson.getObject(2).getObject("response") != null) { // age-restricted videos
600+
initialData = initialAjaxJson.getObject(2).getObject("response");
601+
612602
final EmbeddedInfo info = getEmbeddedInfo();
613603
final String videoInfoUrl = getVideoInfoUrl(getId(), info.sts);
614604
final String infoPageResponse = downloader.get(videoInfoUrl, getExtractorLocalization()).responseBody();
615605
videoInfoPage.putAll(Parser.compatParseMap(infoPageResponse));
616606
playerUrl = info.url;
617607
} else {
618-
final JsonObject ytPlayerConfig = getPlayerConfig();
619-
playerArgs = getPlayerArgs(ytPlayerConfig);
620-
playerUrl = getPlayerUrl(ytPlayerConfig);
608+
initialData = initialAjaxJson.getObject(3).getObject("response");
609+
610+
playerArgs = getPlayerArgs(initialAjaxJson.getObject(2).getObject("player"));
611+
playerUrl = getPlayerUrl(initialAjaxJson.getObject(2).getObject("player"));
621612
}
613+
622614
playerResponse = getPlayerResponse();
623615

624616
if (decryptionCode.isEmpty()) {
@@ -630,21 +622,6 @@ public void onFetchPage(@Nonnull Downloader downloader) throws IOException, Extr
630622
}
631623
}
632624

633-
private JsonObject getPlayerConfig() throws ParsingException {
634-
try {
635-
String ytPlayerConfigRaw = Parser.matchGroup1("ytplayer.config\\s*=\\s*(\\{.*?\\});", pageHtml);
636-
return JsonParser.object().from(ytPlayerConfigRaw);
637-
} catch (Parser.RegexException e) {
638-
String errorReason = getErrorMessage();
639-
if (errorReason.isEmpty()) {
640-
throw new ContentNotAvailableException("Content not available: player config empty", e);
641-
}
642-
throw new ContentNotAvailableException("Content not available", e);
643-
} catch (Exception e) {
644-
throw new ParsingException("Could not parse yt player config", e);
645-
}
646-
}
647-
648625
private JsonObject getPlayerArgs(JsonObject playerConfig) throws ParsingException {
649626
JsonObject playerArgs;
650627

@@ -950,17 +927,7 @@ private Map<String, ItagItem> getItags(String streamingDataKey, ItagItem.ItagTyp
950927
@Override
951928
public List<Frameset> getFrames() throws ExtractionException {
952929
try {
953-
final String script = doc.select("#player-api").first().siblingElements().select("script").html();
954-
int p = script.indexOf("ytplayer.config");
955-
if (p == -1) {
956-
return Collections.emptyList();
957-
}
958-
p = script.indexOf('{', p);
959-
int e = script.indexOf("ytplayer.load", p);
960-
if (e == -1) {
961-
return Collections.emptyList();
962-
}
963-
JsonObject jo = JsonParser.object().from(script.substring(p, e - 1));
930+
JsonObject jo = initialAjaxJson.getObject(2).getObject("player");
964931
final String resp = jo.getObject("args").getString("player_response");
965932
jo = JsonParser.object().from(resp);
966933
final String[] spec = jo.getObject("storyboards").getObject("playerStoryboardSpecRenderer").getString("spec").split("\\|");

0 commit comments

Comments
 (0)