Skip to content

Commit 5842b9a

Browse files
TobiGrTobiGr
committed
Add getClientVersion() and HARDCODED_CLIENT_VERSION to YouTubeParsingHelper
Prefer hardcoded client version above the current one when making requests to retrieve the same JSON structure for each request.
1 parent 5d883d1 commit 5842b9a

3 files changed

Lines changed: 100 additions & 10 deletions

File tree

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java‎

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -211,14 +211,30 @@ public InfoItemsPage<StreamInfoItem> getPage(String pageUrl) throws IOException,
211211

212212
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
213213
JsonArray ajaxJson;
214+
215+
Map<String, List<String>> headers = new HashMap<>();
216+
headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
214217
try {
215-
Map<String, List<String>> headers = new HashMap<>();
216-
headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
217-
headers.put("X-YouTube-Client-Version", Collections.singletonList("2.20200221.03.00")); // TODO: Automatically get YouTube client version somehow
218+
// Use the hardcoded client version first to get JSON with a structure we know
219+
headers.put("X-YouTube-Client-Version",
220+
Collections.singletonList(YoutubeParsingHelper.HARDCODED_CLIENT_VERSION));
218221
final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody();
222+
if (response.length() > 50) { // ensure to have a valid response
223+
throw new ParsingException("Could not parse json data for next streams");
224+
}
219225
ajaxJson = JsonParser.array().from(response);
220-
} catch (JsonParserException pe) {
221-
throw new ParsingException("Could not parse json data for next streams", pe);
226+
} catch (Exception e) {
227+
try {
228+
headers.put("X-YouTube-Client-Version",
229+
Collections.singletonList(YoutubeParsingHelper.getClientVersion(initialData, doc.toString())));
230+
final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody();
231+
if (response.length() > 50) { // ensure to have a valid response
232+
throw new ParsingException("Could not parse json data for next streams");
233+
}
234+
ajaxJson = JsonParser.array().from(response);
235+
} catch (JsonParserException ignored) {
236+
throw new ParsingException("Could not parse json data for next streams", e);
237+
}
222238
}
223239

224240
JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response")

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java‎

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -198,14 +198,30 @@ public InfoItemsPage<StreamInfoItem> getPage(final String pageUrl) throws IOExce
198198

199199
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
200200
JsonArray ajaxJson;
201+
202+
Map<String, List<String>> headers = new HashMap<>();
203+
headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
201204
try {
202-
Map<String, List<String>> headers = new HashMap<>();
203-
headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
204-
headers.put("X-YouTube-Client-Version", Collections.singletonList("2.20200221.03.00")); // TODO: Automatically get YouTube client version somehow
205+
// Use the hardcoded client version first to get JSON with a structure we know
206+
headers.put("X-YouTube-Client-Version",
207+
Collections.singletonList(YoutubeParsingHelper.HARDCODED_CLIENT_VERSION));
205208
final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody();
209+
if (response.length() > 50) { // ensure to have a valid response
210+
throw new ParsingException("Could not parse json data for next streams");
211+
}
206212
ajaxJson = JsonParser.array().from(response);
207-
} catch (JsonParserException pe) {
208-
throw new ParsingException("Could not parse json data for next streams", pe);
213+
} catch (Exception e) {
214+
try {
215+
headers.put("X-YouTube-Client-Version",
216+
Collections.singletonList(YoutubeParsingHelper.getClientVersion(initialData, doc.toString())));
217+
final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody();
218+
if (response.length() > 50) { // ensure to have a valid response
219+
throw new ParsingException("Could not parse json data for next streams");
220+
}
221+
ajaxJson = JsonParser.array().from(response);
222+
} catch (JsonParserException ignored) {
223+
throw new ParsingException("Could not parse json data for next streams", e);
224+
}
209225
}
210226

211227
JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response")

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package org.schabi.newpipe.extractor.services.youtube.linkHandler;
22

33

4+
import com.grack.nanojson.JsonArray;
45
import com.grack.nanojson.JsonObject;
56
import com.grack.nanojson.JsonParser;
67
import com.grack.nanojson.JsonParserException;
@@ -42,6 +43,8 @@ public class YoutubeParsingHelper {
4243
private YoutubeParsingHelper() {
4344
}
4445

46+
public static final String HARDCODED_CLIENT_VERSION = "2.20200214.04.00";
47+
4548
private static final String FEED_BASE_CHANNEL_ID = "https://www.youtube.com/feeds/videos.xml?channel_id=";
4649
private static final String FEED_BASE_USER = "https://www.youtube.com/feeds/videos.xml?user=";
4750

@@ -157,4 +160,59 @@ public static JsonObject getInitialData(String html) throws ParsingException {
157160
}
158161
}
159162

163+
/**
164+
* Get the client version from a page
165+
* @param initialData
166+
* @param html The page HTML
167+
* @return
168+
* @throws ParsingException
169+
*/
170+
public static String getClientVersion(JsonObject initialData, String html) throws ParsingException {
171+
if (initialData == null) initialData = getInitialData(html);
172+
JsonArray serviceTrackingParams = initialData.getObject("responseContext").getArray("serviceTrackingParams");
173+
String shortClientVersion = null;
174+
175+
// try to get version from initial data first
176+
for (Object service : serviceTrackingParams) {
177+
JsonObject s = (JsonObject) service;
178+
if (s.getString("service").equals("CSI")) {
179+
JsonArray params = s.getArray("params");
180+
for (Object param: params) {
181+
JsonObject p = (JsonObject) param;
182+
String key = p.getString("key");
183+
if (key != null && key.equals("cver")) {
184+
return p.getString("value");
185+
}
186+
}
187+
} else if (s.getString("service").equals("ECATCHER")) {
188+
// fallback to get a shortened client version which does not contain the last do digits
189+
JsonArray params = s.getArray("params");
190+
for (Object param: params) {
191+
JsonObject p = (JsonObject) param;
192+
String key = p.getString("key");
193+
if (key != null && key.equals("client.version")) {
194+
shortClientVersion = p.getString("value");
195+
}
196+
}
197+
}
198+
}
199+
200+
String clientVersion;
201+
String[] patterns = {
202+
"INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"",
203+
"innertube_context_client_version\":\"([0-9\\.]+?)\"",
204+
"client.version=([0-9\\.]+)"
205+
};
206+
for (String pattern: patterns) {
207+
try {
208+
clientVersion = Parser.matchGroup1(pattern, html);
209+
if (clientVersion != null && !clientVersion.isEmpty()) return clientVersion;
210+
} catch (Exception ignored) {}
211+
}
212+
213+
if (shortClientVersion != null) return shortClientVersion;
214+
215+
throw new ParsingException("Could not get client version");
216+
}
217+
160218
}

0 commit comments

Comments
 (0)