using mobile website since it is faster

Author: yausername

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java

@@ -8,6 +8,8 @@
 import java.util.List;
 import java.util.Map;
 
+import javax.annotation.Nonnull;
+
 import org.schabi.newpipe.extractor.DownloadRequest;
 import org.schabi.newpipe.extractor.DownloadResponse;
 import org.schabi.newpipe.extractor.Downloader;
@@ -26,14 +28,12 @@
 import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonObject;
 import com.grack.nanojson.JsonParser;
-import com.grack.nanojson.JsonParserException;
+
 
 public class YoutubeCommentsExtractor extends CommentsExtractor {
 
-    private static final String USER_AGENT = "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:61.0) Gecko/20100101 Firefox/61.0";
+    private static final String USER_AGENT = "Mozilla/5.0 (Android 8.1.0; Mobile; rv:62.0) Gecko/62.0 Firefox/62.0";
 
-    private List<String> cookies;
-    private String sessionToken;
     private String ytClientVersion;
     private String ytClientName;
     private String title;
@@ -45,17 +45,14 @@ public YoutubeCommentsExtractor(StreamingService service, ListLinkHandler uiHand
 
     @Override
     public InfoItemsPage<CommentsInfoItem> getInitialPage() throws IOException, ExtractionException {
-        // initial page does not load any comments but is required to get session token
-        // and cookies
+        // initial page does not load any comments but is required to get comments token
         super.fetchPage();
         return initPage;
     }
 
-    // isn't this method redundant. you can just call getnextpage on getInitialPage
     @Override
     public String getNextPageUrl() throws IOException, ExtractionException {
-        // initial page does not load any comments but is required to get session token
-        // and cookies
+        // initial page does not load any comments but is required to get comments token
         super.fetchPage();
         return initPage.getNextPageUrl();
     }
@@ -64,7 +61,7 @@ private String getNextPageUrl(JsonObject ajaxJson) throws IOException, ParsingEx
 
         JsonArray arr;
         try {
-            arr = (JsonArray) JsonUtils.getValue(ajaxJson, "response.continuationContents.itemSectionContinuation.continuations");
+            arr = (JsonArray) JsonUtils.getValue(ajaxJson, "response.continuationContents.commentSectionContinuation.continuations");
         } catch (Exception e) {
             return "";
         }
@@ -85,9 +82,8 @@ private String getNextPageUrl(String continuation) throws ParsingException {
         params.put("action_get_comments", "1");
         params.put("pbj", "1");
         params.put("ctoken", continuation);
-        params.put("continuation", continuation);
         try {
-            return "https://www.youtube.com/comment_service_ajax?" + getDataString(params);
+            return "https://m.youtube.com/watch_comment?" + getDataString(params);
         } catch (UnsupportedEncodingException e) {
             throw new ParsingException("Could not get next page url", e);
         }
@@ -101,8 +97,8 @@ public InfoItemsPage<CommentsInfoItem> getPage(String pageUrl) throws IOExceptio
         String ajaxResponse = makeAjaxRequest(pageUrl);
         JsonObject ajaxJson;
         try {
-            ajaxJson = JsonParser.object().from(ajaxResponse);
-        } catch (JsonParserException e) {
+            ajaxJson = JsonParser.array().from(ajaxResponse).getObject(1);
+        } catch (Exception e) {
             throw new ParsingException("Could not parse json data for comments", e);
         }
         CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId());
@@ -114,7 +110,7 @@ private void collectCommentsFrom(CommentsInfoItemsCollector collector, JsonObjec
 
         JsonArray contents;
         try {
-            contents = (JsonArray) JsonUtils.getValue(ajaxJson, "response.continuationContents.itemSectionContinuation.contents");
+            contents = (JsonArray) JsonUtils.getValue(ajaxJson, "response.continuationContents.commentSectionContinuation.items");
         }catch(Exception e) {
             //no comments
             return;
@@ -138,7 +134,7 @@ private void collectCommentsFrom(CommentsInfoItemsCollector collector, JsonObjec
     private void fetchTitle(JsonArray contents) {
         if(null == title) {
             try {
-                title = (String) JsonUtils.getValue(contents.getObject(0), "commentThreadRenderer.commentTargetTitle.simpleText");
+                title = getYoutubeText((JsonObject) JsonUtils.getValue(contents.getObject(0), "commentThreadRenderer.commentTargetTitle"));
             } catch (Exception e) {
                 title = "Youtube Comments";
             }
@@ -152,11 +148,9 @@ public void onFetchPage(Downloader downloader) throws IOException, ExtractionExc
         DownloadRequest request = new DownloadRequest(null, requestHeaders);
         DownloadResponse response = downloader.get(getUrl(), request);
         String responseBody = response.getResponseBody();
-        cookies = response.getResponseCookies();
-        sessionToken = findValue(responseBody, "XSRF_TOKEN\":\"", "\"");
         ytClientVersion = findValue(responseBody, "INNERTUBE_CONTEXT_CLIENT_VERSION\":\"", "\"");
         ytClientName = findValue(responseBody, "INNERTUBE_CONTEXT_CLIENT_NAME\":", ",");
-        String commentsTokenInside = findValue(responseBody, "itemSectionRenderer", "comment-item-section");
+        String commentsTokenInside = findValue(responseBody, "commentSectionRenderer", "}");
         String commentsToken = findValue(commentsTokenInside, "continuation\":\"", "\"");
         initPage = getPage(getNextPageUrl(commentsToken));
     }
@@ -168,20 +162,14 @@ public String getName() throws ParsingException {
 
     private String makeAjaxRequest(String siteUrl) throws IOException, ReCaptchaException {
 
-        Map<String, String> postDataMap = new HashMap<>();
-        postDataMap.put("session_token", sessionToken);
-        String postData = getDataString(postDataMap);
-
         Map<String, List<String>> requestHeaders = new HashMap<>();
-        requestHeaders.put("Content-Type", Arrays.asList("application/x-www-form-urlencoded"));
         requestHeaders.put("Accept", Arrays.asList("*/*"));
         requestHeaders.put("User-Agent", Arrays.asList(USER_AGENT));
         requestHeaders.put("X-YouTube-Client-Version", Arrays.asList(ytClientVersion));
         requestHeaders.put("X-YouTube-Client-Name", Arrays.asList(ytClientName));
-        DownloadRequest request = new DownloadRequest(postData, requestHeaders);
-        request.setRequestCookies(cookies);
+        DownloadRequest request = new DownloadRequest(null, requestHeaders);
 
-        return NewPipe.getDownloader().post(siteUrl, request).getResponseBody();
+        return NewPipe.getDownloader().get(siteUrl, request).getResponseBody();
     }
 
     private String getDataString(Map<String, String> params) throws UnsupportedEncodingException {
@@ -205,4 +193,21 @@ private String findValue(String doc, String start, String end) {
         return doc.substring(beginIndex, endIndex);
     }
 
+    public static String getYoutubeText(@Nonnull JsonObject object) throws ParsingException {
+        try {
+            return (String) JsonUtils.getValue(object, "simpleText");
+        } catch (Exception e1) {
+            try {
+                JsonArray arr = (JsonArray) JsonUtils.getValue(object, "runs");
+                String result = "";
+                for(int i=0; i<arr.size();i++) {
+                    result = result + (String) JsonUtils.getValue(arr.getObject(i), "text");
+                }
+                return result;
+            } catch (Exception e2) {
+                throw new ParsingException("Could not get text", e2);
+            }
+        }
+    }
+    
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java

@@ -7,6 +7,7 @@
 import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonObject;
 
+
 public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtractor {
 
     private final JsonObject json;
@@ -35,7 +36,7 @@ public String getThumbnailUrl() throws ParsingException {
     @Override
     public String getName() throws ParsingException {
         try {
-            return (String) JsonUtils.getValue(json, "authorText.simpleText");
+            return YoutubeCommentsExtractor.getYoutubeText((JsonObject) JsonUtils.getValue(json, "authorText"));
         } catch (Exception e) {
             throw new ParsingException("Could not get author name", e);
         }
@@ -44,8 +45,7 @@ public String getName() throws ParsingException {
     @Override
     public String getPublishedTime() throws ParsingException {
         try {
-            JsonArray arr = (JsonArray) JsonUtils.getValue(json, "publishedTimeText.runs");
-            return (String) JsonUtils.getValue(arr.getObject(0), "text");
+            return YoutubeCommentsExtractor.getYoutubeText((JsonObject) JsonUtils.getValue(json, "publishedTimeText"));
         } catch (Exception e) {
             throw new ParsingException("Could not get publishedTimeText", e);
         }
@@ -63,14 +63,9 @@ public Integer getLikeCount() throws ParsingException {
     @Override
     public String getCommentText() throws ParsingException {
         try {
-            return (String) JsonUtils.getValue(json, "contentText.simpleText");
-        } catch (Exception e1) {
-            try {
-                JsonArray arr = (JsonArray) JsonUtils.getValue(json, "contentText.runs");
-                return (String) JsonUtils.getValue(arr.getObject(0), "text");
-            } catch (Exception e2) {
-                throw new ParsingException("Could not get comment text", e2);
-            }
+            return YoutubeCommentsExtractor.getYoutubeText((JsonObject) JsonUtils.getValue(json, "contentText"));
+        } catch (Exception e) {
+            throw new ParsingException("Could not get comment text", e);
         }
     }
 
@@ -96,7 +91,7 @@ public String getAuthorThumbnail() throws ParsingException {
     @Override
     public String getAuthorName() throws ParsingException {
         try {
-            return (String) JsonUtils.getValue(json, "authorText.simpleText");
+            return YoutubeCommentsExtractor.getYoutubeText((JsonObject) JsonUtils.getValue(json, "authorText"));
         } catch (Exception e) {
             throw new ParsingException("Could not get author name", e);
         }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeCommentsLinkHandlerFactory.java

@@ -20,7 +20,7 @@ public static YoutubeCommentsLinkHandlerFactory getInstance() {
 
     @Override
     public String getUrl(String id) {
-        return "https://www.youtube.com/watch?v=" + id;
+        return "https://m.youtube.com/watch?v=" + id;
     }
 
     @Override
@@ -102,6 +102,6 @@ public boolean onAcceptUrl(final String url) throws FoundAdException {
 
     @Override
     public String getUrl(String id, List<String> contentFilter, String sortFilter) throws ParsingException {
-        return "https://www.youtube.com/watch?v=" + id;
+        return getUrl(id);
     }
 }