Adress again reviews, fix some rebase issues

Author: AudricV

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java

@@ -66,7 +66,7 @@ private YoutubeParsingHelper() {
 
     public static final String YOUTUBEI_V1_URL = "https://www.youtube.com/youtubei/v1/";
 
-    private static final String HARDCODED_CLIENT_VERSION = "2.20210622.10.00";
+    private static final String HARDCODED_CLIENT_VERSION = "2.20210623.00.00";
     private static final String HARDCODED_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8";
     private static final String MOBILE_YOUTUBE_KEY = "AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w";
     private static final String MOBILE_YOUTUBE_CLIENT_VERSION = "16.23.36";
@@ -324,8 +324,6 @@ public static boolean areHardcodedClientVersionAndKeyValid()
                         .value("clientVersion", HARDCODED_CLIENT_VERSION)
                     .end()
                 .object("user")
-                    // TO DO: provide a way to enable restricted mode with:
-                    // .value("enableSafetyMode", boolean)
                     .value("lockedSafetyMode", false)
                 .end()
                 .value("fetchLiveState", true)
@@ -353,7 +351,7 @@ private static void extractClientVersionAndKey() throws IOException, ExtractionE
         // Don't extract the client version and the innertube key if it has been already extracted
         if (keyAndVersionExtracted) return;
         // Don't provide a search term in order to have a smaller response
-        final String url = "https://www.youtube.com/results?search_query=";
+        final String url = "https://www.youtube.com/results?search_query=&ucbcb=1";
         final Map<String, List<String>> headers = new HashMap<>();
         addCookieHeader(headers);
         final String html = getDownloader().get(url, headers).responseBody();
@@ -472,7 +470,7 @@ public static void resetClientVersionAndKey() {
      * <b>Only use in tests.</b>
      * </p>
      */
-    public static void setNumberGenerator(Random random) {
+    public static void setNumberGenerator(final Random random) {
         numberGenerator = random;
     }
 
@@ -749,6 +747,7 @@ public static JsonObject getJsonPostResponse(final String endpoint,
             throws IOException, ExtractionException {
         final Map<String, List<String>> headers = new HashMap<>();
         addClientInfoHeaders(headers);
+        headers.put("Content-Type", Collections.singletonList("application/json"));
 
         final Response response = getDownloader().post(YOUTUBEI_V1_URL + endpoint + "?key="
                 + getKey(), headers, body, localization);
@@ -814,6 +813,8 @@ public static JsonBuilder<JsonObject> prepareJsonBuilder(@Nonnull final Localiza
                         .value("clientVersion", getClientVersion())
                     .end()
                     .object("user")
+                        // TO DO: provide a way to enable restricted mode with:
+                        // .value("enableSafetyMode", boolean)
                         .value("lockedSafetyMode", false)
                     .end()
                 .end();
@@ -877,7 +878,7 @@ public static void addClientInfoHeaders(@Nonnull final Map<String, List<String>>
      * @see #CONSENT_COOKIE
      * @param headers the headers which should be completed
      */
-    public static void addCookieHeader(final Map<String, List<String>> headers) {
+    public static void addCookieHeader(@Nonnull final Map<String, List<String>> headers) {
         if (headers.get("Cookie") == null) {
             headers.put("Cookie", Arrays.asList(generateConsentCookie()));
         } else {

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java

@@ -85,7 +85,7 @@ public void onFetchPage(@Nonnull final Downloader downloader) throws IOException
         final String[] channelInfo = channel_path.split("/");
         String id = "";
         // If the url is an URL which is not a /channel URL, we need to use the
-        // navigation/resolve_url endpoint of the internal API to get the channel id. Otherwise, we
+        // navigation/resolve_url endpoint of the youtubei API to get the channel id. Otherwise, we
         // couldn't get information about the channel associated with this URL, if there is one.
         if (!channelInfo[0].equals("channel")) {
             final byte[] body = JsonWriter.string(prepareJsonBuilder(getExtractorLocalization(),

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java

@@ -189,7 +189,7 @@ private void collectCommentsFrom(final CommentsInfoItemsCollector collector, fin
     public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException {
         final Map<String, List<String>> requestHeaders = new HashMap<>();
         requestHeaders.put("User-Agent", singletonList(USER_AGENT));
-        final Response response = downloader.get(getUrl(), requestHeaders, getExtractorLocalization());
+        final Response response = downloader.get(getUrl() + "&ucbcb=1", requestHeaders, getExtractorLocalization());
         responseBody = YoutubeParsingHelper.unescapeDocument(response.responseBody());
         ytClientVersion = findValue(responseBody, "INNERTUBE_CONTEXT_CLIENT_VERSION\":\"", "\"");
         ytClientName = Parser.matchGroup1(YT_CLIENT_NAME_PATTERN, responseBody);

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java

@@ -645,7 +645,8 @@ public List<SubtitlesStream> getSubtitles(final MediaFormat format) throws Parsi
     public StreamType getStreamType() {
         assertPageFetched();
 
-        if (playerResponse.getObject("videoDetails").getBoolean("isLiveContent", false)) {
+        if (playerResponse.getObject("playabilityStatus").has("liveStreamability")
+                || playerResponse.getObject("videoDetails").getBoolean("isPostLiveDvr", false)) {
             return StreamType.LIVE_STREAM;
         }
         return StreamType.VIDEO_STREAM;

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeFeedExtractorTest.java

@@ -93,6 +93,5 @@ public void AccountTerminatedFetch() throws Exception {
                     .getFeedExtractor("https://www.youtube.com/channel/UCTGjY2I-ZUGnwVoWAGRd7XQ");
             extractor.fetchPage();
         }
-
     }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java

@@ -3,6 +3,7 @@
 import com.grack.nanojson.JsonWriter;
 import org.hamcrest.MatcherAssert;
 import org.junit.BeforeClass;
+import org.junit.Ignore;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Suite;
@@ -132,7 +133,7 @@ public void getStreamCount() {
         }
     }
 
-    @Ignore
+    @Ignore("Test broken, video was removed by the uploader")
     public static class MixWithIndex {
 
         private static final int INDEX = 13;

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java

@@ -1,7 +1,6 @@
 package org.schabi.newpipe.extractor.services.youtube.stream;
 
 import org.junit.BeforeClass;
-import org.junit.Ignore;
 import org.schabi.newpipe.downloader.DownloaderFactory;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.StreamingService;

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java

@@ -1,7 +1,6 @@
 package org.schabi.newpipe.extractor.services.youtube.stream;
 
 import org.junit.BeforeClass;
-import org.junit.Ignore;
 import org.junit.Test;
 import org.schabi.newpipe.downloader.DownloaderFactory;
 import org.schabi.newpipe.extractor.NewPipe;