Merge pull request #1277 from litetex/fix-tests

Fix tests and cleanup other stuff

Author: Stypox

extractor/build.gradle

@@ -29,8 +29,6 @@ dependencies {
     implementation 'org.jsoup:jsoup:1.17.2'
     implementation "com.google.code.findbugs:jsr305:$jsr305Version"
 
-    // do not upgrade to 1.7.14, since in 1.7.14 Rhino uses the `SourceVersion` class, which is not
-    // available on Android (even when using desugaring), and `NoClassDefFoundError` is thrown
     implementation 'org.mozilla:rhino:1.7.15'
 
     checkstyle "com.puppycrawl.tools:checkstyle:$checkstyleVersion"
@@ -40,6 +38,6 @@ dependencies {
     testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine'
     testImplementation 'org.junit.jupiter:junit-jupiter-params'
 
-    testImplementation "com.squareup.okhttp3:okhttp:3.12.13"
-    testImplementation 'com.google.code.gson:gson:2.11.0'
+    testImplementation "com.squareup.okhttp3:okhttp:4.12.0"
+    testImplementation 'com.google.code.gson:gson:2.12.1'
 }

extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java

@@ -13,11 +13,12 @@
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 import org.schabi.newpipe.extractor.utils.ExtractorHelper;
 
-import javax.annotation.Nonnull;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 
+import javax.annotation.Nonnull;
+
 public final class PlaylistInfo extends ListInfo<StreamInfoItem> {
 
     /**
@@ -45,7 +46,11 @@ public enum PlaylistType {
         /**
          * A mix made only of streams from (or related to) the same channel, for example YouTube
          * channel mixes
+         *
+         * @deprecated There is currently no service that implements this.
+         * YouTube removed all playlists with this type around 2024-06
          */
+        @Deprecated
         MIX_CHANNEL,
 
         /**

extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSearchExtractor.java

@@ -48,6 +48,7 @@ public List<MetaInfo> getMetaInfo() throws ParsingException {
         return Collections.emptyList();
     }
 
+    @Override
     public InfoItemsPage<InfoItem> getPage(final Page page)
             throws IOException, ExtractionException {
         final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId());
@@ -97,16 +98,12 @@ public InfoItemsPage<InfoItem> getPage(final Page page)
             }
         }
 
-        // Search results appear to be capped at six pages
-        assert pages.size() < 10;
-
         String nextUrl = null;
         if (currentPage < pages.size()) {
             nextUrl = page.getUrl().substring(0, page.getUrl().length() - 1) + (currentPage + 1);
         }
 
         return new InfoItemsPage<>(collector, new Page(nextUrl));
-
     }
 
     @Nonnull

extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSuggestionExtractor.java

@@ -7,22 +7,24 @@
 import com.grack.nanojson.JsonObject;
 import com.grack.nanojson.JsonParser;
 import com.grack.nanojson.JsonParserException;
+import com.grack.nanojson.JsonWriter;
 
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.downloader.Downloader;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
-import org.schabi.newpipe.extractor.utils.Utils;
 
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 import java.util.Collections;
 import java.util.List;
 import java.util.stream.Collectors;
 
 public class BandcampSuggestionExtractor extends SuggestionExtractor {
 
-    private static final String AUTOCOMPLETE_URL = BASE_API_URL + "/fuzzysearch/1/autocomplete?q=";
+    private static final String AUTOCOMPLETE_URL = BASE_API_URL
+            + "/bcsearch_public_api/1/autocomplete_elastic";
     public BandcampSuggestionExtractor(final StreamingService service) {
         super(service);
     }
@@ -33,7 +35,18 @@ public List<String> suggestionList(final String query) throws IOException, Extra
 
         try {
             final JsonObject fuzzyResults = JsonParser.object().from(downloader
-                    .get(AUTOCOMPLETE_URL + Utils.encodeUrlUtf8(query)).responseBody());
+                    .postWithContentTypeJson(
+                            AUTOCOMPLETE_URL,
+                            Collections.emptyMap(),
+                            JsonWriter.string()
+                                    .object()
+                                    .value("fan_id", (String) null)
+                                    .value("full_page", false)
+                                    .value("search_filter", "")
+                                    .value("search_text", query)
+                                    .end()
+                                    .done()
+                                    .getBytes(StandardCharsets.UTF_8)).responseBody());
 
             return fuzzyResults.getObject("auto").getArray("results").stream()
                     .filter(JsonObject.class::isInstance)

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java

@@ -60,7 +60,6 @@
 import org.schabi.newpipe.extractor.stream.AudioTrackType;
 import org.schabi.newpipe.extractor.utils.JsonUtils;
 import org.schabi.newpipe.extractor.utils.Parser;
-import org.schabi.newpipe.extractor.utils.ProtoBuilder;
 import org.schabi.newpipe.extractor.utils.RandomStringFromAlphabetGenerator;
 import org.schabi.newpipe.extractor.utils.Utils;
 
@@ -235,23 +234,6 @@ public static boolean isY2ubeURL(@Nonnull final URL url) {
         return url.getHost().equalsIgnoreCase("y2u.be");
     }
 
-    public static String randomVisitorData(final ContentCountry country) {
-        final ProtoBuilder pbE2 = new ProtoBuilder();
-        pbE2.string(2, "");
-        pbE2.varint(4, numberGenerator.nextInt(255) + 1);
-
-        final ProtoBuilder pbE = new ProtoBuilder();
-        pbE.string(1, country.getCountryCode());
-        pbE.bytes(2, pbE2.toBytes());
-
-        final ProtoBuilder pb = new ProtoBuilder();
-        pb.string(1, RandomStringFromAlphabetGenerator.generate(
-                CONTENT_PLAYBACK_NONCE_ALPHABET, 11, numberGenerator));
-        pb.varint(5, System.currentTimeMillis() / 1000 - numberGenerator.nextInt(600000));
-        pb.bytes(6, pbE.toBytes());
-        return pb.toUrlencodedBase64();
-    }
-
     /**
      * Parses the duration string of the video expecting ":" or "." as separators
      *
@@ -359,16 +341,6 @@ public static boolean isYoutubeMusicMixId(@Nonnull final String playlistId) {
         return playlistId.startsWith("RDAMVM") || playlistId.startsWith("RDCLAK");
     }
 
-    /**
-     * Checks if the given playlist id is a YouTube Channel Mix (auto-generated playlist)
-     * Ids from a YouTube channel Mix start with "RDCM"
-     *
-     * @return Whether given id belongs to a YouTube Channel Mix
-     */
-    public static boolean isYoutubeChannelMixId(@Nonnull final String playlistId) {
-        return playlistId.startsWith("RDCM");
-    }
-
     /**
      * Checks if the given playlist id is a YouTube Genre Mix (auto-generated playlist)
      * Ids from a YouTube Genre Mix start with "RDGMEM"
@@ -399,11 +371,6 @@ public static String extractVideoIdFromMixId(final String playlistId)
         } else if (isYoutubeMusicMixId(playlistId)) {
             return playlistId.substring(6);
 
-        } else if (isYoutubeChannelMixId(playlistId)) {
-            // Channel mixes are of the form RMCM{channelId}, so videoId can't be determined
-            throw new ParsingException("Video id could not be determined from channel mix id: "
-                    + playlistId);
-
         } else if (isYoutubeGenreMixId(playlistId)) {
             // Genre mixes are of the form RDGMEM{garbage}, so videoId can't be determined
             throw new ParsingException("Video id could not be determined from genre mix id: "
@@ -438,8 +405,6 @@ public static PlaylistInfo.PlaylistType extractPlaylistTypeFromPlaylistId(
             throw new ParsingException("Could not extract playlist type from empty playlist id");
         } else if (isYoutubeMusicMixId(playlistId)) {
             return PlaylistInfo.PlaylistType.MIX_MUSIC;
-        } else if (isYoutubeChannelMixId(playlistId)) {
-            return PlaylistInfo.PlaylistType.MIX_CHANNEL;
         } else if (isYoutubeGenreMixId(playlistId)) {
             return PlaylistInfo.PlaylistType.MIX_GENRE;
         } else if (isYoutubeMixId(playlistId)) { // normal mix

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicArtistInfoItemExtractor.java

@@ -1,5 +1,6 @@
 package org.schabi.newpipe.extractor.services.youtube.extractors;
 
+import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonObject;
 import org.schabi.newpipe.extractor.Image;
 import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
@@ -61,10 +62,16 @@ public String getUrl() throws ParsingException {
 
     @Override
     public long getSubscriberCount() throws ParsingException {
-        final String subscriberCount = getTextFromObject(artistInfoItem.getArray("flexColumns")
-                .getObject(2)
+        final JsonArray flexColumns = artistInfoItem.getArray("flexColumns");
+        final JsonArray runs = flexColumns
+                .getObject(flexColumns.size() - 1)
                 .getObject("musicResponsiveListItemFlexColumnRenderer")
-                .getObject("text"));
+                .getObject("text")
+                .getArray("runs");
+        // NOTE: YoutubeParsingHelper#getTextFromObject would use all entries from the run array,
+        // which is not wanted as only the last entry contains the actual subscriberCount
+        final String subscriberCount = runs.getObject(runs.size() - 1)
+                .getString("text");
         if (!isNullOrEmpty(subscriberCount)) {
             try {
                 return Utils.mixedNumberWordToLong(subscriberCount);

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java

@@ -1,6 +1,5 @@
 package org.schabi.newpipe.extractor.services.youtube.linkHandler;
 
-import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
@@ -57,14 +56,6 @@ public String getId(final String url) throws ParsingException, UnsupportedOperat
                         "the list-ID given in the URL does not match the list pattern");
             }
 
-            if (YoutubeParsingHelper.isYoutubeChannelMixId(listID)
-                    && Utils.getQueryValue(urlObj, "v") == null) {
-                // Video id can't be determined from the channel mix id.
-                // See YoutubeParsingHelper#extractVideoIdFromMixId
-                throw new ContentNotSupportedException(
-                        "Channel Mix without a video id are not supported");
-            }
-
             return listID;
         } catch (final Exception exception) {
             throw new ParsingException("Error could not parse URL: " + exception.getMessage(),

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java

@@ -114,7 +114,7 @@ public String getId(final String theUrlString)
         try {
             url = Utils.stringToURL(urlString);
         } catch (final MalformedURLException e) {
-            throw new IllegalArgumentException("The given URL is not valid");
+            throw new ParsingException("The given URL is not valid", e);
         }
 
         final String host = url.getHost();

extractor/src/main/java/org/schabi/newpipe/extractor/utils/ProtoBuilder.java

@@ -36,7 +36,7 @@ public static DownloaderType getDownloaderType() {
      * @param path The path to the folder where mocks are saved/retrieved.
      *             Preferably starting with {@link DownloaderFactory#RESOURCE_PATH}
      */
-    public static Downloader getDownloader(final String path) throws IOException {
+    public static Downloader getDownloader(final String path) {
         final DownloaderType type = getDownloaderType();
         switch (type) {
             case REAL: