Merge pull request TeamNewPipe#1332 from litetex/fix-all-tests

Fix all tests and make everything work offline/with mocks.
Remove old mocks and generate new ones with new structure.

Remove SoundCloud "Top 50" kiosk.

Author: litetex

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java

@@ -402,10 +402,9 @@ public static String getInfoItemsFromApi(final MultiInfoItemsCollector collector
                             // `Likes` feed, so they should be handled by the correct extractor.
                             final JsonObject likedPlaylist =
                                     searchResult.getObject("playlist", null);
-                            collector.commit(
-                                    null == likedPlaylist
-                                            ? new SoundcloudLikesInfoItemExtractor(searchResult)
-                                            : new SoundcloudPlaylistInfoItemExtractor(likedPlaylist)
+                            collector.commit(likedPlaylist == null
+                                ? new SoundcloudLikesInfoItemExtractor(searchResult)
+                                : new SoundcloudPlaylistInfoItemExtractor(likedPlaylist)
                             );
                             break;
                     }

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java

@@ -126,7 +126,6 @@ public KioskList getKioskList() throws ExtractionException {
 
         // add kiosks here e.g.:
         try {
-            list.addKioskEntry(chartsFactory, h, "Top 50");
             list.addKioskEntry(chartsFactory, h, "New & hot");
             list.setDefaultKiosk("New & hot");
         } catch (final Exception e) {

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChartsExtractor.java

@@ -1,5 +1,9 @@
 package org.schabi.newpipe.extractor.services.soundcloud.extractors;
 
+import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
+import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL;
+import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
+
 import org.schabi.newpipe.extractor.Page;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.downloader.Downloader;
@@ -11,22 +15,54 @@
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
 
-import javax.annotation.Nonnull;
 import java.io.IOException;
 
-import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
-import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL;
-import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
+import javax.annotation.Nonnull;
 
 public class SoundcloudChartsExtractor extends KioskExtractor<StreamInfoItem> {
+
+    private String initialFetchNextPageUrl;
+    private StreamInfoItemsCollector initialFetchCollector;
+
     public SoundcloudChartsExtractor(final StreamingService service,
                                      final ListLinkHandler linkHandler,
                                      final String kioskId) {
         super(service, linkHandler, kioskId);
     }
 
     @Override
-    public void onFetchPage(@Nonnull final Downloader downloader) {
+    public void onFetchPage(@Nonnull final Downloader downloader)
+        throws ExtractionException, IOException {
+        // Check if already run
+        if (initialFetchNextPageUrl != null) {
+            return;
+        }
+
+        initialFetchCollector = new StreamInfoItemsCollector(getServiceId());
+
+        final String apiUrl = SOUNDCLOUD_API_V2_URL + "charts"
+            + "?genre=soundcloud:genres:all-music"
+            + "&client_id=" + SoundcloudParsingHelper.clientId()
+            + "&kind=trending";
+
+        final ContentCountry contentCountry = SoundCloud.getContentCountry();
+        String apiUrlWithRegion = null;
+        if (getService().getSupportedCountries().contains(contentCountry)) {
+            apiUrlWithRegion = apiUrl + "&region=soundcloud:regions:"
+                + contentCountry.getCountryCode();
+        }
+
+        try {
+            initialFetchNextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(
+                initialFetchCollector,
+                apiUrlWithRegion == null ? apiUrl : apiUrlWithRegion, true);
+        } catch (final IOException e) {
+            // Request to other region may be geo-restricted.
+            // See https://github.com/TeamNewPipe/NewPipeExtractor/issues/537.
+            // We retry without the specified region.
+            initialFetchNextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(
+                initialFetchCollector, apiUrl, true);
+        }
     }
 
     @Nonnull
@@ -52,35 +88,6 @@ public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException
     @Nonnull
     @Override
     public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
-        final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
-
-        String apiUrl = SOUNDCLOUD_API_V2_URL + "charts" + "?genre=soundcloud:genres:all-music"
-                + "&client_id=" + SoundcloudParsingHelper.clientId();
-
-        if (getId().equals("Top 50")) {
-            apiUrl += "&kind=top";
-        } else {
-            apiUrl += "&kind=trending";
-        }
-
-        final ContentCountry contentCountry = SoundCloud.getContentCountry();
-        String apiUrlWithRegion = null;
-        if (getService().getSupportedCountries().contains(contentCountry)) {
-            apiUrlWithRegion = apiUrl + "&region=soundcloud:regions:"
-                    + contentCountry.getCountryCode();
-        }
-
-        String nextPageUrl;
-        try {
-            nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector,
-                    apiUrlWithRegion == null ? apiUrl : apiUrlWithRegion, true);
-        } catch (final IOException e) {
-            // Request to other region may be geo-restricted.
-            // See https://github.com/TeamNewPipe/NewPipeExtractor/issues/537.
-            // We retry without the specified region.
-            nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl, true);
-        }
-
-        return new InfoItemsPage<>(collector, new Page(nextPageUrl));
+        return new InfoItemsPage<>(initialFetchCollector, new Page(initialFetchNextPageUrl));
     }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudChartsLinkHandlerFactory.java

@@ -11,8 +11,6 @@ public final class SoundcloudChartsLinkHandlerFactory extends ListLinkHandlerFac
     private static final SoundcloudChartsLinkHandlerFactory INSTANCE =
             new SoundcloudChartsLinkHandlerFactory();
 
-    private static final String TOP_URL_PATTERN =
-            "^https?://(www\\.|m\\.)?soundcloud.com/charts(/top)?/?([#?].*)?$";
     private static final String URL_PATTERN =
             "^https?://(www\\.|m\\.)?soundcloud.com/charts(/top|/new)?/?([#?].*)?$";
 
@@ -25,23 +23,15 @@ public static SoundcloudChartsLinkHandlerFactory getInstance() {
 
     @Override
     public String getId(final String url) throws ParsingException, UnsupportedOperationException {
-        if (Parser.isMatch(TOP_URL_PATTERN, url.toLowerCase())) {
-            return "Top 50";
-        } else {
-            return "New & hot";
-        }
+        return "New & hot";
     }
 
     @Override
     public String getUrl(final String id,
                          final List<String> contentFilter,
                          final String sortFilter)
             throws ParsingException, UnsupportedOperationException {
-        if (id.equals("Top 50")) {
-            return "https://soundcloud.com/charts/top";
-        } else {
-            return "https://soundcloud.com/charts/new";
-        }
+        return "https://soundcloud.com/charts/new";
     }
 
     @Override

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeJavaScriptPlayerManager.java

@@ -1,5 +1,7 @@
 package org.schabi.newpipe.extractor.services.youtube;
 
+import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
+
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.utils.JavaScript;
 
@@ -264,6 +266,10 @@ public static String getUrlWithThrottlingParameterDeobfuscated(
                     cachedThrottlingDeobfuscationFunctionName,
                     obfuscatedThrottlingParameter);
 
+            if (isNullOrEmpty(deobfuscatedThrottlingParameter)) {
+                throw new IllegalStateException("Extracted n-parameter is empty");
+            }
+
             CACHED_THROTTLING_PARAMETERS.put(
                     obfuscatedThrottlingParameter, deobfuscatedThrottlingParameter);
 

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java

@@ -34,15 +34,17 @@
 import java.nio.charset.StandardCharsets;
 import java.util.Collections;
 import java.util.List;
-import java.util.stream.Collectors;
 import java.util.Objects;
+import java.util.stream.Collectors;
 
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
 public class YoutubeMusicSearchExtractor extends SearchExtractor {
     private JsonObject initialData;
 
+    private List<JsonObject> cachedItemSectionRendererContents;
+
     public YoutubeMusicSearchExtractor(final StreamingService service,
                                        final SearchQueryHandler linkHandler) {
         super(service, linkHandler);
@@ -116,7 +118,11 @@ public void onFetchPage(@Nonnull final Downloader downloader)
     }
 
     private List<JsonObject> getItemSectionRendererContents() {
-        return initialData
+        if (cachedItemSectionRendererContents != null) {
+            return cachedItemSectionRendererContents;
+        }
+
+        cachedItemSectionRendererContents = initialData
                 .getObject("contents")
                 .getObject("tabbedSearchResultsRenderer")
                 .getArray("tabs")
@@ -134,6 +140,7 @@ private List<JsonObject> getItemSectionRendererContents() {
                         .getArray("contents")
                         .getObject(0))
                 .collect(Collectors.toList());
+        return cachedItemSectionRendererContents;
     }
 
     @Nonnull
@@ -142,12 +149,16 @@ public String getSearchSuggestion() throws ParsingException {
         for (final JsonObject obj : getItemSectionRendererContents()) {
             final JsonObject didYouMeanRenderer = obj
                     .getObject("didYouMeanRenderer");
-            final JsonObject showingResultsForRenderer = obj
-                    .getObject("showingResultsForRenderer");
 
             if (!didYouMeanRenderer.isEmpty()) {
                 return getTextFromObject(didYouMeanRenderer.getObject("correctedQuery"));
-            } else if (!showingResultsForRenderer.isEmpty()) {
+            }
+
+            // NOTE: As of 2025-07 "showing results for ..." doesn't seem to be returned by
+            // the backend anymore, however the code is still present in the JS frontend.
+            final JsonObject showingResultsForRenderer = obj
+                .getObject("showingResultsForRenderer");
+            if (!showingResultsForRenderer.isEmpty()) {
                 return JsonUtils.getString(showingResultsForRenderer,
                         "correctedQueryEndpoint.searchEndpoint.query");
             }

extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java

@@ -91,22 +91,19 @@ public static Matcher matchMultiplePatterns(final Pattern[] patterns, final Stri
             if (matcher.find()) {
                 return matcher;
             } else if (exception == null) {
-                // only pass input to exception message when it is not too long
-                if (input.length() > 1024) {
-                    exception = new RegexException("Failed to find pattern \"" + pattern.pattern()
-                            + "\"");
-                } else {
-                    exception = new RegexException("Failed to find pattern \"" + pattern.pattern()
-                            + "\" inside of \"" + input + "\"");
-                }
+                exception = new RegexException("Failed to find pattern \"" + pattern.pattern()
+                    + "\""
+                    // only pass input to exception message when it is not too long
+                    + (input.length() <= 1000
+                    ? "inside of \"" + input + "\""
+                    : "")
+                );
             }
         }
 
-        if (exception == null) {
-            throw new RegexException("Empty patterns array passed to matchMultiplePatterns");
-        } else {
-            throw exception;
-        }
+        throw exception != null
+            ? exception
+            : new RegexException("Empty patterns array passed to matchMultiplePatterns");
     }
 
     public static boolean isMatch(final String pattern, final String input) {

extractor/src/test/java/org/schabi/newpipe/downloader/DownloaderFactory.java

@@ -2,22 +2,56 @@
 
 import org.schabi.newpipe.extractor.downloader.Downloader;
 
-import java.io.IOException;
+import java.util.Locale;
 
 public class DownloaderFactory {
 
-    public static final String RESOURCE_PATH = "src/test/resources/org/schabi/newpipe/extractor/";
-
     private static final DownloaderType DEFAULT_DOWNLOADER = DownloaderType.REAL;
 
-    public static DownloaderType getDownloaderType() {
+    private static DownloaderType cachedDownloaderType;
+
+    private static DownloaderType getDownloaderType() {
+        if (cachedDownloaderType == null) {
+            cachedDownloaderType = determineDownloaderType();
+        }
+
+        return cachedDownloaderType;
+    }
+
+    private static DownloaderType determineDownloaderType() {
+        String propValue = System.getProperty("downloader");
+        if (propValue == null) {
+            return DEFAULT_DOWNLOADER;
+        }
+        propValue = propValue.toUpperCase();
+        // Use shortcut because RECORDING is quite long
+        if (propValue.equals("REC")) {
+            return DownloaderType.RECORDING;
+        }
         try {
-            return DownloaderType.valueOf(System.getProperty("downloader"));
+            return DownloaderType.valueOf(propValue);
         } catch (final Exception e) {
             return DEFAULT_DOWNLOADER;
         }
     }
 
+    public static Downloader getDownloader(final Class<?> clazz) {
+        return getDownloader(clazz, null);
+    }
+
+    public static Downloader getDownloader(final Class<?> clazz, final String specificUseCase) {
+        String baseName = clazz.getName();
+        if (specificUseCase != null) {
+            baseName += "." + specificUseCase;
+        }
+        return getDownloader("src/test/resources/mocks/v1/"
+            + baseName
+            .toLowerCase(Locale.ENGLISH)
+            .replace('$', '.')
+            .replace("test", "")
+            .replace('.', '/'));
+    }
+
     /**
      * <p>
      * Returns a implementation of a {@link Downloader}.
@@ -34,9 +68,8 @@ public static DownloaderType getDownloaderType() {
      * </p>
      *
      * @param path The path to the folder where mocks are saved/retrieved.
-     *             Preferably starting with {@link DownloaderFactory#RESOURCE_PATH}
      */
-    public static Downloader getDownloader(final String path) {
+    protected static Downloader getDownloader(final String path) {
         final DownloaderType type = getDownloaderType();
         switch (type) {
             case REAL: