Merge branch 'TeamNewPipe:dev' into yt_premieres-lockups-support

Author: ShareASmile

build.gradle

@@ -8,7 +8,7 @@ allprojects {
     sourceCompatibility = JavaVersion.VERSION_11
     targetCompatibility = JavaVersion.VERSION_11
 
-    version 'v0.24.6'
+    version 'v0.24.8'
     group 'com.github.TeamNewPipe'
 
     repositories {
@@ -29,7 +29,7 @@ allprojects {
     ext {
         nanojsonVersion = "e9d656ddb49a412a5a0a5d5ef20ca7ef09549996"
         jsr305Version = "3.0.2"
-        junitVersion = "5.13.3"
+        junitVersion = "5.13.4"
         checkstyleVersion = "10.4"
     }
 }
@@ -68,10 +68,10 @@ subprojects {
 
 // https://discuss.gradle.org/t/best-approach-gradle-multi-module-project-generate-just-one-global-javadoc/18657/21
 tasks.register('aggregatedJavadocs', Javadoc) {
-    destinationDir = file("${layout.buildDirectory}/docs/javadoc")
+    destinationDir = layout.buildDirectory.file("docs/javadoc").get().asFile
     title = "$project.name $version"
     // options.memberLevel = JavadocMemberLevel.PRIVATE
-    options.links 'https://docs.oracle.com/javase/8/docs/api/'
+    options.links 'https://docs.oracle.com/javase/11/docs/api/'
     options.encoding 'UTF-8'
     // Fixes unknown tag @implNote; the other two were added precautionary
     options.tags = [

extractor/build.gradle

@@ -28,7 +28,7 @@ checkstyleTest {
 
 ext {
     rhinoVersion = '1.8.0'
-    protobufVersion = '4.30.2'
+    protobufVersion = '4.31.1'
 }
 
 dependencies {

extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/UnsupportedContentInCountryException.java

@@ -0,0 +1,20 @@
+package org.schabi.newpipe.extractor.exceptions;
+
+/**
+ * Exception for contents not supported in a country.
+ *
+ * <p>
+ * Unsupported content means content is not intentionally geographically restricted such as for
+ * distribution rights, for which {@link GeographicRestrictionException} should be used instead.
+ * </p>
+ */
+public class UnsupportedContentInCountryException extends ContentNotAvailableException {
+
+    public UnsupportedContentInCountryException(final String message) {
+        super(message);
+    }
+
+    public UnsupportedContentInCountryException(final String message, final Throwable cause) {
+        super(message, cause);
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java

@@ -402,10 +402,9 @@ public static String getInfoItemsFromApi(final MultiInfoItemsCollector collector
                             // `Likes` feed, so they should be handled by the correct extractor.
                             final JsonObject likedPlaylist =
                                     searchResult.getObject("playlist", null);
-                            collector.commit(
-                                    null == likedPlaylist
-                                            ? new SoundcloudLikesInfoItemExtractor(searchResult)
-                                            : new SoundcloudPlaylistInfoItemExtractor(likedPlaylist)
+                            collector.commit(likedPlaylist == null
+                                ? new SoundcloudLikesInfoItemExtractor(searchResult)
+                                : new SoundcloudPlaylistInfoItemExtractor(likedPlaylist)
                             );
                             break;
                     }

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java

@@ -126,7 +126,6 @@ public KioskList getKioskList() throws ExtractionException {
 
         // add kiosks here e.g.:
         try {
-            list.addKioskEntry(chartsFactory, h, "Top 50");
             list.addKioskEntry(chartsFactory, h, "New & hot");
             list.setDefaultKiosk("New & hot");
         } catch (final Exception e) {

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChartsExtractor.java

@@ -1,5 +1,9 @@
 package org.schabi.newpipe.extractor.services.soundcloud.extractors;
 
+import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
+import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL;
+import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
+
 import org.schabi.newpipe.extractor.Page;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.downloader.Downloader;
@@ -11,22 +15,54 @@
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
 
-import javax.annotation.Nonnull;
 import java.io.IOException;
 
-import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
-import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL;
-import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
+import javax.annotation.Nonnull;
 
 public class SoundcloudChartsExtractor extends KioskExtractor<StreamInfoItem> {
+
+    private String initialFetchNextPageUrl;
+    private StreamInfoItemsCollector initialFetchCollector;
+
     public SoundcloudChartsExtractor(final StreamingService service,
                                      final ListLinkHandler linkHandler,
                                      final String kioskId) {
         super(service, linkHandler, kioskId);
     }
 
     @Override
-    public void onFetchPage(@Nonnull final Downloader downloader) {
+    public void onFetchPage(@Nonnull final Downloader downloader)
+        throws ExtractionException, IOException {
+        // Check if already run
+        if (initialFetchNextPageUrl != null) {
+            return;
+        }
+
+        initialFetchCollector = new StreamInfoItemsCollector(getServiceId());
+
+        final String apiUrl = SOUNDCLOUD_API_V2_URL + "charts"
+            + "?genre=soundcloud:genres:all-music"
+            + "&client_id=" + SoundcloudParsingHelper.clientId()
+            + "&kind=trending";
+
+        final ContentCountry contentCountry = SoundCloud.getContentCountry();
+        String apiUrlWithRegion = null;
+        if (getService().getSupportedCountries().contains(contentCountry)) {
+            apiUrlWithRegion = apiUrl + "&region=soundcloud:regions:"
+                + contentCountry.getCountryCode();
+        }
+
+        try {
+            initialFetchNextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(
+                initialFetchCollector,
+                apiUrlWithRegion == null ? apiUrl : apiUrlWithRegion, true);
+        } catch (final IOException e) {
+            // Request to other region may be geo-restricted.
+            // See https://github.com/TeamNewPipe/NewPipeExtractor/issues/537.
+            // We retry without the specified region.
+            initialFetchNextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(
+                initialFetchCollector, apiUrl, true);
+        }
     }
 
     @Nonnull
@@ -52,35 +88,6 @@ public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException
     @Nonnull
     @Override
     public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
-        final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
-
-        String apiUrl = SOUNDCLOUD_API_V2_URL + "charts" + "?genre=soundcloud:genres:all-music"
-                + "&client_id=" + SoundcloudParsingHelper.clientId();
-
-        if (getId().equals("Top 50")) {
-            apiUrl += "&kind=top";
-        } else {
-            apiUrl += "&kind=trending";
-        }
-
-        final ContentCountry contentCountry = SoundCloud.getContentCountry();
-        String apiUrlWithRegion = null;
-        if (getService().getSupportedCountries().contains(contentCountry)) {
-            apiUrlWithRegion = apiUrl + "&region=soundcloud:regions:"
-                    + contentCountry.getCountryCode();
-        }
-
-        String nextPageUrl;
-        try {
-            nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector,
-                    apiUrlWithRegion == null ? apiUrl : apiUrlWithRegion, true);
-        } catch (final IOException e) {
-            // Request to other region may be geo-restricted.
-            // See https://github.com/TeamNewPipe/NewPipeExtractor/issues/537.
-            // We retry without the specified region.
-            nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl, true);
-        }
-
-        return new InfoItemsPage<>(collector, new Page(nextPageUrl));
+        return new InfoItemsPage<>(initialFetchCollector, new Page(initialFetchNextPageUrl));
     }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudChartsLinkHandlerFactory.java

@@ -11,8 +11,6 @@ public final class SoundcloudChartsLinkHandlerFactory extends ListLinkHandlerFac
     private static final SoundcloudChartsLinkHandlerFactory INSTANCE =
             new SoundcloudChartsLinkHandlerFactory();
 
-    private static final String TOP_URL_PATTERN =
-            "^https?://(www\\.|m\\.)?soundcloud.com/charts(/top)?/?([#?].*)?$";
     private static final String URL_PATTERN =
             "^https?://(www\\.|m\\.)?soundcloud.com/charts(/top|/new)?/?([#?].*)?$";
 
@@ -25,23 +23,15 @@ public static SoundcloudChartsLinkHandlerFactory getInstance() {
 
     @Override
     public String getId(final String url) throws ParsingException, UnsupportedOperationException {
-        if (Parser.isMatch(TOP_URL_PATTERN, url.toLowerCase())) {
-            return "Top 50";
-        } else {
-            return "New & hot";
-        }
+        return "New & hot";
     }
 
     @Override
     public String getUrl(final String id,
                          final List<String> contentFilter,
                          final String sortFilter)
             throws ParsingException, UnsupportedOperationException {
-        if (id.equals("Top 50")) {
-            return "https://soundcloud.com/charts/top";
-        } else {
-            return "https://soundcloud.com/charts/new";
-        }
+        return "https://soundcloud.com/charts/new";
     }
 
     @Override

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ClientsConstants.java

@@ -45,6 +45,12 @@ private ClientsConstants() {
     static final String WEB_EMBEDDED_CLIENT_NAME = "WEB_EMBEDDED_PLAYER";
     static final String WEB_EMBEDDED_CLIENT_VERSION = "1.20250121.00.00";
 
+    // WEB_MUSIC_ANALYTICS (YouTube charts)
+
+    static final String WEB_MUSIC_ANALYTICS_CLIENT_ID = "31";
+    static final String WEB_MUSIC_ANALYTICS_CLIENT_NAME = "WEB_MUSIC_ANALYTICS";
+    static final String WEB_MUSIC_ANALYTICS_CLIENT_VERSION = "2.0";
+
     // IOS (iOS YouTube app) client fields
 
     static final String IOS_CLIENT_ID = "5";

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/InnertubeClientRequestInfo.java

@@ -1,5 +1,8 @@
 package org.schabi.newpipe.extractor.services.youtube;
 
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
 import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.ANDROID_CLIENT_ID;
 import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.ANDROID_CLIENT_NAME;
 import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.ANDROID_CLIENT_VERSION;
@@ -16,11 +19,11 @@
 import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_CLIENT_NAME;
 import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_EMBEDDED_CLIENT_ID;
 import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_EMBEDDED_CLIENT_NAME;
+import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_EMBEDDED_CLIENT_VERSION;
 import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_HARDCODED_CLIENT_VERSION;
-import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_REMIX_HARDCODED_CLIENT_VERSION;
-
-import javax.annotation.Nonnull;
-import javax.annotation.Nullable;
+import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_MUSIC_ANALYTICS_CLIENT_ID;
+import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_MUSIC_ANALYTICS_CLIENT_NAME;
+import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_MUSIC_ANALYTICS_CLIENT_VERSION;
 
 // TODO: add docs
 
@@ -38,28 +41,28 @@ public static final class ClientInfo {
         @Nonnull
         public String clientVersion;
         @Nonnull
-        public String clientScreen;
-        @Nullable
         public String clientId;
         @Nullable
+        public String clientScreen;
+        @Nullable
         public String visitorData;
 
         private ClientInfo(@Nonnull final String clientName,
                            @Nonnull final String clientVersion,
-                           @Nonnull final String clientScreen,
-                           @Nullable final String clientId,
+                           @Nonnull final String clientId,
+                           @Nullable final String clientScreen,
                            @Nullable final String visitorData) {
             this.clientName = clientName;
             this.clientVersion = clientVersion;
-            this.clientScreen = clientScreen;
             this.clientId = clientId;
+            this.clientScreen = clientScreen;
             this.visitorData = visitorData;
         }
     }
 
     public static final class DeviceInfo {
 
-        @Nonnull
+        @Nullable
         public String platform;
         @Nullable
         public String deviceMake;
@@ -71,7 +74,7 @@ public static final class DeviceInfo {
         public String osVersion;
         public int androidSdkVersion;
 
-        private DeviceInfo(@Nonnull final String platform,
+        private DeviceInfo(@Nullable final String platform,
                            @Nullable final String deviceMake,
                            @Nullable final String deviceModel,
                            @Nullable final String osName,
@@ -96,8 +99,8 @@ private InnertubeClientRequestInfo(@Nonnull final ClientInfo clientInfo,
     public static InnertubeClientRequestInfo ofWebClient() {
         return new InnertubeClientRequestInfo(
                 new InnertubeClientRequestInfo.ClientInfo(
-                        WEB_CLIENT_NAME, WEB_HARDCODED_CLIENT_VERSION, WATCH_CLIENT_SCREEN,
-                        WEB_CLIENT_ID, null),
+                        WEB_CLIENT_NAME, WEB_HARDCODED_CLIENT_VERSION, WEB_CLIENT_ID,
+                        WATCH_CLIENT_SCREEN, null),
                 new InnertubeClientRequestInfo.DeviceInfo(DESKTOP_CLIENT_PLATFORM, null, null,
                         null, null, -1));
     }
@@ -106,17 +109,27 @@ public static InnertubeClientRequestInfo ofWebClient() {
     public static InnertubeClientRequestInfo ofWebEmbeddedPlayerClient() {
         return new InnertubeClientRequestInfo(
                 new InnertubeClientRequestInfo.ClientInfo(WEB_EMBEDDED_CLIENT_NAME,
-                        WEB_REMIX_HARDCODED_CLIENT_VERSION, EMBED_CLIENT_SCREEN,
-                        WEB_EMBEDDED_CLIENT_ID, null),
+                        WEB_EMBEDDED_CLIENT_VERSION, WEB_EMBEDDED_CLIENT_ID, EMBED_CLIENT_SCREEN,
+                        null),
                 new InnertubeClientRequestInfo.DeviceInfo(DESKTOP_CLIENT_PLATFORM, null, null,
                         null, null, -1));
     }
 
+    @Nonnull
+    public static InnertubeClientRequestInfo ofWebMusicAnalyticsChartsClient() {
+        return new InnertubeClientRequestInfo(
+                new InnertubeClientRequestInfo.ClientInfo(WEB_MUSIC_ANALYTICS_CLIENT_NAME,
+                        WEB_MUSIC_ANALYTICS_CLIENT_VERSION, WEB_MUSIC_ANALYTICS_CLIENT_ID, null,
+                        null),
+                new InnertubeClientRequestInfo.DeviceInfo(null, null, null,
+                        null, null, -1));
+    }
+
     @Nonnull
     public static InnertubeClientRequestInfo ofAndroidClient() {
         return new InnertubeClientRequestInfo(
                 new InnertubeClientRequestInfo.ClientInfo(ANDROID_CLIENT_NAME,
-                        ANDROID_CLIENT_VERSION, WATCH_CLIENT_SCREEN, ANDROID_CLIENT_ID, null),
+                        ANDROID_CLIENT_VERSION, ANDROID_CLIENT_ID, WATCH_CLIENT_SCREEN, null),
                 new InnertubeClientRequestInfo.DeviceInfo(MOBILE_CLIENT_PLATFORM, null, null,
                         "Android", "15", 35));
     }
@@ -125,7 +138,7 @@ public static InnertubeClientRequestInfo ofAndroidClient() {
     public static InnertubeClientRequestInfo ofIosClient() {
         return new InnertubeClientRequestInfo(
                 new InnertubeClientRequestInfo.ClientInfo(IOS_CLIENT_NAME, IOS_CLIENT_VERSION,
-                        WATCH_CLIENT_SCREEN, IOS_CLIENT_ID, null),
+                        IOS_CLIENT_ID, WATCH_CLIENT_SCREEN, null),
                 new InnertubeClientRequestInfo.DeviceInfo(MOBILE_CLIENT_PLATFORM, "Apple",
                         IOS_DEVICE_MODEL, "iOS", IOS_OS_VERSION, -1));
     }