Merge branch 'dev' into Refactor-date-parsing

# Conflicts:
#	extractor/src/main/java/org/schabi/newpipe/extractor/localization/DateWrapper.java

Author: Isira-Seneviratne

README.md

@@ -20,7 +20,8 @@ If you're using Gradle, you could add NewPipe Extractor as a dependency with the
 -dontwarn org.mozilla.javascript.tools.**
 ```
 
-**Note:** To use NewPipe Extractor in Android projects with a `minSdk` below 33, [core library desugaring](https://developer.android.com/studio/write/java8-support#library-desugaring) with the `desugar_jdk_libs_nio` artifact is required.
+> [!NOTE]
+> To use NewPipe Extractor in Android projects with a `minSdk` below 33, [core library desugaring](https://developer.android.com/studio/write/java8-support#library-desugaring) with the `desugar_jdk_libs_nio` artifact is required.
 
 ### Testing changes
 
@@ -41,6 +42,8 @@ Another approach would be to use the local Maven repository, here's a gist of ho
 3. Run gradle's `ìnstall` task to deploy this library to your local repository (using the wrapper, present in the root of this project: `./gradlew install`)
 4. Change the dependency version used in your project to match the one you chose in step 2 (`implementation 'com.github.teamnewpipe:NewPipeExtractor:LOCAL_SNAPSHOT'`)
 
+
+> [!TIP]
 > Tip for Android Studio users: After you make changes and run the `install` task, use the menu option `File → "Sync with File System"` to refresh the library in your project.
 
 ## Supported sites

build.gradle

@@ -29,8 +29,8 @@ allprojects {
     ext {
         nanojsonVersion = "e9d656ddb49a412a5a0a5d5ef20ca7ef09549996"
         jsr305Version = "3.0.2"
-        junitVersion = "5.13.4"
-        checkstyleVersion = "10.4"
+        junitVersion = "5.14.0"
+        checkstyleVersion = "10.26.1"
     }
 }
 

extractor/build.gradle

@@ -28,7 +28,7 @@ checkstyleTest {
 
 ext {
     rhinoVersion = '1.8.0'
-    protobufVersion = '4.32.0'
+    protobufVersion = '4.32.1'
 }
 
 dependencies {
@@ -51,7 +51,7 @@ dependencies {
     testImplementation 'org.junit.jupiter:junit-jupiter-params'
 
     testImplementation "com.squareup.okhttp3:okhttp:4.12.0"
-    testImplementation 'com.google.code.gson:gson:2.13.1'
+    testImplementation 'com.google.code.gson:gson:2.13.2'
 }
 
 protobuf {

extractor/src/main/java/org/schabi/newpipe/extractor/localization/DateWrapper.java

@@ -82,6 +82,14 @@ public boolean isApproximation() {
         return isApproximation;
     }
 
+    @Override
+    public String toString() {
+        return "DateWrapper{" +
+                "instant=" + instant +
+                ", isApproximation=" + isApproximation +
+                '}';
+    }
+
     public static DateWrapper fromOffsetDateTime(final String date) throws ParsingException {
         if (date == null) {
             return null;

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java

@@ -119,7 +119,8 @@ public long getLength() {
 
     @Override
     public long getTimeStamp() throws ParsingException {
-        return getTimestampSeconds("(#t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)");
+        final var timestamp = getTimestampSeconds("(#t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)");
+        return timestamp == -2 ? 0 : timestamp;
     }
 
     @Override
@@ -168,7 +169,7 @@ public List<AudioStream> getAudioStreams() throws ExtractionException {
 
         try {
             final JsonArray transcodings = track.getObject("media")
-                    .getArray("transcodings");
+                                                .getArray("transcodings");
             if (!isNullOrEmpty(transcodings)) {
                 // Get information about what stream formats are available
                 extractAudioStreams(transcodings, audioStreams);

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudStreamLinkHandlerFactory.java

@@ -1,5 +1,7 @@
 package org.schabi.newpipe.extractor.services.soundcloud.linkHandler;
 
+import java.util.regex.Pattern;
+
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
 import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper;
@@ -9,11 +11,18 @@
 public final class SoundcloudStreamLinkHandlerFactory extends LinkHandlerFactory {
     private static final SoundcloudStreamLinkHandlerFactory INSTANCE
             = new SoundcloudStreamLinkHandlerFactory();
-    private static final String URL_PATTERN = "^https?://(www\\.|m\\.|on\\.)?"
-            + "soundcloud.com/[0-9a-z_-]+"
-            + "/(?!(tracks|albums|sets|reposts|followers|following)/?$)[0-9a-z_-]+/?([#?].*)?$";
-    private static final String API_URL_PATTERN = "^https?://api-v2\\.soundcloud.com"
-            + "/(tracks|albums|sets|reposts|followers|following)/([0-9a-z_-]+)/";
+
+    private static final Pattern URL_PATTERN = Pattern.compile(
+        "^https?://(?:www\\.|m\\.|on\\.)?"
+        + "soundcloud.com/[0-9a-z_-]+"
+        + "/(?!(?:tracks|albums|sets|reposts|followers|following)/?$)[0-9a-z_-]+/?(?:[#?].*)?$"
+        );
+
+    private static final Pattern API_URL_PATTERN = Pattern.compile(
+        "^https?://api-v2\\.soundcloud.com"
+        + "/(tracks|albums|sets|reposts|followers|following)/([0-9a-z_-]+)/"
+        );
+
     private SoundcloudStreamLinkHandlerFactory() {
     }
 

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java

@@ -35,6 +35,7 @@
 import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
 import org.schabi.newpipe.extractor.stream.StreamType;
+import org.schabi.newpipe.extractor.stream.ContentAvailability;
 import org.schabi.newpipe.extractor.utils.JsonUtils;
 import org.schabi.newpipe.extractor.utils.Parser;
 import org.schabi.newpipe.extractor.utils.Utils;
@@ -472,4 +473,33 @@ public boolean isShortFormContent() throws ParsingException {
             throw new ParsingException("Could not determine if this is short-form content", e);
         }
     }
+
+    private boolean isMembersOnly() throws ParsingException {
+        return videoInfo.getArray("badges")
+            .stream()
+            .filter(JsonObject.class::isInstance)
+            .map(JsonObject.class::cast)
+            .map(badge -> badge.getObject("metadataBadgeRenderer").getString("style"))
+            .anyMatch("BADGE_STYLE_TYPE_MEMBERS_ONLY"::equals);
+    }
+
+
+    @Nonnull
+    @Override
+    public ContentAvailability getContentAvailability() throws ParsingException {
+        if (isPremiere()) {
+            return ContentAvailability.UPCOMING;
+        }
+
+        if (isMembersOnly()) {
+            return ContentAvailability.MEMBERSHIP;
+        }
+
+        if (isPremium()) {
+            return ContentAvailability.PAID;
+        }
+
+        return ContentAvailability.AVAILABLE;
+    }
+
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemLockupExtractor.java

@@ -17,6 +17,11 @@
 import org.schabi.newpipe.extractor.utils.JsonUtils;
 import org.schabi.newpipe.extractor.utils.Utils;
 
+import java.time.LocalDateTime;
+import java.time.OffsetDateTime;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeParseException;
 import java.util.List;
 import java.util.Optional;
 import java.util.stream.Collectors;
@@ -30,20 +35,24 @@
  * The following features are currently not implemented because they have never been observed:
  * <ul>
  *     <li>Shorts</li>
- *     <li>Premieres</li>
  *     <li>Paid content (Premium, members first or only)</li>
  * </ul>
  */
 public class YoutubeStreamInfoItemLockupExtractor implements StreamInfoItemExtractor {
 
     private static final String NO_VIEWS_LOWERCASE = "no views";
+    // This approach is language dependant (en-GB)
+    // Leading end space is voluntary included
+    private static final String PREMIERES_TEXT = "Premieres ";
+    private static final DateTimeFormatter PREMIERES_DATE_FORMATTER =
+            DateTimeFormatter.ofPattern("dd/MM/yyyy, HH:mm");
 
     private final JsonObject lockupViewModel;
     private final TimeAgoParser timeAgoParser;
 
     private StreamType cachedStreamType;
     private String cachedName;
-    private Optional<String> cachedTextualUploadDate;
+    private Optional<String> cachedDateText;
 
     private ChannelImageViewModel cachedChannelImageViewModel;
     private JsonArray cachedMetadataRows;
@@ -137,7 +146,9 @@ public String getName() throws ParsingException {
     @Override
     public long getDuration() throws ParsingException {
         // Duration cannot be extracted for live streams, but only for normal videos
-        if (isLive()) {
+        // Exact duration cannot be extracted for premieres, an approximation is only available in
+        // accessibility context label
+        if (isLive() || isPremiere()) {
             return -1;
         }
 
@@ -237,20 +248,27 @@ public boolean isUploaderVerified() throws ParsingException {
     @Nullable
     @Override
     public String getTextualUploadDate() throws ParsingException {
-        if (cachedTextualUploadDate != null) {
-            return cachedTextualUploadDate.orElse(null);
-        }
-
         // Live streams have no upload date
         if (isLive()) {
-            cachedTextualUploadDate = Optional.empty();
             return null;
         }
 
-        // This might be null e.g. for live streams
-        this.cachedTextualUploadDate = metadataPart(1, 1)
-            .map(this::getTextContentFromMetadataPart);
-        return cachedTextualUploadDate.orElse(null);
+        // Date string might be null e.g. for live streams
+        final Optional<String> dateText = getDateText();
+
+        if (isPremiere()) {
+            return getDateFromPremiere(dateText);
+        }
+
+        return dateText.orElse(null);
+    }
+
+    @Nullable
+    private String getDateFromPremiere(final Optional<String> dateText) {
+        // This approach is language dependent
+        // Remove the premieres text from the upload date metadata part
+        return dateText.map(str -> str.replace(PREMIERES_TEXT, ""))
+                        .orElse(null);
     }
 
     @Nullable
@@ -265,11 +283,32 @@ public DateWrapper getUploadDate() throws ParsingException {
         if (textualUploadDate == null) {
             return null;
         }
+
+        if (isPremiere()) {
+            final String premiereDate = getDateFromPremiere(getDateText());
+            if (premiereDate == null) {
+                throw new ParsingException("Could not get upload date from premiere");
+            }
+
+            try {
+                // As we request a UTC offset of 0 minutes, we get the UTC date
+                return new DateWrapper(OffsetDateTime.of(LocalDateTime.parse(
+                        premiereDate, PREMIERES_DATE_FORMATTER), ZoneOffset.UTC));
+            } catch (final DateTimeParseException e) {
+                throw new ParsingException("Could not parse premiere upload date", e);
+            }
+        }
+
         return timeAgoParser.parse(textualUploadDate);
     }
 
     @Override
     public long getViewCount() throws ParsingException {
+        if (isPremiere()) {
+            // The number of people returned for premieres is the one currently waiting
+            return -1;
+        }
+
         final Optional<String> optTextContent = metadataPart(1, 0)
             .map(this::getTextContentFromMetadataPart);
         // We could do this inline if the ParsingException would be a RuntimeException -.-
@@ -357,6 +396,20 @@ private boolean isLive() throws ParsingException {
         return getStreamType() != StreamType.VIDEO_STREAM;
     }
 
+    private Optional<String> getDateText() throws ParsingException {
+        if (cachedDateText == null) {
+            cachedDateText = metadataPart(1, 1)
+                    .map(this::getTextContentFromMetadataPart);
+        }
+        return cachedDateText;
+    }
+
+    private boolean isPremiere() throws ParsingException {
+        return getDateText().map(dateText -> dateText.contains(PREMIERES_TEXT))
+                // If we can't get date text, assume it is not a premiere, it should be a livestream
+                .orElse(false);
+    }
+
     abstract static class ChannelImageViewModel {
         protected JsonObject viewModel;