From 953189f0ba883a63ed7b47b89c6c64126fb52ab7 Mon Sep 17 00:00:00 2001 From: Isira Seneviratne Date: Thu, 11 Sep 2025 07:34:49 +0530 Subject: [PATCH 01/20] Update DateWrapper to store an Instant --- .../extractor/localization/DateWrapper.java | 45 +++++++------------ .../extractor/localization/TimeAgoParser.java | 3 +- .../extractors/BandcampExtractorHelper.java | 2 +- .../extractors/MediaCCCRecentKiosk.java | 2 +- .../MediaCCCRecentKioskExtractor.java | 2 +- .../peertube/PeertubeParsingHelper.java | 7 +-- .../PeertubeCommentsInfoItemExtractor.java | 5 +-- .../extractors/PeertubeStreamExtractor.java | 2 +- .../PeertubeStreamInfoItemExtractor.java | 4 +- .../youtube/YoutubeParsingHelper.java | 10 +++-- .../extractors/YoutubeStreamExtractor.java | 27 ++++++----- .../YoutubeStreamInfoItemExtractor.java | 18 ++++---- .../YoutubeChartsBaseKioskExtractor.java | 23 ++++------ 13 files changed, 65 insertions(+), 85 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/localization/DateWrapper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/localization/DateWrapper.java index ffc29a61ce..b26c3b84fa 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/localization/DateWrapper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/localization/DateWrapper.java @@ -1,12 +1,10 @@ package org.schabi.newpipe.extractor.localization; - import javax.annotation.Nonnull; import java.io.Serializable; +import java.time.Instant; import java.time.OffsetDateTime; import java.time.ZoneOffset; -import java.util.Calendar; -import java.util.GregorianCalendar; /** * A wrapper class that provides a field to describe if the date/time is precise or just an @@ -14,52 +12,41 @@ */ public class DateWrapper implements Serializable { @Nonnull - private final OffsetDateTime offsetDateTime; + private final Instant instant; private final boolean isApproximation; - /** - * @deprecated Use {@link #DateWrapper(OffsetDateTime)} instead. - */ - @Deprecated - public DateWrapper(@Nonnull final Calendar calendar) { - //noinspection deprecation - this(calendar, false); - } - - /** - * @deprecated Use {@link #DateWrapper(OffsetDateTime, boolean)} instead. - */ - @Deprecated - public DateWrapper(@Nonnull final Calendar calendar, final boolean isApproximation) { - this(OffsetDateTime.ofInstant(calendar.toInstant(), ZoneOffset.UTC), isApproximation); - } - public DateWrapper(@Nonnull final OffsetDateTime offsetDateTime) { this(offsetDateTime, false); } public DateWrapper(@Nonnull final OffsetDateTime offsetDateTime, final boolean isApproximation) { - this.offsetDateTime = offsetDateTime.withOffsetSameInstant(ZoneOffset.UTC); + this(offsetDateTime.toInstant(), isApproximation); + } + + public DateWrapper(@Nonnull final Instant instant) { + this(instant, false); + } + + public DateWrapper(@Nonnull final Instant instant, final boolean isApproximation) { + this.instant = instant; this.isApproximation = isApproximation; } /** - * @return the wrapped date/time as a {@link Calendar}. - * @deprecated use {@link #offsetDateTime()} instead. + * @return the wrapped {@link Instant} */ - @Deprecated @Nonnull - public Calendar date() { - return GregorianCalendar.from(offsetDateTime.toZonedDateTime()); + public Instant getInstant() { + return instant; } /** - * @return the wrapped date/time. + * @return the wrapped {@link Instant} as an {@link OffsetDateTime} set to UTC. */ @Nonnull public OffsetDateTime offsetDateTime() { - return offsetDateTime; + return instant.atOffset(ZoneOffset.UTC); } /** diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/localization/TimeAgoParser.java b/extractor/src/main/java/org/schabi/newpipe/extractor/localization/TimeAgoParser.java index 619de9e74b..0eb06e1032 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/localization/TimeAgoParser.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/localization/TimeAgoParser.java @@ -5,7 +5,6 @@ import org.schabi.newpipe.extractor.utils.Parser; import java.time.OffsetDateTime; -import java.time.ZoneOffset; import java.time.temporal.ChronoUnit; import java.util.Map; import java.util.regex.Pattern; @@ -28,7 +27,7 @@ public class TimeAgoParser { * language word separator. */ public TimeAgoParser(final PatternsHolder patternsHolder) { - this(patternsHolder, OffsetDateTime.now(ZoneOffset.UTC)); + this(patternsHolder, OffsetDateTime.now()); } /** diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java index b230079109..99f026d7ed 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java @@ -204,7 +204,7 @@ public static DateWrapper parseDate(final String textDate) throws ParsingExcepti try { final ZonedDateTime zonedDateTime = ZonedDateTime.parse(textDate, DateTimeFormatter.ofPattern("dd MMM yyyy HH:mm:ss zzz", Locale.ENGLISH)); - return new DateWrapper(zonedDateTime.toOffsetDateTime(), false); + return new DateWrapper(zonedDateTime.toInstant()); } catch (final DateTimeException e) { throw new ParsingException("Could not parse date '" + textDate + "'", e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCRecentKiosk.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCRecentKiosk.java index d38d65fd53..33583bc644 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCRecentKiosk.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCRecentKiosk.java @@ -54,7 +54,7 @@ public InfoItemsPage getInitialPage() throws IOException, Extrac // Sort them to have the latest stream at the beginning of the list. final Comparator comparator = Comparator .comparing(StreamInfoItem::getUploadDate, Comparator - .nullsLast(Comparator.comparing(DateWrapper::offsetDateTime))) + .nullsLast(Comparator.comparing(DateWrapper::getInstant))) .reversed(); final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId(), comparator); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCRecentKioskExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCRecentKioskExtractor.java index df25b28d89..a8d5ae3c36 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCRecentKioskExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCRecentKioskExtractor.java @@ -92,6 +92,6 @@ public String getTextualUploadDate() throws ParsingException { public DateWrapper getUploadDate() throws ParsingException { final ZonedDateTime zonedDateTime = ZonedDateTime.parse(event.getString("date"), DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSzzzz")); - return new DateWrapper(zonedDateTime.toOffsetDateTime(), false); + return new DateWrapper(zonedDateTime.toInstant()); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java index 4e8bd2d350..8f2c7e2761 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java @@ -19,8 +19,6 @@ import javax.annotation.Nonnull; import java.time.Instant; -import java.time.OffsetDateTime; -import java.time.ZoneOffset; import java.time.format.DateTimeParseException; import java.util.ArrayList; import java.util.Collections; @@ -48,10 +46,9 @@ public static void validate(final JsonObject json) throws ContentNotAvailableExc } } - public static OffsetDateTime parseDateFrom(final String textualUploadDate) - throws ParsingException { + public static Instant parseInstantFrom(final String textualUploadDate) throws ParsingException { try { - return OffsetDateTime.ofInstant(Instant.parse(textualUploadDate), ZoneOffset.UTC); + return Instant.parse(textualUploadDate); } catch (final DateTimeParseException e) { throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java index 57a0444906..f53149d538 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java @@ -23,7 +23,7 @@ import static org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeCommentsExtractor.CHILDREN; import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.getAvatarsFromOwnerAccountOrVideoChannelObject; -import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.parseDateFrom; +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.parseInstantFrom; public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtractor { @Nonnull @@ -73,8 +73,7 @@ public String getTextualUploadDate() throws ParsingException { @Override public DateWrapper getUploadDate() throws ParsingException { - final String textualUploadDate = getTextualUploadDate(); - return new DateWrapper(parseDateFrom(textualUploadDate)); + return new DateWrapper(parseInstantFrom(getTextualUploadDate())); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index 5406368854..c1deea6c6b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -85,7 +85,7 @@ public DateWrapper getUploadDate() throws ParsingException { return null; } - return new DateWrapper(PeertubeParsingHelper.parseDateFrom(textualUploadDate)); + return new DateWrapper(PeertubeParsingHelper.parseInstantFrom(textualUploadDate)); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java index 46aae43ccf..c7730a8d32 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java @@ -14,7 +14,7 @@ import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.getAvatarsFromOwnerAccountOrVideoChannelObject; import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.getThumbnailsFromPlaylistOrVideoItem; -import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.parseDateFrom; +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.parseInstantFrom; public class PeertubeStreamInfoItemExtractor implements StreamInfoItemExtractor { @@ -91,7 +91,7 @@ public DateWrapper getUploadDate() throws ParsingException { return null; } - return new DateWrapper(parseDateFrom(textualUploadDate)); + return new DateWrapper(parseInstantFrom(textualUploadDate)); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index 70aac8d2b5..ce53dc8882 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -67,9 +67,10 @@ import java.net.MalformedURLException; import java.net.URL; import java.nio.charset.StandardCharsets; +import java.time.Instant; import java.time.LocalDate; import java.time.OffsetDateTime; -import java.time.ZoneOffset; +import java.time.ZoneId; import java.time.format.DateTimeParseException; import java.util.HashMap; import java.util.List; @@ -294,13 +295,14 @@ public static String getFeedUrlFrom(@Nonnull final String channelIdOrUser) { } } - public static OffsetDateTime parseDateFrom(final String textualUploadDate) + public static Instant parseInstantFrom(final String textualUploadDate) throws ParsingException { try { - return OffsetDateTime.parse(textualUploadDate); + return OffsetDateTime.parse(textualUploadDate).toInstant(); } catch (final DateTimeParseException e) { try { - return LocalDate.parse(textualUploadDate).atStartOfDay().atOffset(ZoneOffset.UTC); + return LocalDate.parse(textualUploadDate).atStartOfDay(ZoneId.systemDefault()) + .toInstant(); } catch (final DateTimeParseException e1) { throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e1); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index eb050fd13e..61d8a25e7d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -87,7 +87,7 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.time.LocalDate; -import java.time.OffsetDateTime; +import java.time.ZoneId; import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Arrays; @@ -196,24 +196,24 @@ public String getTextualUploadDate() throws ParsingException { final String time = videoPrimaryInfoRendererDateText.substring(13); try { // Premiered 20 hours ago - final TimeAgoParser timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor( + final var timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor( new Localization("en")); - final OffsetDateTime parsedTime = timeAgoParser.parse(time).offsetDateTime(); - return DateTimeFormatter.ISO_LOCAL_DATE.format(parsedTime); + final var instant = timeAgoParser.parse(time).getInstant(); + return LocalDate.ofInstant(instant, ZoneId.systemDefault()).toString(); } catch (final Exception ignored) { } try { // Premiered Feb 21, 2020 - final LocalDate localDate = LocalDate.parse(time, - DateTimeFormatter.ofPattern("MMM dd, yyyy", Locale.ENGLISH)); - return DateTimeFormatter.ISO_LOCAL_DATE.format(localDate); + final var formatter = DateTimeFormatter.ofPattern("MMM dd, yyyy", + Locale.ENGLISH); + return LocalDate.parse(time, formatter).toString(); } catch (final Exception ignored) { } try { // Premiered on 21 Feb 2020 - final LocalDate localDate = LocalDate.parse(time, - DateTimeFormatter.ofPattern("dd MMM yyyy", Locale.ENGLISH)); - return DateTimeFormatter.ISO_LOCAL_DATE.format(localDate); + final var formatter = DateTimeFormatter.ofPattern("dd MMM yyyy", + Locale.ENGLISH); + return LocalDate.parse(time, formatter).toString(); } catch (final Exception ignored) { } } @@ -221,9 +221,8 @@ public String getTextualUploadDate() throws ParsingException { try { // TODO: this parses English formatted dates only, we need a better approach to // parse the textual date - final LocalDate localDate = LocalDate.parse(videoPrimaryInfoRendererDateText, - DateTimeFormatter.ofPattern("dd MMM yyyy", Locale.ENGLISH)); - return DateTimeFormatter.ISO_LOCAL_DATE.format(localDate); + final var formatter = DateTimeFormatter.ofPattern("dd MMM yyyy", Locale.ENGLISH); + return LocalDate.parse(videoPrimaryInfoRendererDateText, formatter).toString(); } catch (final Exception e) { throw new ParsingException("Could not get upload date", e); } @@ -240,7 +239,7 @@ public DateWrapper getUploadDate() throws ParsingException { return null; } - return new DateWrapper(YoutubeParsingHelper.parseDateFrom(textualUploadDate), true); + return new DateWrapper(YoutubeParsingHelper.parseInstantFrom(textualUploadDate), true); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java index d00cee987a..f75a972189 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java @@ -43,8 +43,8 @@ import javax.annotation.Nullable; import java.time.Instant; -import java.time.OffsetDateTime; -import java.time.ZoneOffset; +import java.time.LocalDateTime; +import java.time.ZoneId; import java.time.format.DateTimeFormatter; import java.util.List; import java.util.regex.Pattern; @@ -251,7 +251,9 @@ public String getTextualUploadDate() throws ParsingException { } if (isPremiere()) { - return DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm").format(getDateFromPremiere()); + final var localDateTime = LocalDateTime.ofInstant(getInstantFromPremiere(), + ZoneId.systemDefault()); + return DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm").format(localDateTime); } String publishedTimeText = getTextFromObject(videoInfo.getObject("publishedTimeText")); @@ -277,7 +279,7 @@ public DateWrapper getUploadDate() throws ParsingException { } if (isPremiere()) { - return new DateWrapper(getDateFromPremiere()); + return new DateWrapper(getInstantFromPremiere()); } final String textualUploadDate = getTextualUploadDate(); @@ -401,15 +403,15 @@ private boolean isPremiere() { return isPremiere; } - private OffsetDateTime getDateFromPremiere() throws ParsingException { + private Instant getInstantFromPremiere() throws ParsingException { final JsonObject upcomingEventData = videoInfo.getObject("upcomingEventData"); final String startTime = upcomingEventData.getString("startTime"); try { - return OffsetDateTime.ofInstant(Instant.ofEpochSecond(Long.parseLong(startTime)), - ZoneOffset.UTC); + return Instant.ofEpochSecond(Long.parseLong(startTime)); } catch (final Exception e) { - throw new ParsingException("Could not parse date from premiere: \"" + startTime + "\""); + final String message = "Could not parse date from premiere: \"" + startTime + "\""; + throw new ParsingException(message, e); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/kiosk/YoutubeChartsBaseKioskExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/kiosk/YoutubeChartsBaseKioskExtractor.java index 3ede87619e..329fe8d261 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/kiosk/YoutubeChartsBaseKioskExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/kiosk/YoutubeChartsBaseKioskExtractor.java @@ -27,7 +27,7 @@ import javax.annotation.Nullable; import java.io.IOException; import java.nio.charset.StandardCharsets; -import java.time.OffsetDateTime; +import java.time.LocalDate; import java.time.ZoneOffset; import java.util.HashMap; import java.util.List; @@ -204,19 +204,14 @@ public String getTextualUploadDate() { @Override public DateWrapper getUploadDate() { final JsonObject releaseDate = videoObject.getObject("releaseDate"); - return new DateWrapper(OffsetDateTime.of( - releaseDate.getInt("year"), - releaseDate.getInt("month"), - releaseDate.getInt("day"), - 0, - 0, - 0, - 0, - // We request that times should be returned with 0 offset to UTC timezone in - // the JSON body, but YouTube charts does it only in its HTTP headers - ZoneOffset.UTC), - // We don't have more info than the release day - true); + final var localDate = LocalDate.of(releaseDate.getInt("year"), + releaseDate.getInt("month"), releaseDate.getInt("day")); + // We request that times should be returned with 0 offset to UTC timezone in + // the JSON body, but YouTube charts does it only in its HTTP headers + final var instant = localDate.atStartOfDay(ZoneOffset.UTC).toInstant(); + + // We don't have more info than the release day + return new DateWrapper(instant, true); } @Override From 758c31b0a2f2dcce5cbb09abaceb77397852659b Mon Sep 17 00:00:00 2001 From: Isira Seneviratne Date: Fri, 12 Sep 2025 08:16:32 +0530 Subject: [PATCH 02/20] Allow retrieval of a non-approximate DateWrapper in YoutubeStreamExtractor --- .../youtube/YoutubeParsingHelper.java | 20 --- .../extractors/YoutubeStreamExtractor.java | 114 +++++++++--------- 2 files changed, 56 insertions(+), 78 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index ce53dc8882..df295f0d5c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -67,11 +67,6 @@ import java.net.MalformedURLException; import java.net.URL; import java.nio.charset.StandardCharsets; -import java.time.Instant; -import java.time.LocalDate; -import java.time.OffsetDateTime; -import java.time.ZoneId; -import java.time.format.DateTimeParseException; import java.util.HashMap; import java.util.List; import java.util.Locale; @@ -295,21 +290,6 @@ public static String getFeedUrlFrom(@Nonnull final String channelIdOrUser) { } } - public static Instant parseInstantFrom(final String textualUploadDate) - throws ParsingException { - try { - return OffsetDateTime.parse(textualUploadDate).toInstant(); - } catch (final DateTimeParseException e) { - try { - return LocalDate.parse(textualUploadDate).atStartOfDay(ZoneId.systemDefault()) - .toInstant(); - } catch (final DateTimeParseException e1) { - throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", - e1); - } - } - } - /** * Checks if the given playlist id is a YouTube Mix (auto-generated playlist) * Ids from a YouTube Mix start with "RD" diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 61d8a25e7d..e6e5320346 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -20,6 +20,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; +import static org.schabi.newpipe.extractor.localization.TimeAgoPatternsManager.getTimeAgoParserFor; import static org.schabi.newpipe.extractor.services.youtube.ItagItem.APPROX_DURATION_MS_UNKNOWN; import static org.schabi.newpipe.extractor.services.youtube.ItagItem.CONTENT_LENGTH_UNKNOWN; import static org.schabi.newpipe.extractor.services.youtube.YoutubeDescriptionHelper.attributedDescriptionToHtml; @@ -59,7 +60,6 @@ import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.localization.TimeAgoParser; -import org.schabi.newpipe.extractor.localization.TimeAgoPatternsManager; import org.schabi.newpipe.extractor.services.youtube.ItagItem; import org.schabi.newpipe.extractor.services.youtube.PoTokenProvider; import org.schabi.newpipe.extractor.services.youtube.PoTokenResult; @@ -87,14 +87,17 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.time.LocalDate; +import java.time.OffsetDateTime; import java.time.ZoneId; import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Locale; import java.util.Objects; +import java.util.Optional; import java.util.stream.Collectors; import javax.annotation.Nonnull; @@ -169,77 +172,72 @@ public String getName() throws ParsingException { @Nullable @Override public String getTextualUploadDate() throws ParsingException { - if (!playerMicroFormatRenderer.getString("uploadDate", "").isEmpty()) { - return playerMicroFormatRenderer.getString("uploadDate"); - } else if (!playerMicroFormatRenderer.getString("publishDate", "").isEmpty()) { - return playerMicroFormatRenderer.getString("publishDate"); + final var uploadDate = getUploadDate(); + if (uploadDate == null) { + return null; } + return LocalDate.ofInstant(uploadDate.getInstant(), ZoneId.systemDefault()).toString(); + } + + @Override + public DateWrapper getUploadDate() throws ParsingException { + final String dateStr = playerMicroFormatRenderer.getString("uploadDate", + playerMicroFormatRenderer.getString("publishDate", "")); + if (!dateStr.isEmpty()) { + return new DateWrapper(OffsetDateTime.parse(dateStr)); + } + + final var liveDetails = playerMicroFormatRenderer.getObject("liveBroadcastDetails"); + final String timestamp = liveDetails.getString("endTimestamp", // an ended live stream + liveDetails.getString("startTimestamp", "")); // a running live stream - final JsonObject liveDetails = playerMicroFormatRenderer.getObject( - "liveBroadcastDetails"); - if (!liveDetails.getString("endTimestamp", "").isEmpty()) { - // an ended live stream - return liveDetails.getString("endTimestamp"); - } else if (!liveDetails.getString("startTimestamp", "").isEmpty()) { - // a running live stream - return liveDetails.getString("startTimestamp"); + if (!timestamp.isEmpty()) { + return new DateWrapper(OffsetDateTime.parse(timestamp)); } else if (getStreamType() == StreamType.LIVE_STREAM) { // this should never be reached, but a live stream without upload date is valid return null; } - final String videoPrimaryInfoRendererDateText = - getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")); + final var textObject = getVideoPrimaryInfoRenderer().getObject("dateText"); + return Optional.ofNullable(getTextFromObject(textObject)) + .flatMap(rendererDateText -> { + final Optional dateOptional; - if (videoPrimaryInfoRendererDateText != null) { - if (videoPrimaryInfoRendererDateText.startsWith("Premiered")) { - final String time = videoPrimaryInfoRendererDateText.substring(13); + if (rendererDateText.startsWith("Premiered")) { + final String time = rendererDateText.substring(13); - try { // Premiered 20 hours ago - final var timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor( - new Localization("en")); - final var instant = timeAgoParser.parse(time).getInstant(); - return LocalDate.ofInstant(instant, ZoneId.systemDefault()).toString(); - } catch (final Exception ignored) { - } - - try { // Premiered Feb 21, 2020 - final var formatter = DateTimeFormatter.ofPattern("MMM dd, yyyy", - Locale.ENGLISH); - return LocalDate.parse(time, formatter).toString(); - } catch (final Exception ignored) { - } - - try { // Premiered on 21 Feb 2020 - final var formatter = DateTimeFormatter.ofPattern("dd MMM yyyy", - Locale.ENGLISH); - return LocalDate.parse(time, formatter).toString(); - } catch (final Exception ignored) { - } - } + try { // Premiered 20 hours ago + final var localization = new Localization("en"); + return Optional.of(getTimeAgoParserFor(localization).parse(time)); + } catch (final Exception e) { + } - try { - // TODO: this parses English formatted dates only, we need a better approach to - // parse the textual date - final var formatter = DateTimeFormatter.ofPattern("dd MMM yyyy", Locale.ENGLISH); - return LocalDate.parse(videoPrimaryInfoRendererDateText, formatter).toString(); - } catch (final Exception e) { - throw new ParsingException("Could not get upload date", e); - } - } + // Premiered Feb 21, 2020 + dateOptional = parseOptionalDate(time, "MMM dd, yyyy") + // Premiered on 21 Feb 2020 + .or(() -> parseOptionalDate(time, "dd MMM yyyy")); + } else { + // Premiered on 21 Feb 2020 + dateOptional = parseOptionalDate(rendererDateText, "dd MMM yyyy"); + } - throw new ParsingException("Could not get upload date"); + return dateOptional.map(date -> { + final var instant = date.atStartOfDay(ZoneId.systemDefault()).toInstant(); + return new DateWrapper(instant, true); + }); + }) + .orElseThrow(() -> new ParsingException("Could not get upload date")); } - @Override - public DateWrapper getUploadDate() throws ParsingException { - final String textualUploadDate = getTextualUploadDate(); - - if (isNullOrEmpty(textualUploadDate)) { - return null; + private Optional parseOptionalDate(String date, String pattern) { + try { + // TODO: this parses English formatted dates only, we need a better approach to + // parse the textual date + final var formatter = DateTimeFormatter.ofPattern(pattern, Locale.ENGLISH); + return Optional.of(LocalDate.parse(date, formatter)); + } catch (DateTimeParseException e) { + return Optional.empty(); } - - return new DateWrapper(YoutubeParsingHelper.parseInstantFrom(textualUploadDate), true); } @Nonnull From 2dc17d4a53a09815b59de211689ea2d9e98cc462 Mon Sep 17 00:00:00 2001 From: Isira Seneviratne Date: Fri, 12 Sep 2025 08:16:40 +0530 Subject: [PATCH 03/20] Improve tests --- .../services/DefaultStreamExtractorTest.java | 6 ++++-- .../BandcampRadioStreamExtractorTest.java | 17 +++++++---------- .../youtube/YoutubeChannelLocalizationTest.java | 6 +++++- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java index fab5c8f3a1..c868b88547 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java @@ -16,6 +16,7 @@ import javax.annotation.Nullable; import java.time.LocalDateTime; +import java.time.ZoneId; import java.time.format.DateTimeFormatter; import java.net.MalformedURLException; import java.net.URL; @@ -200,9 +201,10 @@ public void testUploadDate() throws Exception { } else { assertNotNull(dateWrapper); - final LocalDateTime expectedDateTime = LocalDateTime.parse(expectedUploadDate(), + final var expectedDateTime = LocalDateTime.parse(expectedUploadDate(), DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS")); - final LocalDateTime actualDateTime = dateWrapper.offsetDateTime().toLocalDateTime(); + final var actualDateTime = LocalDateTime.ofInstant(dateWrapper.getInstant(), + ZoneId.systemDefault()); assertEquals(expectedDateTime, actualDateTime); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java index 78d490bd92..3d16145aa1 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java @@ -19,10 +19,11 @@ import org.schabi.newpipe.extractor.stream.StreamType; import java.io.IOException; -import java.util.Calendar; +import java.time.LocalDate; +import java.time.Month; +import java.time.ZoneOffset; import java.util.Collections; import java.util.List; -import java.util.TimeZone; public class BandcampRadioStreamExtractorTest extends DefaultStreamExtractorTest { @@ -82,14 +83,10 @@ public List expectedDescriptionContains() { @Override @Test public void testUploadDate() throws ParsingException { - final Calendar expectedCalendar = Calendar.getInstance(); - - // 16 May 2017 00:00:00 GMT - expectedCalendar.setTimeZone(TimeZone.getTimeZone("GMT")); - expectedCalendar.setTimeInMillis(0); - expectedCalendar.set(2017, Calendar.MAY, 16); - - assertEquals(expectedCalendar.getTimeInMillis(), extractor().getUploadDate().offsetDateTime().toInstant().toEpochMilli()); + final var expectedDate = LocalDate.of(2017, Month.MAY, 16); + final var actualDate = LocalDate.ofInstant(extractor().getUploadDate().getInstant(), + ZoneOffset.UTC); + assertEquals(expectedDate, actualDate); } @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLocalizationTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLocalizationTest.java index 6633a76f89..b00914fd14 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLocalizationTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLocalizationTest.java @@ -13,6 +13,8 @@ import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import java.time.LocalDateTime; +import java.time.ZoneId; import java.time.format.DateTimeFormatter; import java.time.temporal.ChronoUnit; import java.util.LinkedHashMap; @@ -73,7 +75,9 @@ private void testLocalizationsFor(final String channelUrl) throws Exception { + "\n:::: " + item.getStreamType() + ", views = " + item.getViewCount(); final DateWrapper uploadDate = item.getUploadDate(); if (uploadDate != null) { - final String dateAsText = dateTimeFormatter.format(uploadDate.offsetDateTime()); + final var dateTime = LocalDateTime.ofInstant(uploadDate.getInstant(), + ZoneId.systemDefault()); + final String dateAsText = dateTimeFormatter.format(dateTime); debugMessage += "\n:::: " + item.getTextualUploadDate() + "\n:::: " + dateAsText; } From 6bc9d32d65fcd2ca0dcaa3f3dda2b7d25a3ea96e Mon Sep 17 00:00:00 2001 From: Isira Seneviratne Date: Fri, 12 Sep 2025 22:05:27 +0530 Subject: [PATCH 04/20] Fix checkstyle issues, added more refactorings --- .../extractors/MediaCCCParsingHelper.java | 12 --- .../extractors/MediaCCCStreamExtractor.java | 4 +- .../MediaCCCStreamInfoItemExtractor.java | 8 +- .../YoutubeFeedInfoItemExtractor.java | 11 +-- .../extractors/YoutubeStreamExtractor.java | 81 ++++++++++--------- .../YoutubeStreamInfoItemExtractor.java | 4 +- .../extractor/stream/StreamExtractor.java | 3 +- .../extractor/utils/ExtractorHelper.java | 15 ++++ 8 files changed, 69 insertions(+), 69 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCParsingHelper.java index 2a4ea6a9f8..89ef3943b0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCParsingHelper.java @@ -8,15 +8,12 @@ import org.schabi.newpipe.extractor.Image.ResolutionLevel; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; -import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.localization.Localization; import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.io.IOException; -import java.time.OffsetDateTime; -import java.time.format.DateTimeParseException; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -33,15 +30,6 @@ public final class MediaCCCParsingHelper { private MediaCCCParsingHelper() { } - public static OffsetDateTime parseDateFrom(final String textualUploadDate) - throws ParsingException { - try { - return OffsetDateTime.parse(textualUploadDate); - } catch (final DateTimeParseException e) { - throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e); - } - } - /** * Check whether an id is a live stream id * @param id the {@code id} to check diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java index 99ddf5e08c..23ebf44330 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java @@ -2,7 +2,6 @@ import static org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCParsingHelper.getImageListFromLogoImageUrl; import static org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCParsingHelper.getThumbnailsFromStreamItem; -import static org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCParsingHelper.parseDateFrom; import static org.schabi.newpipe.extractor.stream.AudioStream.UNKNOWN_BITRATE; import static org.schabi.newpipe.extractor.stream.Stream.ID_UNKNOWN; @@ -27,6 +26,7 @@ import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.stream.VideoStream; +import org.schabi.newpipe.extractor.utils.ExtractorHelper; import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.LocaleCompat; @@ -55,7 +55,7 @@ public String getTextualUploadDate() { @Nonnull @Override public DateWrapper getUploadDate() throws ParsingException { - return new DateWrapper(parseDateFrom(getTextualUploadDate())); + return ExtractorHelper.parseDateWrapper(getTextualUploadDate()); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCStreamInfoItemExtractor.java index ec9d00f3a0..82687c3285 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCStreamInfoItemExtractor.java @@ -4,9 +4,9 @@ import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; -import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamType; +import org.schabi.newpipe.extractor.utils.ExtractorHelper; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -66,10 +66,8 @@ public String getTextualUploadDate() { @Override public DateWrapper getUploadDate() throws ParsingException { final String date = getTextualUploadDate(); - if (date == null) { - return null; // event is in the future... - } - return new DateWrapper(MediaCCCParsingHelper.parseDateFrom(date)); + // if null, event is in the future... + return date == null ? null : ExtractorHelper.parseDateWrapper(date); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedInfoItemExtractor.java index d917eb2d7b..74dc19b915 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedInfoItemExtractor.java @@ -7,11 +7,10 @@ import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamType; +import org.schabi.newpipe.extractor.utils.ExtractorHelper; import javax.annotation.Nonnull; import javax.annotation.Nullable; -import java.time.OffsetDateTime; -import java.time.format.DateTimeParseException; import java.util.List; public class YoutubeFeedInfoItemExtractor implements StreamInfoItemExtractor { @@ -69,12 +68,8 @@ public String getTextualUploadDate() { @Nullable @Override public DateWrapper getUploadDate() throws ParsingException { - try { - return new DateWrapper(OffsetDateTime.parse(getTextualUploadDate())); - } catch (final DateTimeParseException e) { - throw new ParsingException("Could not parse date (\"" + getTextualUploadDate() + "\")", - e); - } + final String date = getTextualUploadDate(); + return date == null ? null : ExtractorHelper.parseDateWrapper(date); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index e6e5320346..01a0883178 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -20,7 +20,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; -import static org.schabi.newpipe.extractor.localization.TimeAgoPatternsManager.getTimeAgoParserFor; import static org.schabi.newpipe.extractor.services.youtube.ItagItem.APPROX_DURATION_MS_UNKNOWN; import static org.schabi.newpipe.extractor.services.youtube.ItagItem.CONTENT_LENGTH_UNKNOWN; import static org.schabi.newpipe.extractor.services.youtube.YoutubeDescriptionHelper.attributedDescriptionToHtml; @@ -60,6 +59,7 @@ import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.localization.TimeAgoParser; +import org.schabi.newpipe.extractor.localization.TimeAgoPatternsManager; import org.schabi.newpipe.extractor.services.youtube.ItagItem; import org.schabi.newpipe.extractor.services.youtube.PoTokenProvider; import org.schabi.newpipe.extractor.services.youtube.PoTokenResult; @@ -104,6 +104,8 @@ import javax.annotation.Nullable; public class YoutubeStreamExtractor extends StreamExtractor { + private static final String PREMIERED = "Premiered "; + private static final String PREMIERED_ON = "Premiered on "; @Nullable private static PoTokenProvider poTokenProvider; @@ -172,19 +174,10 @@ public String getName() throws ParsingException { @Nullable @Override public String getTextualUploadDate() throws ParsingException { - final var uploadDate = getUploadDate(); - if (uploadDate == null) { - return null; - } - return LocalDate.ofInstant(uploadDate.getInstant(), ZoneId.systemDefault()).toString(); - } - - @Override - public DateWrapper getUploadDate() throws ParsingException { final String dateStr = playerMicroFormatRenderer.getString("uploadDate", playerMicroFormatRenderer.getString("publishDate", "")); if (!dateStr.isEmpty()) { - return new DateWrapper(OffsetDateTime.parse(dateStr)); + return dateStr; } final var liveDetails = playerMicroFormatRenderer.getObject("liveBroadcastDetails"); @@ -192,50 +185,60 @@ public DateWrapper getUploadDate() throws ParsingException { liveDetails.getString("startTimestamp", "")); // a running live stream if (!timestamp.isEmpty()) { - return new DateWrapper(OffsetDateTime.parse(timestamp)); + return timestamp; } else if (getStreamType() == StreamType.LIVE_STREAM) { // this should never be reached, but a live stream without upload date is valid return null; } final var textObject = getVideoPrimaryInfoRenderer().getObject("dateText"); - return Optional.ofNullable(getTextFromObject(textObject)) - .flatMap(rendererDateText -> { - final Optional dateOptional; + final String rendererDateText = getTextFromObject(textObject); + if (rendererDateText == null) { + return null; + } else if (rendererDateText.startsWith(PREMIERED_ON)) { // Premiered on 21 Feb 2020 + return rendererDateText.substring(PREMIERED_ON.length()); + } else if (rendererDateText.startsWith(PREMIERED)) { + // Premiered 20 hours ago / Premiered Feb 21, 2020 + return rendererDateText.substring(PREMIERED.length()); + } else { + return rendererDateText; + } + } - if (rendererDateText.startsWith("Premiered")) { - final String time = rendererDateText.substring(13); + @Override + public DateWrapper getUploadDate() throws ParsingException { + final String dateText = getTextualUploadDate(); + if (dateText == null) { + return null; + } - try { // Premiered 20 hours ago - final var localization = new Localization("en"); - return Optional.of(getTimeAgoParserFor(localization).parse(time)); - } catch (final Exception e) { - } + try { + return new DateWrapper(OffsetDateTime.parse(dateText)); + } catch (final DateTimeParseException e) { + } - // Premiered Feb 21, 2020 - dateOptional = parseOptionalDate(time, "MMM dd, yyyy") - // Premiered on 21 Feb 2020 - .or(() -> parseOptionalDate(time, "dd MMM yyyy")); - } else { - // Premiered on 21 Feb 2020 - dateOptional = parseOptionalDate(rendererDateText, "dd MMM yyyy"); - } + try { // Premiered 20 hours ago + final var localization = new Localization("en"); + return TimeAgoPatternsManager.getTimeAgoParserFor(localization).parse(dateText); + } catch (final ParsingException e) { + } - return dateOptional.map(date -> { - final var instant = date.atStartOfDay(ZoneId.systemDefault()).toInstant(); - return new DateWrapper(instant, true); - }); + return parseOptionalDate(dateText, "MMM dd, yyyy") + .or(() -> parseOptionalDate(dateText.substring(3), "dd MMM yyyy")) + .map(date -> { + final var instant = date.atStartOfDay(ZoneId.systemDefault()).toInstant(); + return new DateWrapper(instant, true); }) - .orElseThrow(() -> new ParsingException("Could not get upload date")); + .orElseThrow(() -> new ParsingException("Could not parse upload date")); } - private Optional parseOptionalDate(String date, String pattern) { + private Optional parseOptionalDate(final String date, final String pattern) { try { - // TODO: this parses English formatted dates only, we need a better approach to - // parse the textual date + // TODO: this parses English formatted dates only, we need a better approach to parse + // the textual date final var formatter = DateTimeFormatter.ofPattern(pattern, Locale.ENGLISH); return Optional.of(LocalDate.parse(date, formatter)); - } catch (DateTimeParseException e) { + } catch (final DateTimeParseException e) { return Optional.empty(); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java index f75a972189..0d4b6e5d78 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java @@ -246,7 +246,7 @@ public boolean isUploaderVerified() throws ParsingException { @Nullable @Override public String getTextualUploadDate() throws ParsingException { - if (getStreamType().equals(StreamType.LIVE_STREAM)) { + if (getStreamType() == StreamType.LIVE_STREAM) { return null; } @@ -274,7 +274,7 @@ public String getTextualUploadDate() throws ParsingException { @Nullable @Override public DateWrapper getUploadDate() throws ParsingException { - if (getStreamType().equals(StreamType.LIVE_STREAM)) { + if (getStreamType() == StreamType.LIVE_STREAM) { return null; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java index 1702917409..27f97e2cab 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java @@ -39,6 +39,7 @@ import javax.annotation.Nullable; import java.io.IOException; +import java.time.Instant; import java.util.Collections; import java.util.List; import java.util.Locale; @@ -71,7 +72,7 @@ public String getTextualUploadDate() throws ParsingException { } /** - * A more general {@code Calendar} instance set to the date provided by the service.
+ * A more general {@link Instant} instance set to the date provided by the service.
* Implementations usually will just parse the date returned from the {@link * #getTextualUploadDate()}. * diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ExtractorHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ExtractorHelper.java index 12023ca4c9..0619b52e5d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ExtractorHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ExtractorHelper.java @@ -5,16 +5,31 @@ import org.schabi.newpipe.extractor.InfoItemsCollector; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.StreamInfo; +import java.time.OffsetDateTime; +import java.time.format.DateTimeParseException; import java.util.Collections; import java.util.List; +import javax.annotation.Nonnull; + public final class ExtractorHelper { private ExtractorHelper() { } + @Nonnull + public static DateWrapper parseDateWrapper(@Nonnull final String date) throws ParsingException { + try { + return new DateWrapper(OffsetDateTime.parse(date)); + } catch (final DateTimeParseException e) { + throw new ParsingException("Could not parse date: \"" + date + "\"", e); + } + } + public static InfoItemsPage getItemsPageOrLogError( final Info info, final ListExtractor extractor) { try { From 380c30334d266ecf63b8506b8d12a9602340e5ba Mon Sep 17 00:00:00 2001 From: Isira Seneviratne Date: Fri, 12 Sep 2025 22:12:56 +0530 Subject: [PATCH 05/20] Refactor PeerTube date parsing method --- .../services/peertube/PeertubeParsingHelper.java | 6 ++++-- .../extractors/PeertubeCommentsInfoItemExtractor.java | 4 ++-- .../peertube/extractors/PeertubeStreamExtractor.java | 8 ++------ .../extractors/PeertubeStreamInfoItemExtractor.java | 9 ++------- 4 files changed, 10 insertions(+), 17 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java index 8f2c7e2761..2619db6db9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java @@ -10,6 +10,7 @@ import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeChannelInfoItemExtractor; import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubePlaylistInfoItemExtractor; import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeSepiaStreamInfoItemExtractor; @@ -46,9 +47,10 @@ public static void validate(final JsonObject json) throws ContentNotAvailableExc } } - public static Instant parseInstantFrom(final String textualUploadDate) throws ParsingException { + public static DateWrapper parseDateWrapper(final String textualUploadDate) + throws ParsingException { try { - return Instant.parse(textualUploadDate); + return new DateWrapper(Instant.parse(textualUploadDate)); } catch (final DateTimeParseException e) { throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java index f53149d538..4f590a883b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java @@ -23,7 +23,7 @@ import static org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeCommentsExtractor.CHILDREN; import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.getAvatarsFromOwnerAccountOrVideoChannelObject; -import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.parseInstantFrom; +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.parseDateWrapper; public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtractor { @Nonnull @@ -73,7 +73,7 @@ public String getTextualUploadDate() throws ParsingException { @Override public DateWrapper getUploadDate() throws ParsingException { - return new DateWrapper(parseInstantFrom(getTextualUploadDate())); + return parseDateWrapper(getTextualUploadDate()); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index c1deea6c6b..1c706ae86b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -2,6 +2,7 @@ import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.getAvatarsFromOwnerAccountOrVideoChannelObject; import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.getThumbnailsFromPlaylistOrVideoItem; +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.parseDateWrapper; import static org.schabi.newpipe.extractor.stream.AudioStream.UNKNOWN_BITRATE; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; @@ -80,12 +81,7 @@ public String getTextualUploadDate() throws ParsingException { @Override public DateWrapper getUploadDate() throws ParsingException { final String textualUploadDate = getTextualUploadDate(); - - if (textualUploadDate == null) { - return null; - } - - return new DateWrapper(PeertubeParsingHelper.parseInstantFrom(textualUploadDate)); + return textualUploadDate == null ? null : parseDateWrapper(textualUploadDate); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java index c7730a8d32..65ceab3a08 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java @@ -14,7 +14,7 @@ import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.getAvatarsFromOwnerAccountOrVideoChannelObject; import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.getThumbnailsFromPlaylistOrVideoItem; -import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.parseInstantFrom; +import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.parseDateWrapper; public class PeertubeStreamInfoItemExtractor implements StreamInfoItemExtractor { @@ -86,12 +86,7 @@ public String getTextualUploadDate() throws ParsingException { @Override public DateWrapper getUploadDate() throws ParsingException { final String textualUploadDate = getTextualUploadDate(); - - if (textualUploadDate == null) { - return null; - } - - return new DateWrapper(parseInstantFrom(textualUploadDate)); + return textualUploadDate == null ? null : parseDateWrapper(textualUploadDate); } @Override From e9dd84fe66a3c79a89ef042dbbf9e179e689c39e Mon Sep 17 00:00:00 2001 From: Isira Seneviratne Date: Sat, 13 Sep 2025 05:04:37 +0530 Subject: [PATCH 06/20] Rm substring call --- .../services/youtube/extractors/YoutubeStreamExtractor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 01a0883178..f977aa297c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -224,7 +224,7 @@ public DateWrapper getUploadDate() throws ParsingException { } return parseOptionalDate(dateText, "MMM dd, yyyy") - .or(() -> parseOptionalDate(dateText.substring(3), "dd MMM yyyy")) + .or(() -> parseOptionalDate(dateText, "dd MMM yyyy")) .map(date -> { final var instant = date.atStartOfDay(ZoneId.systemDefault()).toInstant(); return new DateWrapper(instant, true); From c79afc00764a5f2deb8f73e78244d93920cb3456 Mon Sep 17 00:00:00 2001 From: Isira Seneviratne Date: Sat, 13 Sep 2025 06:43:21 +0530 Subject: [PATCH 07/20] Apply code review suggestions --- .../extractors/YoutubeStreamExtractor.java | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index f977aa297c..43b7bed368 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -174,16 +174,20 @@ public String getName() throws ParsingException { @Nullable @Override public String getTextualUploadDate() throws ParsingException { - final String dateStr = playerMicroFormatRenderer.getString("uploadDate", - playerMicroFormatRenderer.getString("publishDate", "")); - if (!dateStr.isEmpty()) { - return dateStr; + String timestamp = playerMicroFormatRenderer.getString("uploadDate", ""); + if (timestamp.isEmpty()) { + timestamp = playerMicroFormatRenderer.getString("publishDate", ""); + } + if (!timestamp.isEmpty()) { + return timestamp; } final var liveDetails = playerMicroFormatRenderer.getObject("liveBroadcastDetails"); - final String timestamp = liveDetails.getString("endTimestamp", // an ended live stream - liveDetails.getString("startTimestamp", "")); // a running live stream - + timestamp = liveDetails.getString("endTimestamp", ""); // an ended live stream + if (timestamp.isEmpty()) { + // a running live stream + timestamp = liveDetails.getString("startTimestamp", ""); + } if (!timestamp.isEmpty()) { return timestamp; } else if (getStreamType() == StreamType.LIVE_STREAM) { @@ -229,7 +233,8 @@ public DateWrapper getUploadDate() throws ParsingException { final var instant = date.atStartOfDay(ZoneId.systemDefault()).toInstant(); return new DateWrapper(instant, true); }) - .orElseThrow(() -> new ParsingException("Could not parse upload date")); + .orElseThrow(() -> new ParsingException("Could not parse upload date \"" + + dateText + "\"")); } private Optional parseOptionalDate(final String date, final String pattern) { From f8d6f94e30ecfb7afec958d4591e41d690e8cb83 Mon Sep 17 00:00:00 2001 From: Isira Seneviratne Date: Tue, 16 Sep 2025 05:30:38 +0530 Subject: [PATCH 08/20] Do additional refactoring --- .../extractor/localization/DateWrapper.java | 18 +++++++ .../extractor/localization/TimeAgoParser.java | 53 ++++--------------- .../localization/TimeAgoPatternsManager.java | 22 ++------ .../extractors/MediaCCCRecentKiosk.java | 8 ++- .../extractors/YoutubeStreamExtractor.java | 12 ++--- .../YoutubeChartsBaseKioskExtractor.java | 2 +- .../localization/TimeAgoParserTest.java | 28 +++++----- 7 files changed, 55 insertions(+), 88 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/localization/DateWrapper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/localization/DateWrapper.java index b26c3b84fa..6eaa993381 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/localization/DateWrapper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/localization/DateWrapper.java @@ -3,7 +3,9 @@ import javax.annotation.Nonnull; import java.io.Serializable; import java.time.Instant; +import java.time.LocalDate; import java.time.OffsetDateTime; +import java.time.ZoneId; import java.time.ZoneOffset; /** @@ -33,6 +35,14 @@ public DateWrapper(@Nonnull final Instant instant, final boolean isApproximation this.isApproximation = isApproximation; } + public DateWrapper(@Nonnull final LocalDate localDate) { + this(localDate, true); + } + + public DateWrapper(@Nonnull final LocalDate localDate, final boolean isApproximation) { + this(localDate.atStartOfDay(ZoneId.systemDefault()).toInstant(), isApproximation); + } + /** * @return the wrapped {@link Instant} */ @@ -49,6 +59,14 @@ public OffsetDateTime offsetDateTime() { return instant.atOffset(ZoneOffset.UTC); } + /** + * @return the wrapped {@link Instant} as a {@link LocalDate} in the current time zone. + */ + @Nonnull + public LocalDate getLocalDate() { + return LocalDate.ofInstant(instant, ZoneId.systemDefault()); + } + /** * @return if the date is considered is precise or just an approximation (e.g. service only * returns an approximation like 2 weeks ago instead of a precise date). diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/localization/TimeAgoParser.java b/extractor/src/main/java/org/schabi/newpipe/extractor/localization/TimeAgoParser.java index 0eb06e1032..a775d4b9f3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/localization/TimeAgoParser.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/localization/TimeAgoParser.java @@ -4,7 +4,8 @@ import org.schabi.newpipe.extractor.timeago.PatternsHolder; import org.schabi.newpipe.extractor.utils.Parser; -import java.time.OffsetDateTime; +import java.time.LocalDateTime; +import java.time.ZoneId; import java.time.temporal.ChronoUnit; import java.util.Map; import java.util.regex.Pattern; @@ -15,20 +16,7 @@ */ public class TimeAgoParser { private final PatternsHolder patternsHolder; - private final OffsetDateTime now; - - /** - * Creates a helper to parse upload dates in the format '2 days ago'. - *

- * Instantiate a new {@link TimeAgoParser} every time you extract a new batch of items. - *

- * - * @param patternsHolder An object that holds the "time ago" patterns, special cases, and the - * language word separator. - */ - public TimeAgoParser(final PatternsHolder patternsHolder) { - this(patternsHolder, OffsetDateTime.now()); - } + private final LocalDateTime now; /** * Creates a helper to parse upload dates in the format '2 days ago'. @@ -40,7 +28,7 @@ public TimeAgoParser(final PatternsHolder patternsHolder) { * language word separator. * @param now The current time */ - public TimeAgoParser(final PatternsHolder patternsHolder, final OffsetDateTime now) { + public TimeAgoParser(final PatternsHolder patternsHolder, final LocalDateTime now) { this.patternsHolder = patternsHolder; this.now = now; } @@ -117,34 +105,15 @@ private boolean textualDateMatches(final String textualDate, final String agoPhr } private DateWrapper getResultFor(final int timeAgoAmount, final ChronoUnit chronoUnit) { - OffsetDateTime offsetDateTime = now; - boolean isApproximation = false; - - switch (chronoUnit) { - case SECONDS: - case MINUTES: - case HOURS: - offsetDateTime = offsetDateTime.minus(timeAgoAmount, chronoUnit); - break; - - case DAYS: - case WEEKS: - case MONTHS: - offsetDateTime = offsetDateTime.minus(timeAgoAmount, chronoUnit); - isApproximation = true; - break; - - case YEARS: + final var resolvedDateTime = chronoUnit == ChronoUnit.YEARS // minusDays is needed to prevent `PrettyTime` from showing '12 months ago'. - offsetDateTime = offsetDateTime.minusYears(timeAgoAmount).minusDays(1); - isApproximation = true; - break; - } + ? now.minusYears(timeAgoAmount).minusDays(1) + : now.minus(timeAgoAmount, chronoUnit); - if (isApproximation) { - offsetDateTime = offsetDateTime.truncatedTo(ChronoUnit.HOURS); + if (chronoUnit.isDateBased()) { + return new DateWrapper(resolvedDateTime.toLocalDate()); + } else { + return new DateWrapper(resolvedDateTime.atZone(ZoneId.systemDefault()).toInstant()); } - - return new DateWrapper(offsetDateTime, isApproximation); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/localization/TimeAgoPatternsManager.java b/extractor/src/main/java/org/schabi/newpipe/extractor/localization/TimeAgoPatternsManager.java index 47889a5d32..67cfb93a64 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/localization/TimeAgoPatternsManager.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/localization/TimeAgoPatternsManager.java @@ -3,7 +3,7 @@ import org.schabi.newpipe.extractor.timeago.PatternsHolder; import org.schabi.newpipe.extractor.timeago.PatternsManager; -import java.time.OffsetDateTime; +import java.time.LocalDateTime; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -20,25 +20,13 @@ private static PatternsHolder getPatternsFor(@Nonnull final Localization localiz @Nullable public static TimeAgoParser getTimeAgoParserFor(@Nonnull final Localization localization) { - final PatternsHolder holder = getPatternsFor(localization); - - if (holder == null) { - return null; - } - - return new TimeAgoParser(holder); + return getTimeAgoParserFor(localization, LocalDateTime.now()); } @Nullable - public static TimeAgoParser getTimeAgoParserFor( - @Nonnull final Localization localization, - @Nonnull final OffsetDateTime now) { + public static TimeAgoParser getTimeAgoParserFor(@Nonnull final Localization localization, + @Nonnull final LocalDateTime now) { final PatternsHolder holder = getPatternsFor(localization); - - if (holder == null) { - return null; - } - - return new TimeAgoParser(holder, now); + return holder == null ? null : new TimeAgoParser(holder, now); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCRecentKiosk.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCRecentKiosk.java index 33583bc644..6223304597 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCRecentKiosk.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCRecentKiosk.java @@ -52,12 +52,10 @@ public InfoItemsPage getInitialPage() throws IOException, Extrac // Streams in the recent kiosk are not ordered by the release date. // Sort them to have the latest stream at the beginning of the list. - final Comparator comparator = Comparator - .comparing(StreamInfoItem::getUploadDate, Comparator - .nullsLast(Comparator.comparing(DateWrapper::getInstant))) + final var comparator = Comparator.comparing(StreamInfoItem::getUploadDate, + Comparator.nullsLast(Comparator.comparing(DateWrapper::getInstant))) .reversed(); - final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId(), - comparator); + final var collector = new StreamInfoItemsCollector(getServiceId(), comparator); events.stream() .filter(JsonObject.class::isInstance) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 43b7bed368..feba0e2d86 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -88,7 +88,6 @@ import java.nio.charset.StandardCharsets; import java.time.LocalDate; import java.time.OffsetDateTime; -import java.time.ZoneId; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; import java.util.ArrayList; @@ -219,22 +218,21 @@ public DateWrapper getUploadDate() throws ParsingException { try { return new DateWrapper(OffsetDateTime.parse(dateText)); } catch (final DateTimeParseException e) { + // Try other patterns first } try { // Premiered 20 hours ago final var localization = new Localization("en"); return TimeAgoPatternsManager.getTimeAgoParserFor(localization).parse(dateText); } catch (final ParsingException e) { + // Try other patterns first } return parseOptionalDate(dateText, "MMM dd, yyyy") .or(() -> parseOptionalDate(dateText, "dd MMM yyyy")) - .map(date -> { - final var instant = date.atStartOfDay(ZoneId.systemDefault()).toInstant(); - return new DateWrapper(instant, true); - }) - .orElseThrow(() -> new ParsingException("Could not parse upload date \"" - + dateText + "\"")); + .map(DateWrapper::new) + .orElseThrow(() -> + new ParsingException("Could not parse upload date \"" + dateText + "\"")); } private Optional parseOptionalDate(final String date, final String pattern) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/kiosk/YoutubeChartsBaseKioskExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/kiosk/YoutubeChartsBaseKioskExtractor.java index 329fe8d261..f96c603875 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/kiosk/YoutubeChartsBaseKioskExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/kiosk/YoutubeChartsBaseKioskExtractor.java @@ -203,7 +203,7 @@ public String getTextualUploadDate() { @Nonnull @Override public DateWrapper getUploadDate() { - final JsonObject releaseDate = videoObject.getObject("releaseDate"); + final var releaseDate = videoObject.getObject("releaseDate"); final var localDate = LocalDate.of(releaseDate.getInt("year"), releaseDate.getInt("month"), releaseDate.getInt("day")); // We request that times should be returned with 0 offset to UTC timezone in diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/localization/TimeAgoParserTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/localization/TimeAgoParserTest.java index db45e807b9..6394297722 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/localization/TimeAgoParserTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/localization/TimeAgoParserTest.java @@ -11,10 +11,9 @@ import java.time.Duration; import java.time.LocalDateTime; -import java.time.OffsetDateTime; -import java.time.ZoneOffset; +import java.time.Month; +import java.time.ZoneId; import java.time.temporal.ChronoUnit; -import java.util.Objects; import java.util.function.Function; import java.util.stream.Stream; @@ -41,13 +40,10 @@ public static Stream parseTimeAgo() { @ParameterizedTest @MethodSource void parseTimeAgo(final ParseTimeAgoTestData testData) { - final OffsetDateTime now = OffsetDateTime.of( - LocalDateTime.of(2020, 1, 1, 1, 1, 1), - ZoneOffset.UTC); - final TimeAgoParser parser = Objects.requireNonNull( - TimeAgoPatternsManager.getTimeAgoParserFor(Localization.DEFAULT, now)); - - final OffsetDateTime expected = testData.getExpectedApplyToNow().apply(now); + final var now = LocalDateTime.of(2020, Month.JANUARY, 1, 1, 1, 1); + final var parser = TimeAgoPatternsManager.getTimeAgoParserFor(Localization.DEFAULT, now); + final var expected = testData.getExpectedApplyToNow().apply(now); + final var zoneId = ZoneId.systemDefault(); assertAll( Stream.of( @@ -55,7 +51,7 @@ void parseTimeAgo(final ParseTimeAgoTestData testData) { testData.getTextualDateShort()) .map(textualDate -> () -> assertEquals( expected, - parser.parse(textualDate).offsetDateTime(), + LocalDateTime.ofInstant(parser.parse(textualDate).getInstant(), zoneId), "Expected " + expected + " for " + textualDate )) ); @@ -63,12 +59,12 @@ void parseTimeAgo(final ParseTimeAgoTestData testData) { static class ParseTimeAgoTestData { public static final String AGO_SUFFIX = " ago"; - private final Function expectedApplyToNow; + private final Function expectedApplyToNow; private final String textualDateLong; private final String textualDateShort; ParseTimeAgoTestData( - final Function expectedApplyToNow, + final Function expectedApplyToNow, final String textualDateLong, final String textualDateShort ) { @@ -89,17 +85,17 @@ public static ParseTimeAgoTestData lessThanDay( } public static ParseTimeAgoTestData greaterThanDay( - final Function expectedApplyToNow, + final Function expectedApplyToNow, final String textualDateLong, final String textualDateShort ) { return new ParseTimeAgoTestData( - d -> expectedApplyToNow.apply(d).truncatedTo(ChronoUnit.HOURS), + expectedApplyToNow.andThen(d -> d.truncatedTo(ChronoUnit.DAYS)), textualDateLong + AGO_SUFFIX, textualDateShort + AGO_SUFFIX); } - public Function getExpectedApplyToNow() { + public Function getExpectedApplyToNow() { return expectedApplyToNow; } From 8727659d2d4ccad9cd370d05b90dc77fa43f2a4c Mon Sep 17 00:00:00 2001 From: Isira Seneviratne Date: Tue, 16 Sep 2025 08:07:54 +0530 Subject: [PATCH 09/20] Use LocalDateTime instead of LocalDate --- .../newpipe/extractor/localization/DateWrapper.java | 13 +++++++------ .../extractor/localization/TimeAgoParser.java | 13 +++++-------- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/localization/DateWrapper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/localization/DateWrapper.java index 6eaa993381..4e106f536f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/localization/DateWrapper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/localization/DateWrapper.java @@ -4,6 +4,7 @@ import java.io.Serializable; import java.time.Instant; import java.time.LocalDate; +import java.time.LocalDateTime; import java.time.OffsetDateTime; import java.time.ZoneId; import java.time.ZoneOffset; @@ -35,12 +36,12 @@ public DateWrapper(@Nonnull final Instant instant, final boolean isApproximation this.isApproximation = isApproximation; } - public DateWrapper(@Nonnull final LocalDate localDate) { - this(localDate, true); + public DateWrapper(@Nonnull final LocalDateTime dateTime) { + this(dateTime, false); } - public DateWrapper(@Nonnull final LocalDate localDate, final boolean isApproximation) { - this(localDate.atStartOfDay(ZoneId.systemDefault()).toInstant(), isApproximation); + public DateWrapper(@Nonnull final LocalDateTime dateTime, final boolean isApproximation) { + this(dateTime.atZone(ZoneId.systemDefault()).toInstant(), isApproximation); } /** @@ -63,8 +64,8 @@ public OffsetDateTime offsetDateTime() { * @return the wrapped {@link Instant} as a {@link LocalDate} in the current time zone. */ @Nonnull - public LocalDate getLocalDate() { - return LocalDate.ofInstant(instant, ZoneId.systemDefault()); + public LocalDateTime getLocalDateTime() { + return LocalDateTime.ofInstant(instant, ZoneId.systemDefault()); } /** diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/localization/TimeAgoParser.java b/extractor/src/main/java/org/schabi/newpipe/extractor/localization/TimeAgoParser.java index a775d4b9f3..88e04b803c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/localization/TimeAgoParser.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/localization/TimeAgoParser.java @@ -5,7 +5,6 @@ import org.schabi.newpipe.extractor.utils.Parser; import java.time.LocalDateTime; -import java.time.ZoneId; import java.time.temporal.ChronoUnit; import java.util.Map; import java.util.regex.Pattern; @@ -105,15 +104,13 @@ private boolean textualDateMatches(final String textualDate, final String agoPhr } private DateWrapper getResultFor(final int timeAgoAmount, final ChronoUnit chronoUnit) { - final var resolvedDateTime = chronoUnit == ChronoUnit.YEARS + final var localDateTime = chronoUnit == ChronoUnit.YEARS // minusDays is needed to prevent `PrettyTime` from showing '12 months ago'. ? now.minusYears(timeAgoAmount).minusDays(1) : now.minus(timeAgoAmount, chronoUnit); - - if (chronoUnit.isDateBased()) { - return new DateWrapper(resolvedDateTime.toLocalDate()); - } else { - return new DateWrapper(resolvedDateTime.atZone(ZoneId.systemDefault()).toInstant()); - } + final boolean isApproximate = chronoUnit.isDateBased(); + final var resolvedDateTime = + isApproximate ? localDateTime.truncatedTo(ChronoUnit.DAYS) : localDateTime; + return new DateWrapper(resolvedDateTime, isApproximate); } } From f9f71cbbe675e866ee5e39387372f4cd4f67101f Mon Sep 17 00:00:00 2001 From: Isira Seneviratne Date: Wed, 17 Sep 2025 07:11:31 +0530 Subject: [PATCH 10/20] Fix some issues --- .../newpipe/extractor/localization/DateWrapper.java | 4 ---- .../youtube/extractors/YoutubeStreamExtractor.java | 2 +- .../extractor/localization/TimeAgoParserTest.java | 4 +--- .../services/DefaultStreamExtractorTest.java | 13 +++++-------- .../bandcamp/BandcampRadioStreamExtractorTest.java | 4 +--- .../youtube/YoutubeChannelLocalizationTest.java | 9 ++------- 6 files changed, 10 insertions(+), 26 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/localization/DateWrapper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/localization/DateWrapper.java index 4e106f536f..5d559b9ed2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/localization/DateWrapper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/localization/DateWrapper.java @@ -36,10 +36,6 @@ public DateWrapper(@Nonnull final Instant instant, final boolean isApproximation this.isApproximation = isApproximation; } - public DateWrapper(@Nonnull final LocalDateTime dateTime) { - this(dateTime, false); - } - public DateWrapper(@Nonnull final LocalDateTime dateTime, final boolean isApproximation) { this(dateTime.atZone(ZoneId.systemDefault()).toInstant(), isApproximation); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index feba0e2d86..373edf9a71 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -230,7 +230,7 @@ public DateWrapper getUploadDate() throws ParsingException { return parseOptionalDate(dateText, "MMM dd, yyyy") .or(() -> parseOptionalDate(dateText, "dd MMM yyyy")) - .map(DateWrapper::new) + .map(date -> new DateWrapper(date.atStartOfDay(), true)) .orElseThrow(() -> new ParsingException("Could not parse upload date \"" + dateText + "\"")); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/localization/TimeAgoParserTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/localization/TimeAgoParserTest.java index 6394297722..451b66ac35 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/localization/TimeAgoParserTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/localization/TimeAgoParserTest.java @@ -12,7 +12,6 @@ import java.time.Duration; import java.time.LocalDateTime; import java.time.Month; -import java.time.ZoneId; import java.time.temporal.ChronoUnit; import java.util.function.Function; import java.util.stream.Stream; @@ -43,7 +42,6 @@ void parseTimeAgo(final ParseTimeAgoTestData testData) { final var now = LocalDateTime.of(2020, Month.JANUARY, 1, 1, 1, 1); final var parser = TimeAgoPatternsManager.getTimeAgoParserFor(Localization.DEFAULT, now); final var expected = testData.getExpectedApplyToNow().apply(now); - final var zoneId = ZoneId.systemDefault(); assertAll( Stream.of( @@ -51,7 +49,7 @@ void parseTimeAgo(final ParseTimeAgoTestData testData) { testData.getTextualDateShort()) .map(textualDate -> () -> assertEquals( expected, - LocalDateTime.ofInstant(parser.parse(textualDate).getInstant(), zoneId), + parser.parse(textualDate).getLocalDateTime(), "Expected " + expected + " for " + textualDate )) ); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java index c868b88547..51d3fb306e 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java @@ -5,7 +5,6 @@ import org.schabi.newpipe.extractor.InfoItemsCollector; import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.MetaInfo; -import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.stream.AudioStream; import org.schabi.newpipe.extractor.stream.Description; import org.schabi.newpipe.extractor.stream.Frameset; @@ -16,7 +15,6 @@ import javax.annotation.Nullable; import java.time.LocalDateTime; -import java.time.ZoneId; import java.time.format.DateTimeFormatter; import java.net.MalformedURLException; import java.net.URL; @@ -194,19 +192,18 @@ public void testViewCount() throws Exception { @Test @Override public void testUploadDate() throws Exception { - final DateWrapper dateWrapper = extractor().getUploadDate(); + final var dateWrapper = extractor().getUploadDate(); + final var expectedDate = expectedUploadDate(); - if (expectedUploadDate() == null) { + if (expectedDate == null) { assertNull(dateWrapper); } else { assertNotNull(dateWrapper); - final var expectedDateTime = LocalDateTime.parse(expectedUploadDate(), + final var expectedDateTime = LocalDateTime.parse(expectedDate, DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS")); - final var actualDateTime = LocalDateTime.ofInstant(dateWrapper.getInstant(), - ZoneId.systemDefault()); - assertEquals(expectedDateTime, actualDateTime); + assertEquals(expectedDateTime, dateWrapper.getLocalDateTime()); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java index 3d16145aa1..3ff6b4371c 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java @@ -21,7 +21,6 @@ import java.io.IOException; import java.time.LocalDate; import java.time.Month; -import java.time.ZoneOffset; import java.util.Collections; import java.util.List; @@ -84,8 +83,7 @@ public List expectedDescriptionContains() { @Test public void testUploadDate() throws ParsingException { final var expectedDate = LocalDate.of(2017, Month.MAY, 16); - final var actualDate = LocalDate.ofInstant(extractor().getUploadDate().getInstant(), - ZoneOffset.UTC); + final var actualDate = extractor().getUploadDate().getLocalDateTime().toLocalDate(); assertEquals(expectedDate, actualDate); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLocalizationTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLocalizationTest.java index b00914fd14..cae661f8e1 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLocalizationTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLocalizationTest.java @@ -13,8 +13,6 @@ import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.stream.StreamInfoItem; -import java.time.LocalDateTime; -import java.time.ZoneId; import java.time.format.DateTimeFormatter; import java.time.temporal.ChronoUnit; import java.util.LinkedHashMap; @@ -75,11 +73,8 @@ private void testLocalizationsFor(final String channelUrl) throws Exception { + "\n:::: " + item.getStreamType() + ", views = " + item.getViewCount(); final DateWrapper uploadDate = item.getUploadDate(); if (uploadDate != null) { - final var dateTime = LocalDateTime.ofInstant(uploadDate.getInstant(), - ZoneId.systemDefault()); - final String dateAsText = dateTimeFormatter.format(dateTime); - debugMessage += "\n:::: " + item.getTextualUploadDate() + - "\n:::: " + dateAsText; + final String dateStr = dateTimeFormatter.format(uploadDate.getLocalDateTime()); + debugMessage += "\n:::: " + item.getTextualUploadDate() + "\n:::: " + dateStr; } if (DEBUG) System.out.println(debugMessage + "\n"); } From 59e78c2ba45694618932bf0be4cccdb492d15200 Mon Sep 17 00:00:00 2001 From: Isira Seneviratne Date: Thu, 18 Sep 2025 05:49:20 +0530 Subject: [PATCH 11/20] Fix tests --- .../newpipe/extractor/localization/DateWrapper.java | 13 ++++++++++--- .../services/DefaultStreamExtractorTest.java | 3 ++- .../bandcamp/BandcampRadioStreamExtractorTest.java | 4 +++- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/localization/DateWrapper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/localization/DateWrapper.java index 5d559b9ed2..2d71e520b9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/localization/DateWrapper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/localization/DateWrapper.java @@ -3,7 +3,6 @@ import javax.annotation.Nonnull; import java.io.Serializable; import java.time.Instant; -import java.time.LocalDate; import java.time.LocalDateTime; import java.time.OffsetDateTime; import java.time.ZoneId; @@ -57,11 +56,19 @@ public OffsetDateTime offsetDateTime() { } /** - * @return the wrapped {@link Instant} as a {@link LocalDate} in the current time zone. + * @return the wrapped {@link Instant} as a {@link LocalDateTime} in the current time zone. */ @Nonnull public LocalDateTime getLocalDateTime() { - return LocalDateTime.ofInstant(instant, ZoneId.systemDefault()); + return getLocalDateTime(ZoneId.systemDefault()); + } + + /** + * @return the wrapped {@link Instant} as a {@link LocalDateTime} in the given time zone. + */ + @Nonnull + public LocalDateTime getLocalDateTime(@Nonnull final ZoneId zoneId) { + return LocalDateTime.ofInstant(instant, zoneId); } /** diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java index 51d3fb306e..ef137a307f 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java @@ -15,6 +15,7 @@ import javax.annotation.Nullable; import java.time.LocalDateTime; +import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import java.net.MalformedURLException; import java.net.URL; @@ -203,7 +204,7 @@ public void testUploadDate() throws Exception { final var expectedDateTime = LocalDateTime.parse(expectedDate, DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS")); - assertEquals(expectedDateTime, dateWrapper.getLocalDateTime()); + assertEquals(expectedDateTime, dateWrapper.getLocalDateTime(ZoneOffset.UTC)); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java index 3ff6b4371c..378da8053c 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampRadioStreamExtractorTest.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.time.LocalDate; import java.time.Month; +import java.time.ZoneOffset; import java.util.Collections; import java.util.List; @@ -83,7 +84,8 @@ public List expectedDescriptionContains() { @Test public void testUploadDate() throws ParsingException { final var expectedDate = LocalDate.of(2017, Month.MAY, 16); - final var actualDate = extractor().getUploadDate().getLocalDateTime().toLocalDate(); + final var actualDate = extractor().getUploadDate().getLocalDateTime(ZoneOffset.UTC) + .toLocalDate(); assertEquals(expectedDate, actualDate); } From f38b72ce7d672f8f5d7bd23384440a566b879fab Mon Sep 17 00:00:00 2001 From: Isira Seneviratne Date: Thu, 18 Sep 2025 06:04:54 +0530 Subject: [PATCH 12/20] Avoid potential NPEs in Soundcloud extractors --- .../soundcloud/SoundcloudParsingHelper.java | 22 +++++++++---------- .../SoundcloudCommentsInfoItemExtractor.java | 2 +- .../extractors/SoundcloudStreamExtractor.java | 10 ++++----- .../SoundcloudStreamInfoItemExtractor.java | 2 +- .../SoundcloudStreamExtractorTest.java | 6 ++--- 5 files changed, 19 insertions(+), 23 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index ae8fd77d6d..07e359e581 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -23,6 +23,7 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; +import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudChannelInfoItemExtractor; import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudPlaylistInfoItemExtractor; import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudLikesInfoItemExtractor; @@ -39,8 +40,7 @@ import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; -import java.time.OffsetDateTime; -import java.time.format.DateTimeFormatter; +import java.time.Instant; import java.time.format.DateTimeParseException; import java.util.Collections; import java.util.List; @@ -133,18 +133,16 @@ public static synchronized String clientId() throws ExtractionException, IOExcep throw new ExtractionException("Couldn't extract client id"); } - public static OffsetDateTime parseDateFrom(final String textualUploadDate) + @Nullable + public static DateWrapper parseDateFrom(@Nullable final String uploadDate) throws ParsingException { + if (uploadDate == null) { + return null; + } try { - return OffsetDateTime.parse(textualUploadDate); - } catch (final DateTimeParseException e1) { - try { - return OffsetDateTime.parse(textualUploadDate, DateTimeFormatter - .ofPattern("yyyy/MM/dd HH:mm:ss +0000")); - } catch (final DateTimeParseException e2) { - throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"" - + ", " + e1.getMessage(), e2); - } + return new DateWrapper(Instant.parse(uploadDate)); + } catch (final DateTimeParseException e) { + throw new ParsingException("Could not parse date: \"" + uploadDate + "\"", e); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java index 5ff6d29db6..d2bc46b95e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java @@ -69,7 +69,7 @@ public String getTextualUploadDate() { @Nullable @Override public DateWrapper getUploadDate() throws ParsingException { - return new DateWrapper(parseDateFrom(getTextualUploadDate())); + return parseDateFrom(getTextualUploadDate()); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java index 67cd533e9c..486b3c9938 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java @@ -88,18 +88,16 @@ public String getName() { return track.getString("title"); } - @Nonnull + @Nullable @Override public String getTextualUploadDate() { - return track.getString("created_at") - .replace("T", " ") - .replace("Z", ""); + return track.getString("created_at"); } - @Nonnull + @Nullable @Override public DateWrapper getUploadDate() throws ParsingException { - return new DateWrapper(parseDateFrom(track.getString("created_at"))); + return parseDateFrom(getTextualUploadDate()); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamInfoItemExtractor.java index 6fd6232e97..ea6bbe2f8e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamInfoItemExtractor.java @@ -68,7 +68,7 @@ public String getTextualUploadDate() { @Override public DateWrapper getUploadDate() throws ParsingException { - return new DateWrapper(parseDateFrom(getTextualUploadDate())); + return parseDateFrom(getTextualUploadDate()); } @Override diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java index d33ea37443..1cd566c55a 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java @@ -63,7 +63,7 @@ protected void fetchExtractor(final StreamExtractor extractor) throws Exception @Override public long expectedTimestamp() { return TIMESTAMP; } @Override public long expectedViewCountAtLeast() { return 43000; } @Nullable @Override public String expectedUploadDate() { return "2019-05-16 16:28:45.000"; } - @Nullable @Override public String expectedTextualUploadDate() { return "2019-05-16 16:28:45"; } + @Nullable @Override public String expectedTextualUploadDate() { return "2019-05-16T16:28:45Z"; } @Override public long expectedLikeCountAtLeast() { return 600; } @Override public long expectedDislikeCountAtLeast() { return -1; } @Override public boolean expectedHasAudioStreams() { return false; } @@ -127,7 +127,7 @@ public void testRelatedItems() throws Exception { @Override public long expectedTimestamp() { return TIMESTAMP; } @Override public long expectedViewCountAtLeast() { return 386000; } @Nullable @Override public String expectedUploadDate() { return "2016-11-11 01:16:37.000"; } - @Nullable @Override public String expectedTextualUploadDate() { return "2016-11-11 01:16:37"; } + @Nullable @Override public String expectedTextualUploadDate() { return "2016-11-11T01:16:37Z"; } @Override public long expectedLikeCountAtLeast() { return 7350; } @Override public long expectedDislikeCountAtLeast() { return -1; } @Override public boolean expectedHasAudioStreams() { return false; } @@ -168,7 +168,7 @@ protected StreamExtractor createExtractor() throws Exception { @Override public long expectedTimestamp() { return TIMESTAMP; } @Override public long expectedViewCountAtLeast() { return 15000; } @Nullable @Override public String expectedUploadDate() { return "2022-10-03 18:49:49.000"; } - @Nullable @Override public String expectedTextualUploadDate() { return "2022-10-03 18:49:49"; } + @Nullable @Override public String expectedTextualUploadDate() { return "2022-10-03T18:49:49Z"; } @Override public long expectedLikeCountAtLeast() { return 10; } @Override public long expectedDislikeCountAtLeast() { return -1; } @Override public boolean expectedHasRelatedItems() { return false; } From 1a5ac2fa371f0fbc2f84f581ba1d9fe88964d0a2 Mon Sep 17 00:00:00 2001 From: Isira Seneviratne Date: Thu, 18 Sep 2025 06:26:32 +0530 Subject: [PATCH 13/20] Fix nested tests --- .../soundcloud/SoundcloudStreamExtractorTest.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java index 1cd566c55a..99053c7c52 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java @@ -6,6 +6,7 @@ import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.schabi.newpipe.extractor.ExtractorAsserts; import org.schabi.newpipe.extractor.MediaFormat; @@ -27,7 +28,8 @@ public class SoundcloudStreamExtractorTest { private static final String SOUNDCLOUD = "https://soundcloud.com/"; - public static class SoundcloudGeoRestrictedTrack extends DefaultStreamExtractorTest { + @Nested + class SoundcloudGeoRestrictedTrack extends DefaultStreamExtractorTest { private static final String ID = "one-touch"; private static final String UPLOADER = SOUNDCLOUD + "jessglynne"; private static final int TIMESTAMP = 0; @@ -82,7 +84,8 @@ public void testRelatedItems() throws Exception { } } - public static class SoundcloudGoPlusTrack extends DefaultStreamExtractorTest { + @Nested + class SoundcloudGoPlusTrack extends DefaultStreamExtractorTest { private static final String ID = "places"; private static final String UPLOADER = SOUNDCLOUD + "martinsolveig"; private static final int TIMESTAMP = 0; @@ -140,7 +143,8 @@ public void testRelatedItems() throws Exception { @Override public String expectedCategory() { return "Dance"; } } - static class CreativeCommonsOpenMindsEp21 extends DefaultStreamExtractorTest { + @Nested + class CreativeCommonsOpenMindsEp21 extends DefaultStreamExtractorTest { private static final String ID = "open-minds-ep-21-dr-beth-harris-and-dr-steven-zucker-of-smarthistory"; private static final String UPLOADER = SOUNDCLOUD + "wearecc"; private static final int TIMESTAMP = 69; From 89971878a278b6b30b5cd2d760c84be3be033036 Mon Sep 17 00:00:00 2001 From: Isira Seneviratne Date: Wed, 8 Oct 2025 06:51:17 +0530 Subject: [PATCH 14/20] Address code review comments --- .../extractor/localization/DateWrapper.java | 25 +++++++++++++++++++ .../extractors/MediaCCCStreamExtractor.java | 2 +- .../MediaCCCStreamInfoItemExtractor.java | 3 +-- .../peertube/PeertubeParsingHelper.java | 12 --------- .../PeertubeCommentsInfoItemExtractor.java | 3 +-- .../extractors/PeertubeStreamExtractor.java | 4 +-- .../PeertubeStreamInfoItemExtractor.java | 4 +-- .../soundcloud/SoundcloudParsingHelper.java | 19 ++++++++------ .../SoundcloudCommentsInfoItemExtractor.java | 4 +-- .../extractors/SoundcloudStreamExtractor.java | 4 +-- .../SoundcloudStreamInfoItemExtractor.java | 4 +-- .../YoutubeFeedInfoItemExtractor.java | 3 +-- .../extractor/utils/ExtractorHelper.java | 15 ----------- 13 files changed, 48 insertions(+), 54 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/localization/DateWrapper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/localization/DateWrapper.java index 2d71e520b9..2884094084 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/localization/DateWrapper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/localization/DateWrapper.java @@ -1,5 +1,7 @@ package org.schabi.newpipe.extractor.localization; +import org.schabi.newpipe.extractor.exceptions.ParsingException; + import javax.annotation.Nonnull; import java.io.Serializable; import java.time.Instant; @@ -7,6 +9,7 @@ import java.time.OffsetDateTime; import java.time.ZoneId; import java.time.ZoneOffset; +import java.time.format.DateTimeParseException; /** * A wrapper class that provides a field to describe if the date/time is precise or just an @@ -78,4 +81,26 @@ public LocalDateTime getLocalDateTime(@Nonnull final ZoneId zoneId) { public boolean isApproximation() { return isApproximation; } + + public static DateWrapper fromOffsetDateTime(final String date) throws ParsingException { + if (date == null) { + return null; + } + try { + return new DateWrapper(OffsetDateTime.parse(date)); + } catch (final DateTimeParseException e) { + throw new ParsingException("Could not parse date: \"" + date + "\"", e); + } + } + + public static DateWrapper fromInstant(final String date) throws ParsingException { + if (date == null) { + return null; + } + try { + return new DateWrapper(Instant.parse(date)); + } catch (final DateTimeParseException e) { + throw new ParsingException("Could not parse date: \"" + date + "\"", e); + } + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java index 23ebf44330..f50340abf3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java @@ -55,7 +55,7 @@ public String getTextualUploadDate() { @Nonnull @Override public DateWrapper getUploadDate() throws ParsingException { - return ExtractorHelper.parseDateWrapper(getTextualUploadDate()); + return DateWrapper.fromOffsetDateTime(getTextualUploadDate()); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCStreamInfoItemExtractor.java index 82687c3285..797f923bb2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCStreamInfoItemExtractor.java @@ -65,9 +65,8 @@ public String getTextualUploadDate() { @Nullable @Override public DateWrapper getUploadDate() throws ParsingException { - final String date = getTextualUploadDate(); // if null, event is in the future... - return date == null ? null : ExtractorHelper.parseDateWrapper(date); + return DateWrapper.fromOffsetDateTime(getTextualUploadDate()); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java index 2619db6db9..3d87ad88cb 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java @@ -10,7 +10,6 @@ import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeChannelInfoItemExtractor; import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubePlaylistInfoItemExtractor; import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeSepiaStreamInfoItemExtractor; @@ -19,8 +18,6 @@ import org.schabi.newpipe.extractor.utils.Parser; import javax.annotation.Nonnull; -import java.time.Instant; -import java.time.format.DateTimeParseException; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -47,15 +44,6 @@ public static void validate(final JsonObject json) throws ContentNotAvailableExc } } - public static DateWrapper parseDateWrapper(final String textualUploadDate) - throws ParsingException { - try { - return new DateWrapper(Instant.parse(textualUploadDate)); - } catch (final DateTimeParseException e) { - throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e); - } - } - public static Page getNextPage(final String prevPageUrl, final long total) { final String prevStart; try { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java index 4f590a883b..80c8bbbc70 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java @@ -23,7 +23,6 @@ import static org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeCommentsExtractor.CHILDREN; import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.getAvatarsFromOwnerAccountOrVideoChannelObject; -import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.parseDateWrapper; public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtractor { @Nonnull @@ -73,7 +72,7 @@ public String getTextualUploadDate() throws ParsingException { @Override public DateWrapper getUploadDate() throws ParsingException { - return parseDateWrapper(getTextualUploadDate()); + return DateWrapper.fromInstant(getTextualUploadDate()); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index 1c706ae86b..bf0e02b593 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -2,7 +2,6 @@ import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.getAvatarsFromOwnerAccountOrVideoChannelObject; import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.getThumbnailsFromPlaylistOrVideoItem; -import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.parseDateWrapper; import static org.schabi.newpipe.extractor.stream.AudioStream.UNKNOWN_BITRATE; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; @@ -80,8 +79,7 @@ public String getTextualUploadDate() throws ParsingException { @Override public DateWrapper getUploadDate() throws ParsingException { - final String textualUploadDate = getTextualUploadDate(); - return textualUploadDate == null ? null : parseDateWrapper(textualUploadDate); + return DateWrapper.fromInstant(getTextualUploadDate()); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java index 65ceab3a08..7016fe047f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java @@ -14,7 +14,6 @@ import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.getAvatarsFromOwnerAccountOrVideoChannelObject; import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.getThumbnailsFromPlaylistOrVideoItem; -import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.parseDateWrapper; public class PeertubeStreamInfoItemExtractor implements StreamInfoItemExtractor { @@ -85,8 +84,7 @@ public String getTextualUploadDate() throws ParsingException { @Override public DateWrapper getUploadDate() throws ParsingException { - final String textualUploadDate = getTextualUploadDate(); - return textualUploadDate == null ? null : parseDateWrapper(textualUploadDate); + return DateWrapper.fromInstant(getTextualUploadDate()); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index 07e359e581..aeff6bd363 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -40,7 +40,8 @@ import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; -import java.time.Instant; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; import java.util.Collections; import java.util.List; @@ -134,15 +135,17 @@ public static synchronized String clientId() throws ExtractionException, IOExcep } @Nullable - public static DateWrapper parseDateFrom(@Nullable final String uploadDate) - throws ParsingException { - if (uploadDate == null) { - return null; - } + public static DateWrapper parseDate(final String uploadDate) throws ParsingException { try { - return new DateWrapper(Instant.parse(uploadDate)); + return DateWrapper.fromInstant(uploadDate); } catch (final DateTimeParseException e) { - throw new ParsingException("Could not parse date: \"" + uploadDate + "\"", e); + try { + return new DateWrapper(OffsetDateTime.parse(uploadDate, + DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss +0000"))); + } catch (final DateTimeParseException e1) { + e1.addSuppressed(e); + throw new ParsingException("Could not parse date: \"" + uploadDate + "\"", e1); + } } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java index d2bc46b95e..d5446e56bb 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java @@ -13,7 +13,7 @@ import java.util.Objects; import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.getAllImagesFromArtworkOrAvatarUrl; -import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.parseDateFrom; +import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.parseDate; public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtractor { private final JsonObject json; @@ -69,7 +69,7 @@ public String getTextualUploadDate() { @Nullable @Override public DateWrapper getUploadDate() throws ParsingException { - return parseDateFrom(getTextualUploadDate()); + return parseDate(getTextualUploadDate()); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java index 486b3c9938..8f69af0883 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java @@ -5,7 +5,7 @@ import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.getAllImagesFromArtworkOrAvatarUrl; import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.getAllImagesFromTrackObject; import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.getAvatarUrl; -import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.parseDateFrom; +import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.parseDate; import static org.schabi.newpipe.extractor.stream.Stream.ID_UNKNOWN; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; @@ -97,7 +97,7 @@ public String getTextualUploadDate() { @Nullable @Override public DateWrapper getUploadDate() throws ParsingException { - return parseDateFrom(getTextualUploadDate()); + return parseDate(getTextualUploadDate()); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamInfoItemExtractor.java index ea6bbe2f8e..f12301738c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamInfoItemExtractor.java @@ -13,7 +13,7 @@ import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.getAllImagesFromArtworkOrAvatarUrl; import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.getAllImagesFromTrackObject; -import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.parseDateFrom; +import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.parseDate; import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtractor { @@ -68,7 +68,7 @@ public String getTextualUploadDate() { @Override public DateWrapper getUploadDate() throws ParsingException { - return parseDateFrom(getTextualUploadDate()); + return parseDate(getTextualUploadDate()); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedInfoItemExtractor.java index 74dc19b915..f2a545878b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedInfoItemExtractor.java @@ -68,8 +68,7 @@ public String getTextualUploadDate() { @Nullable @Override public DateWrapper getUploadDate() throws ParsingException { - final String date = getTextualUploadDate(); - return date == null ? null : ExtractorHelper.parseDateWrapper(date); + return DateWrapper.fromOffsetDateTime(getTextualUploadDate()); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ExtractorHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ExtractorHelper.java index 0619b52e5d..12023ca4c9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ExtractorHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ExtractorHelper.java @@ -5,31 +5,16 @@ import org.schabi.newpipe.extractor.InfoItemsCollector; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage; -import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.StreamInfo; -import java.time.OffsetDateTime; -import java.time.format.DateTimeParseException; import java.util.Collections; import java.util.List; -import javax.annotation.Nonnull; - public final class ExtractorHelper { private ExtractorHelper() { } - @Nonnull - public static DateWrapper parseDateWrapper(@Nonnull final String date) throws ParsingException { - try { - return new DateWrapper(OffsetDateTime.parse(date)); - } catch (final DateTimeParseException e) { - throw new ParsingException("Could not parse date: \"" + date + "\"", e); - } - } - public static InfoItemsPage getItemsPageOrLogError( final Info info, final ListExtractor extractor) { try { From a84d83177e132478a26ced45789ec5201a22927f Mon Sep 17 00:00:00 2001 From: Isira Seneviratne <31027858+Isira-Seneviratne@users.noreply.github.com> Date: Wed, 8 Oct 2025 07:27:12 +0530 Subject: [PATCH 15/20] Apply suggestion from @Stypox Co-authored-by: Stypox --- .../extractors/kiosk/YoutubeChartsBaseKioskExtractor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/kiosk/YoutubeChartsBaseKioskExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/kiosk/YoutubeChartsBaseKioskExtractor.java index f96c603875..f5618a342a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/kiosk/YoutubeChartsBaseKioskExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/kiosk/YoutubeChartsBaseKioskExtractor.java @@ -210,7 +210,7 @@ public DateWrapper getUploadDate() { // the JSON body, but YouTube charts does it only in its HTTP headers final var instant = localDate.atStartOfDay(ZoneOffset.UTC).toInstant(); - // We don't have more info than the release day + // We don't have more info than the release day, hence isApproximate=true return new DateWrapper(instant, true); } From 9355798bccacc5514533a89de92ab7854a5dca28 Mon Sep 17 00:00:00 2001 From: Isira Seneviratne Date: Wed, 8 Oct 2025 08:21:42 +0530 Subject: [PATCH 16/20] Reduce some lines of code --- .../newpipe/extractor/localization/DateWrapper.java | 10 ++-------- .../youtube/extractors/YoutubeStreamExtractor.java | 9 ++------- 2 files changed, 4 insertions(+), 15 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/localization/DateWrapper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/localization/DateWrapper.java index bbe672fbbc..3eaf2d6314 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/localization/DateWrapper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/localization/DateWrapper.java @@ -91,22 +91,16 @@ public String toString() { } public static DateWrapper fromOffsetDateTime(final String date) throws ParsingException { - if (date == null) { - return null; - } try { - return new DateWrapper(OffsetDateTime.parse(date)); + return date != null ? new DateWrapper(OffsetDateTime.parse(date)) : null; } catch (final DateTimeParseException e) { throw new ParsingException("Could not parse date: \"" + date + "\"", e); } } public static DateWrapper fromInstant(final String date) throws ParsingException { - if (date == null) { - return null; - } try { - return new DateWrapper(Instant.parse(date)); + return date != null ? new DateWrapper(Instant.parse(date)) : null; } catch (final DateTimeParseException e) { throw new ParsingException("Could not parse date: \"" + date + "\"", e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 373edf9a71..5162e0db47 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -87,7 +87,6 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.time.LocalDate; -import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; import java.util.ArrayList; @@ -211,13 +210,9 @@ public String getTextualUploadDate() throws ParsingException { @Override public DateWrapper getUploadDate() throws ParsingException { final String dateText = getTextualUploadDate(); - if (dateText == null) { - return null; - } - try { - return new DateWrapper(OffsetDateTime.parse(dateText)); - } catch (final DateTimeParseException e) { + return DateWrapper.fromOffsetDateTime(dateText); + } catch (final ParsingException e) { // Try other patterns first } From 2d794e7a6d1f3282831e61220c192fb85bb716b1 Mon Sep 17 00:00:00 2001 From: Isira Seneviratne Date: Wed, 8 Oct 2025 08:24:35 +0530 Subject: [PATCH 17/20] Fix nullable annotations --- .../media_ccc/extractors/MediaCCCStreamExtractor.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java index 1838cf2bf9..0cb0ac4de8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java @@ -36,6 +36,7 @@ import java.util.Locale; import javax.annotation.Nonnull; +import javax.annotation.Nullable; public class MediaCCCStreamExtractor extends StreamExtractor { private JsonObject data; @@ -45,13 +46,13 @@ public MediaCCCStreamExtractor(final StreamingService service, final LinkHandler super(service, linkHandler); } - @Nonnull + @Nullable @Override public String getTextualUploadDate() { return data.getString("release_date"); } - @Nonnull + @Nullable @Override public DateWrapper getUploadDate() throws ParsingException { return DateWrapper.fromOffsetDateTime(getTextualUploadDate()); From fa4b7175564278e1b28b1688ef42797ad9631104 Mon Sep 17 00:00:00 2001 From: Isira Seneviratne Date: Fri, 10 Oct 2025 05:31:30 +0530 Subject: [PATCH 18/20] Update YoutubeStreamInfoItemLockupExtractor --- .../extractors/YoutubeStreamInfoItemLockupExtractor.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemLockupExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemLockupExtractor.java index f685f33dec..416b9a7258 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemLockupExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemLockupExtractor.java @@ -18,7 +18,6 @@ import org.schabi.newpipe.extractor.utils.Utils; import java.time.LocalDateTime; -import java.time.OffsetDateTime; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; @@ -292,8 +291,8 @@ public DateWrapper getUploadDate() throws ParsingException { try { // As we request a UTC offset of 0 minutes, we get the UTC date - return new DateWrapper(OffsetDateTime.of(LocalDateTime.parse( - premiereDate, PREMIERES_DATE_FORMATTER), ZoneOffset.UTC)); + final var dateTime = LocalDateTime.parse(premiereDate, PREMIERES_DATE_FORMATTER); + return new DateWrapper(dateTime.atZone(ZoneOffset.UTC).toInstant(), false); } catch (final DateTimeParseException e) { throw new ParsingException("Could not parse premiere upload date", e); } From e26cc84f9985bab0542d13ba21c70b02ea287cd7 Mon Sep 17 00:00:00 2001 From: Isira Seneviratne Date: Fri, 10 Oct 2025 08:15:46 +0530 Subject: [PATCH 19/20] Added documentation for parsing methods --- .../extractor/localization/DateWrapper.java | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/localization/DateWrapper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/localization/DateWrapper.java index 3eaf2d6314..00dcd124df 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/localization/DateWrapper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/localization/DateWrapper.java @@ -3,6 +3,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import javax.annotation.Nonnull; +import javax.annotation.Nullable; import java.io.Serializable; import java.time.Instant; import java.time.LocalDateTime; @@ -90,6 +91,15 @@ public String toString() { + '}'; } + /** + * Parses a date string that matches the ISO-8601 {@link OffsetDateTime} pattern, e.g. + * "2011-12-03T10:15:30+01:00". + * + * @param date The date string + * @return a non-approximate {@link DateWrapper}, or null if the string is null + * @throws ParsingException if the string does not match the expected format + */ + @Nullable public static DateWrapper fromOffsetDateTime(final String date) throws ParsingException { try { return date != null ? new DateWrapper(OffsetDateTime.parse(date)) : null; @@ -98,6 +108,15 @@ public static DateWrapper fromOffsetDateTime(final String date) throws ParsingEx } } + /** + * Parses a date string that matches the ISO-8601 {@link Instant} pattern, e.g. + * "2011-12-03T10:15:30Z". + * + * @param date The date string + * @return a non-approximate {@link DateWrapper}, or null if the string is null + * @throws ParsingException if the string does not match the expected format + */ + @Nullable public static DateWrapper fromInstant(final String date) throws ParsingException { try { return date != null ? new DateWrapper(Instant.parse(date)) : null; From 2b306a32e514d2c8aec665dd769f3ee2bf93b044 Mon Sep 17 00:00:00 2001 From: Isira Seneviratne Date: Fri, 10 Oct 2025 08:31:52 +0530 Subject: [PATCH 20/20] Remove unthrown exception --- .../services/youtube/extractors/YoutubeStreamExtractor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 5162e0db47..e6fa782ce7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -171,7 +171,7 @@ public String getName() throws ParsingException { @Nullable @Override - public String getTextualUploadDate() throws ParsingException { + public String getTextualUploadDate() { String timestamp = playerMicroFormatRenderer.getString("uploadDate", ""); if (timestamp.isEmpty()) { timestamp = playerMicroFormatRenderer.getString("publishDate", "");