diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index 19ae0ac0af..68acc6c361 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -235,6 +235,10 @@ public static boolean isY2ubeURL(@Nonnull final URL url) { */ public static int parseDurationString(@Nonnull final String input) throws ParsingException, NumberFormatException { + if (!input.matches(".*\\d.*") && !input.equalsIgnoreCase("SHORTS")) { + throw new ParsingException("Error duration string contains no digits: " + input); + } + // If time separator : is not detected, try . instead final String[] splitInput = input.contains(":") ? input.split(":") diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java index be951671d9..d0aeea697d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java @@ -49,6 +49,7 @@ import java.time.format.DateTimeFormatter; import java.util.List; import java.util.regex.Pattern; +import java.util.stream.Collectors; public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { @@ -155,19 +156,24 @@ public long getDuration() throws ParsingException { duration = videoInfo.getString("lengthSeconds"); if (isNullOrEmpty(duration)) { - final JsonObject timeOverlay = videoInfo.getArray("thumbnailOverlays") + final List timeOverlays = videoInfo.getArray("thumbnailOverlays") .stream() .filter(JsonObject.class::isInstance) .map(JsonObject.class::cast) .filter(thumbnailOverlay -> thumbnailOverlay.has("thumbnailOverlayTimeStatusRenderer")) - .findFirst() - .orElse(null); - - if (timeOverlay != null) { - duration = getTextFromObject( - timeOverlay.getObject("thumbnailOverlayTimeStatusRenderer") - .getObject("text")); + .map(thumbnailOverlay -> getTextFromObject( + thumbnailOverlay.getObject("thumbnailOverlayTimeStatusRenderer") + .getObject("text"))) + .filter(text -> !isNullOrEmpty(text)) + .collect(Collectors.toList()); + + for (final String timeOverlayText : timeOverlays) { + try { + return YoutubeParsingHelper.parseDurationString(timeOverlayText); + } catch (final ParsingException ex) { + // try next + } } } @@ -452,24 +458,21 @@ public boolean isShortFormContent() throws ParsingException { } if (!isShort) { - final JsonObject thumbnailTimeOverlay = videoInfo.getArray("thumbnailOverlays") - .stream() - .filter(JsonObject.class::isInstance) - .map(JsonObject.class::cast) - .filter(thumbnailOverlay -> thumbnailOverlay.has( - "thumbnailOverlayTimeStatusRenderer")) - .map(thumbnailOverlay -> thumbnailOverlay.getObject( - "thumbnailOverlayTimeStatusRenderer")) - .findFirst() - .orElse(null); - - if (!isNullOrEmpty(thumbnailTimeOverlay)) { - isShort = thumbnailTimeOverlay.getString("style", "") - .equalsIgnoreCase("SHORTS") - || thumbnailTimeOverlay.getObject("icon") - .getString("iconType", "") - .toLowerCase() - .contains("shorts"); + if (videoInfo.has("thumbnailOverlays")) { + isShort = videoInfo.getArray("thumbnailOverlays") + .stream() + .filter(JsonObject.class::isInstance) + .map(JsonObject.class::cast) + .filter(thumbnailOverlay -> thumbnailOverlay.has( + "thumbnailOverlayTimeStatusRenderer")) + .map(thumbnailOverlay -> thumbnailOverlay.getObject( + "thumbnailOverlayTimeStatusRenderer")) + .anyMatch(timeOverlay -> timeOverlay.getString("style", "") + .equalsIgnoreCase("SHORTS") + || timeOverlay.getObject("icon") + .getString("iconType", "") + .toLowerCase() + .contains("shorts")); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemLockupExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemLockupExtractor.java index 83fcccb28f..5325e3e900 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemLockupExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemLockupExtractor.java @@ -31,9 +31,9 @@ /** * Note: * This extractor is currently (2025-07) only used to extract related video streams.
- * The following features are currently not implemented because they have never been observed: + * The following features are currently not implemented: * */ @@ -77,22 +77,22 @@ public StreamType getStreamType() throws ParsingException { } private StreamType determineStreamType() throws ParsingException { - if (JsonUtils.getArray(lockupViewModel, "contentImage.thumbnailViewModel.overlays") - .streamAsJsonObjects() + final JsonArray overlays = JsonUtils.getArray(lockupViewModel, + "contentImage.thumbnailViewModel.overlays"); + + // thumbnailOverlayBadgeViewModel path (legacy/alternate overlay structure) + if (overlays.streamAsJsonObjects() .flatMap(overlay -> overlay .getObject("thumbnailOverlayBadgeViewModel") .getArray("thumbnailBadges") .streamAsJsonObjects()) .map(thumbnailBadge -> thumbnailBadge.getObject("thumbnailBadgeViewModel")) - .anyMatch(thumbnailBadgeViewModel -> { - if ("THUMBNAIL_OVERLAY_BADGE_STYLE_LIVE".equals( - thumbnailBadgeViewModel.getString("badgeStyle"))) { + .anyMatch(vm -> { + if ("THUMBNAIL_OVERLAY_BADGE_STYLE_LIVE".equals(vm.getString("badgeStyle"))) { return true; } - // Fallback: Check if there is a live icon - return thumbnailBadgeViewModel - .getObject("icon") + return vm.getObject("icon") .getArray("sources") .streamAsJsonObjects() .map(source -> source @@ -103,6 +103,18 @@ private StreamType determineStreamType() throws ParsingException { return StreamType.LIVE_STREAM; } + // thumbnailBottomOverlayViewModel path (used in lockup format for both duration and live) + if (overlays.streamAsJsonObjects() + .flatMap(overlay -> overlay + .getObject("thumbnailBottomOverlayViewModel") + .getArray("badges") + .streamAsJsonObjects()) + .map(badge -> badge.getObject("thumbnailBadgeViewModel")) + .anyMatch(vm -> "THUMBNAIL_OVERLAY_BADGE_STYLE_LIVE".equals( + vm.getString("badgeStyle")))) { + return StreamType.LIVE_STREAM; + } + return StreamType.VIDEO_STREAM; } @@ -164,11 +176,14 @@ public long getDuration() throws ParsingException { .collect(Collectors.toList()); if (potentialDurations.isEmpty()) { - throw new ParsingException("Could not get duration: No parsable durations detected"); + return -1; } ParsingException parsingException = null; for (final String potentialDuration : potentialDurations) { + if (potentialDuration == null || !potentialDuration.matches(".*\\d.*")) { + continue; + } try { return YoutubeParsingHelper.parseDurationString(potentialDuration); } catch (final ParsingException ex) { @@ -176,6 +191,10 @@ public long getDuration() throws ParsingException { } } + if (parsingException == null) { + return -1; // e.g. only "SHORTS" or "CC" badge was present, no duration available + } + throw new ParsingException("Could not get duration", parsingException); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamInfoItemTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamInfoItemTest.java index 0f7100caaf..2bb232d862 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamInfoItemTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamInfoItemTest.java @@ -86,6 +86,58 @@ void lockupViewModelPremiere() ); } + @Test + void lockupViewModelVideo() + throws FileNotFoundException, JsonParserException { + final var json = JsonParser.object().from(new FileInputStream(getMockPath( + YoutubeStreamInfoItemTest.class, "lockupViewModelVideo") + ".json")); + final var timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(Localization.DEFAULT); + final var extractor = new YoutubeStreamInfoItemLockupExtractor(json, timeAgoParser); + assertAll( + () -> assertEquals(StreamType.VIDEO_STREAM, extractor.getStreamType()), + () -> assertFalse(extractor.isAd()), + () -> assertEquals("https://www.youtube.com/watch?v=dQw4w9WgXcQ", extractor.getUrl()), + () -> assertEquals("VIDEO_TITLE", extractor.getName()), + () -> assertEquals(974, extractor.getDuration()), + () -> assertFalse(extractor.getThumbnails().isEmpty()) + ); + } + + @Test + void lockupViewModelLiveStream() + throws FileNotFoundException, JsonParserException { + final var json = JsonParser.object().from(new FileInputStream(getMockPath( + YoutubeStreamInfoItemTest.class, "lockupViewModelLiveStream") + ".json")); + final var timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(Localization.DEFAULT); + final var extractor = new YoutubeStreamInfoItemLockupExtractor(json, timeAgoParser); + assertAll( + () -> assertEquals(StreamType.LIVE_STREAM, extractor.getStreamType()), + () -> assertFalse(extractor.isAd()), + () -> assertEquals("https://www.youtube.com/watch?v=LIVE_VIDEO_ID", extractor.getUrl()), + () -> assertEquals("LIVE_VIDEO_TITLE", extractor.getName()), + () -> assertEquals(-1, extractor.getDuration()), + () -> assertNull(extractor.getTextualUploadDate()), + () -> assertNull(extractor.getUploadDate()), + () -> assertEquals(0, extractor.getViewCount()), + () -> assertFalse(extractor.getThumbnails().isEmpty()) + ); + } + + @Test + void lockupViewModelNoDuration() + throws FileNotFoundException, JsonParserException { + final var json = JsonParser.object().from(new FileInputStream(getMockPath( + YoutubeStreamInfoItemTest.class, "lockupViewModelNoDuration") + ".json")); + final var timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(Localization.DEFAULT); + final var extractor = new YoutubeStreamInfoItemLockupExtractor(json, timeAgoParser); + assertAll( + () -> assertEquals(StreamType.VIDEO_STREAM, extractor.getStreamType()), + () -> assertFalse(extractor.isAd()), + () -> assertEquals(-1, extractor.getDuration()), + () -> assertFalse(extractor.getThumbnails().isEmpty()) + ); + } + @Test void emptyTitle() throws FileNotFoundException, JsonParserException { final var json = JsonParser.object().from(new FileInputStream(getMockPath( diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java index 63b4c23ff7..2cc45ea795 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java @@ -235,7 +235,8 @@ protected SearchExtractor createExtractor() throws Exception { Collections.singletonList("Learn more") )); } - // testMoreRelatedItems is broken because a video has no duration shown + // testMoreRelatedItems: a video in this mock has no duration badge; getDuration() now + // returns -1 instead of throwing, but the mock may need re-recording before re-enabling. @Test @Override public void testMoreRelatedItems() { } @Override public StreamingService expectedService() { return YouTube; } @Override public String expectedName() { return QUERY; } diff --git a/extractor/src/test/resources/mocks/v1/org/schabi/newpipe/extractor/services/youtube/youtubestreaminfoitem/lockupviewmodellivestream.json b/extractor/src/test/resources/mocks/v1/org/schabi/newpipe/extractor/services/youtube/youtubestreaminfoitem/lockupviewmodellivestream.json new file mode 100644 index 0000000000..0ed5e8fc44 --- /dev/null +++ b/extractor/src/test/resources/mocks/v1/org/schabi/newpipe/extractor/services/youtube/youtubestreaminfoitem/lockupviewmodellivestream.json @@ -0,0 +1,103 @@ +{ + "contentImage": { + "thumbnailViewModel": { + "image": { + "sources": [ + { + "url": "https://i.ytimg.com/vi/LIVE_VIDEO_ID/hqdefault.jpg", + "width": 168, + "height": 94 + }, + { + "url": "https://i.ytimg.com/vi/LIVE_VIDEO_ID/hqdefault.jpg", + "width": 336, + "height": 188 + } + ] + }, + "overlays": [ + { + "thumbnailBottomOverlayViewModel": { + "badges": [ + { + "thumbnailBadgeViewModel": { + "badgeStyle": "THUMBNAIL_OVERLAY_BADGE_STYLE_LIVE", + "text": "LIVE" + } + } + ] + } + }, + { + "thumbnailHoverOverlayToggleActionsViewModel": { + "buttons": [] + } + } + ] + } + }, + "metadata": { + "lockupMetadataViewModel": { + "title": { + "content": "LIVE_VIDEO_TITLE" + }, + "image": { + "decoratedAvatarViewModel": { + "avatar": { + "avatarViewModel": { + "image": { + "sources": [ + { + "url": "https://yt3.ggpht.com/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", + "width": 68, + "height": 68 + } + ] + }, + "avatarImageSize": "AVATAR_SIZE_M" + } + }, + "a11yLabel": "Go to channel", + "rendererContext": { + "commandContext": { + "onTap": { + "innertubeCommand": { + "commandMetadata": { + "webCommandMetadata": { + "url": "/@LIVE_CHANNEL_HANDLE" + } + }, + "browseEndpoint": { + "browseId": "UCD_on7-zu7Zuc3zissQvrgw", + "canonicalBaseUrl": "/@LIVE_CHANNEL_HANDLE" + } + } + } + } + } + } + }, + "metadata": { + "contentMetadataViewModel": { + "metadataRows": [ + { + "metadataParts": [ + { + "text": { + "content": "LIVE_CHANNEL_NAME", + "styleRuns": [], + "attachmentRuns": [] + } + } + ] + } + ], + "delimiter": " • " + } + }, + "menuButton": {} + } + }, + "contentId": "LIVE_VIDEO_ID", + "contentType": "LOCKUP_CONTENT_TYPE_VIDEO" +} diff --git a/extractor/src/test/resources/mocks/v1/org/schabi/newpipe/extractor/services/youtube/youtubestreaminfoitem/lockupviewmodelnoduration.json b/extractor/src/test/resources/mocks/v1/org/schabi/newpipe/extractor/services/youtube/youtubestreaminfoitem/lockupviewmodelnoduration.json new file mode 100644 index 0000000000..a88db0e153 --- /dev/null +++ b/extractor/src/test/resources/mocks/v1/org/schabi/newpipe/extractor/services/youtube/youtubestreaminfoitem/lockupviewmodelnoduration.json @@ -0,0 +1,105 @@ +{ + "contentImage": { + "thumbnailViewModel": { + "image": { + "sources": [ + { + "url": "https://i.ytimg.com/vi/dQw4w9WgXcQ/hqdefault.jpg", + "width": 168, + "height": 94 + }, + { + "url": "https://i.ytimg.com/vi/dQw4w9WgXcQ/hqdefault.jpg", + "width": 336, + "height": 188 + } + ] + }, + "overlays": [ + { + "thumbnailHoverOverlayToggleActionsViewModel": { + "buttons": [] + } + } + ] + } + }, + "metadata": { + "lockupMetadataViewModel": { + "title": { + "content": "VIDEO_TITLE_SHORT" + }, + "image": { + "decoratedAvatarViewModel": { + "avatar": { + "avatarViewModel": { + "image": { + "sources": [ + { + "url": "https://yt3.ggpht.com/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", + "width": 68, + "height": 68 + } + ] + }, + "avatarImageSize": "AVATAR_SIZE_M" + } + }, + "a11yLabel": "Go to channel", + "rendererContext": { + "commandContext": { + "onTap": { + "innertubeCommand": { + "commandMetadata": { + "webCommandMetadata": { + "url": "/@VIDEO_CHANNEL_HANDLE" + } + }, + "browseEndpoint": { + "browseId": "UCD_on7-zu7Zuc3zissQvrgw", + "canonicalBaseUrl": "/@VIDEO_CHANNEL_HANDLE" + } + } + } + } + } + } + }, + "metadata": { + "contentMetadataViewModel": { + "metadataRows": [ + { + "metadataParts": [ + { + "text": { + "content": "VIDEO_CHANNEL_NAME", + "styleRuns": [], + "attachmentRuns": [] + } + } + ] + }, + { + "metadataParts": [ + { + "text": { + "content": "500K views" + } + }, + { + "text": { + "content": "1 month ago" + } + } + ] + } + ], + "delimiter": " • " + } + }, + "menuButton": {} + } + }, + "contentId": "dQw4w9WgXcQ", + "contentType": "LOCKUP_CONTENT_TYPE_VIDEO" +} diff --git a/extractor/src/test/resources/mocks/v1/org/schabi/newpipe/extractor/services/youtube/youtubestreaminfoitem/lockupviewmodelvideo.json b/extractor/src/test/resources/mocks/v1/org/schabi/newpipe/extractor/services/youtube/youtubestreaminfoitem/lockupviewmodelvideo.json new file mode 100644 index 0000000000..7ba2e99f04 --- /dev/null +++ b/extractor/src/test/resources/mocks/v1/org/schabi/newpipe/extractor/services/youtube/youtubestreaminfoitem/lockupviewmodelvideo.json @@ -0,0 +1,117 @@ +{ + "contentImage": { + "thumbnailViewModel": { + "image": { + "sources": [ + { + "url": "https://i.ytimg.com/vi/dQw4w9WgXcQ/hqdefault.jpg", + "width": 168, + "height": 94 + }, + { + "url": "https://i.ytimg.com/vi/dQw4w9WgXcQ/hqdefault.jpg", + "width": 336, + "height": 188 + } + ] + }, + "overlays": [ + { + "thumbnailBottomOverlayViewModel": { + "badges": [ + { + "thumbnailBadgeViewModel": { + "text": "16:14", + "badgeStyle": "THUMBNAIL_OVERLAY_BADGE_STYLE_DEFAULT" + } + } + ] + } + }, + { + "thumbnailHoverOverlayToggleActionsViewModel": { + "buttons": [] + } + } + ] + } + }, + "metadata": { + "lockupMetadataViewModel": { + "title": { + "content": "VIDEO_TITLE" + }, + "image": { + "decoratedAvatarViewModel": { + "avatar": { + "avatarViewModel": { + "image": { + "sources": [ + { + "url": "https://yt3.ggpht.com/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", + "width": 68, + "height": 68 + } + ] + }, + "avatarImageSize": "AVATAR_SIZE_M" + } + }, + "a11yLabel": "Go to channel", + "rendererContext": { + "commandContext": { + "onTap": { + "innertubeCommand": { + "commandMetadata": { + "webCommandMetadata": { + "url": "/@VIDEO_CHANNEL_HANDLE" + } + }, + "browseEndpoint": { + "browseId": "UCD_on7-zu7Zuc3zissQvrgw", + "canonicalBaseUrl": "/@VIDEO_CHANNEL_HANDLE" + } + } + } + } + } + } + }, + "metadata": { + "contentMetadataViewModel": { + "metadataRows": [ + { + "metadataParts": [ + { + "text": { + "content": "VIDEO_CHANNEL_NAME", + "styleRuns": [], + "attachmentRuns": [] + } + } + ] + }, + { + "metadataParts": [ + { + "text": { + "content": "1.2M views" + } + }, + { + "text": { + "content": "2 years ago" + } + } + ] + } + ], + "delimiter": " • " + } + }, + "menuButton": {} + } + }, + "contentId": "dQw4w9WgXcQ", + "contentType": "LOCKUP_CONTENT_TYPE_VIDEO" +}