Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -235,6 +235,10 @@ public static boolean isY2ubeURL(@Nonnull final URL url) {
*/
public static int parseDurationString(@Nonnull final String input)
throws ParsingException, NumberFormatException {
if (!input.matches(".*\\d.*") && !input.equalsIgnoreCase("SHORTS")) {
throw new ParsingException("Error duration string contains no digits: " + input);
}

Comment on lines +238 to +241
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I do not understand this change. Why exclude shorts here? If SHORTS is passed, it will fail in convertDurationToInt.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

YouTube does something special with shorts in certain places, instead of showing the numeric duration in the corner of the thumbnail, it simply shows the word SHORTS

SHORTS does not actually fail in convertDurationToInt, and that is precisely why the check exists.

Without the !input.equalsIgnoreCase("SHORTS") guard: parseDurationString("SHORTS") would hit this line first

if (!input.matches(".*\\d.*")) {
    throw new ParsingException("Error duration string contains no digits: " + input);
}

It would throw immediately, never reaching convertDurationToInt at all.

The check is not there to exclude shorts, but to allow them thrugh so that the downstream fallback (returning 0) can handle the case where YouTube sends SHORTS instead of an actual timestamp

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah yeah, I see. Thank you.

// If time separator : is not detected, try . instead
final String[] splitInput = input.contains(":")
? input.split(":")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@
import java.time.format.DateTimeFormatter;
import java.util.List;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {

Expand Down Expand Up @@ -143,7 +144,7 @@
}

@Override
public long getDuration() throws ParsingException {

Check failure on line 147 in extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Refactor this method to reduce its Cognitive Complexity from 16 to the 15 allowed.

See more on https://sonarcloud.io/project/issues?id=TeamNewPipe_NewPipeExtractor&issues=AZ2m1x82WySueEBa6Yit&open=AZ2m1x82WySueEBa6Yit&pullRequest=1478
if (getStreamType() == StreamType.LIVE_STREAM) {
return -1;
}
Expand All @@ -155,19 +156,24 @@
duration = videoInfo.getString("lengthSeconds");

if (isNullOrEmpty(duration)) {
final JsonObject timeOverlay = videoInfo.getArray("thumbnailOverlays")
final List<String> timeOverlays = videoInfo.getArray("thumbnailOverlays")
.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.filter(thumbnailOverlay ->
thumbnailOverlay.has("thumbnailOverlayTimeStatusRenderer"))
.findFirst()
.orElse(null);

if (timeOverlay != null) {
duration = getTextFromObject(
timeOverlay.getObject("thumbnailOverlayTimeStatusRenderer")
.getObject("text"));
.map(thumbnailOverlay -> getTextFromObject(
thumbnailOverlay.getObject("thumbnailOverlayTimeStatusRenderer")
.getObject("text")))
.filter(text -> !isNullOrEmpty(text))
.collect(Collectors.toList());

for (final String timeOverlayText : timeOverlays) {
try {
return YoutubeParsingHelper.parseDurationString(timeOverlayText);
} catch (final ParsingException ex) {
// try next
}
}
}

Expand Down Expand Up @@ -452,24 +458,21 @@
}

if (!isShort) {
final JsonObject thumbnailTimeOverlay = videoInfo.getArray("thumbnailOverlays")
.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.filter(thumbnailOverlay -> thumbnailOverlay.has(
"thumbnailOverlayTimeStatusRenderer"))
.map(thumbnailOverlay -> thumbnailOverlay.getObject(
"thumbnailOverlayTimeStatusRenderer"))
.findFirst()
.orElse(null);

if (!isNullOrEmpty(thumbnailTimeOverlay)) {
isShort = thumbnailTimeOverlay.getString("style", "")
.equalsIgnoreCase("SHORTS")
|| thumbnailTimeOverlay.getObject("icon")
.getString("iconType", "")
.toLowerCase()
.contains("shorts");
if (videoInfo.has("thumbnailOverlays")) {

Check warning on line 461 in extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Merge this if statement with the enclosing one.

See more on https://sonarcloud.io/project/issues?id=TeamNewPipe_NewPipeExtractor&issues=AZ2m1x82WySueEBa6Yis&open=AZ2m1x82WySueEBa6Yis&pullRequest=1478
isShort = videoInfo.getArray("thumbnailOverlays")
.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.filter(thumbnailOverlay -> thumbnailOverlay.has(
"thumbnailOverlayTimeStatusRenderer"))
.map(thumbnailOverlay -> thumbnailOverlay.getObject(
"thumbnailOverlayTimeStatusRenderer"))
.anyMatch(timeOverlay -> timeOverlay.getString("style", "")
.equalsIgnoreCase("SHORTS")
|| timeOverlay.getObject("icon")
.getString("iconType", "")
.toLowerCase()
.contains("shorts"));
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,9 @@
/**
* Note:
* This extractor is currently (2025-07) only used to extract related video streams.<br>
* The following features are currently not implemented because they have never been observed:
* The following features are currently not implemented:
* <ul>
* <li>Shorts</li>
* <li>Shorts: appear in related videos without a duration badge; getDuration() returns -1</li>
* <li>Paid content (Premium, members first or only)</li>
* </ul>
*/
Expand Down Expand Up @@ -77,22 +77,22 @@
}

private StreamType determineStreamType() throws ParsingException {
if (JsonUtils.getArray(lockupViewModel, "contentImage.thumbnailViewModel.overlays")
.streamAsJsonObjects()
final JsonArray overlays = JsonUtils.getArray(lockupViewModel,
"contentImage.thumbnailViewModel.overlays");

// thumbnailOverlayBadgeViewModel path (legacy/alternate overlay structure)
if (overlays.streamAsJsonObjects()
.flatMap(overlay -> overlay
.getObject("thumbnailOverlayBadgeViewModel")
.getArray("thumbnailBadges")
.streamAsJsonObjects())
.map(thumbnailBadge -> thumbnailBadge.getObject("thumbnailBadgeViewModel"))

Check failure on line 89 in extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemLockupExtractor.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Define a constant instead of duplicating this literal "thumbnailBadgeViewModel" 3 times.

See more on https://sonarcloud.io/project/issues?id=TeamNewPipe_NewPipeExtractor&issues=AZ2m1x_GWySueEBa6Yiu&open=AZ2m1x_GWySueEBa6Yiu&pullRequest=1478
.anyMatch(thumbnailBadgeViewModel -> {
if ("THUMBNAIL_OVERLAY_BADGE_STYLE_LIVE".equals(
thumbnailBadgeViewModel.getString("badgeStyle"))) {
.anyMatch(vm -> {
if ("THUMBNAIL_OVERLAY_BADGE_STYLE_LIVE".equals(vm.getString("badgeStyle"))) {
return true;
}

// Fallback: Check if there is a live icon
return thumbnailBadgeViewModel
.getObject("icon")
return vm.getObject("icon")
.getArray("sources")
.streamAsJsonObjects()
.map(source -> source
Expand All @@ -103,6 +103,18 @@
return StreamType.LIVE_STREAM;
}

// thumbnailBottomOverlayViewModel path (used in lockup format for both duration and live)
if (overlays.streamAsJsonObjects()
.flatMap(overlay -> overlay
.getObject("thumbnailBottomOverlayViewModel")
.getArray("badges")
.streamAsJsonObjects())
.map(badge -> badge.getObject("thumbnailBadgeViewModel"))
.anyMatch(vm -> "THUMBNAIL_OVERLAY_BADGE_STYLE_LIVE".equals(
vm.getString("badgeStyle")))) {
return StreamType.LIVE_STREAM;
}

return StreamType.VIDEO_STREAM;
}

Expand Down Expand Up @@ -164,18 +176,25 @@
.collect(Collectors.toList());

if (potentialDurations.isEmpty()) {
throw new ParsingException("Could not get duration: No parsable durations detected");
return -1;
}

ParsingException parsingException = null;
for (final String potentialDuration : potentialDurations) {
if (potentialDuration == null || !potentialDuration.matches(".*\\d.*")) {
continue;
}
try {
return YoutubeParsingHelper.parseDurationString(potentialDuration);
} catch (final ParsingException ex) {
parsingException = ex;
}
}

if (parsingException == null) {
return -1; // e.g. only "SHORTS" or "CC" badge was present, no duration available
}

throw new ParsingException("Could not get duration", parsingException);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,58 @@ void lockupViewModelPremiere()
);
}

@Test
void lockupViewModelVideo()
throws FileNotFoundException, JsonParserException {
final var json = JsonParser.object().from(new FileInputStream(getMockPath(
YoutubeStreamInfoItemTest.class, "lockupViewModelVideo") + ".json"));
final var timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(Localization.DEFAULT);
final var extractor = new YoutubeStreamInfoItemLockupExtractor(json, timeAgoParser);
assertAll(
() -> assertEquals(StreamType.VIDEO_STREAM, extractor.getStreamType()),
() -> assertFalse(extractor.isAd()),
() -> assertEquals("https://www.youtube.com/watch?v=dQw4w9WgXcQ", extractor.getUrl()),
() -> assertEquals("VIDEO_TITLE", extractor.getName()),
() -> assertEquals(974, extractor.getDuration()),
() -> assertFalse(extractor.getThumbnails().isEmpty())
);
}

@Test
void lockupViewModelLiveStream()
throws FileNotFoundException, JsonParserException {
final var json = JsonParser.object().from(new FileInputStream(getMockPath(
YoutubeStreamInfoItemTest.class, "lockupViewModelLiveStream") + ".json"));
final var timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(Localization.DEFAULT);
final var extractor = new YoutubeStreamInfoItemLockupExtractor(json, timeAgoParser);
assertAll(
() -> assertEquals(StreamType.LIVE_STREAM, extractor.getStreamType()),
() -> assertFalse(extractor.isAd()),
() -> assertEquals("https://www.youtube.com/watch?v=LIVE_VIDEO_ID", extractor.getUrl()),
() -> assertEquals("LIVE_VIDEO_TITLE", extractor.getName()),
() -> assertEquals(-1, extractor.getDuration()),
() -> assertNull(extractor.getTextualUploadDate()),
() -> assertNull(extractor.getUploadDate()),
() -> assertEquals(0, extractor.getViewCount()),
() -> assertFalse(extractor.getThumbnails().isEmpty())
);
}

@Test
void lockupViewModelNoDuration()
throws FileNotFoundException, JsonParserException {
final var json = JsonParser.object().from(new FileInputStream(getMockPath(
YoutubeStreamInfoItemTest.class, "lockupViewModelNoDuration") + ".json"));
final var timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(Localization.DEFAULT);
final var extractor = new YoutubeStreamInfoItemLockupExtractor(json, timeAgoParser);
assertAll(
() -> assertEquals(StreamType.VIDEO_STREAM, extractor.getStreamType()),
() -> assertFalse(extractor.isAd()),
() -> assertEquals(-1, extractor.getDuration()),
() -> assertFalse(extractor.getThumbnails().isEmpty())
);
}

@Test
void emptyTitle() throws FileNotFoundException, JsonParserException {
final var json = JsonParser.object().from(new FileInputStream(getMockPath(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -235,7 +235,8 @@ protected SearchExtractor createExtractor() throws Exception {
Collections.singletonList("Learn more")
));
}
// testMoreRelatedItems is broken because a video has no duration shown
// testMoreRelatedItems: a video in this mock has no duration badge; getDuration() now
// returns -1 instead of throwing, but the mock may need re-recording before re-enabling.
@Test @Override public void testMoreRelatedItems() { }
@Override public StreamingService expectedService() { return YouTube; }
@Override public String expectedName() { return QUERY; }
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
{
"contentImage": {
"thumbnailViewModel": {
"image": {
"sources": [
{
"url": "https://i.ytimg.com/vi/LIVE_VIDEO_ID/hqdefault.jpg",
"width": 168,
"height": 94
},
{
"url": "https://i.ytimg.com/vi/LIVE_VIDEO_ID/hqdefault.jpg",
"width": 336,
"height": 188
}
]
},
"overlays": [
{
"thumbnailBottomOverlayViewModel": {
"badges": [
{
"thumbnailBadgeViewModel": {
"badgeStyle": "THUMBNAIL_OVERLAY_BADGE_STYLE_LIVE",
"text": "LIVE"
}
}
]
}
},
{
"thumbnailHoverOverlayToggleActionsViewModel": {
"buttons": []
}
}
]
}
},
"metadata": {
"lockupMetadataViewModel": {
"title": {
"content": "LIVE_VIDEO_TITLE"
},
"image": {
"decoratedAvatarViewModel": {
"avatar": {
"avatarViewModel": {
"image": {
"sources": [
{
"url": "https://yt3.ggpht.com/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==",
"width": 68,
"height": 68
}
]
},
"avatarImageSize": "AVATAR_SIZE_M"
}
},
"a11yLabel": "Go to channel",
"rendererContext": {
"commandContext": {
"onTap": {
"innertubeCommand": {
"commandMetadata": {
"webCommandMetadata": {
"url": "/@LIVE_CHANNEL_HANDLE"
}
},
"browseEndpoint": {
"browseId": "UCD_on7-zu7Zuc3zissQvrgw",
"canonicalBaseUrl": "/@LIVE_CHANNEL_HANDLE"
}
}
}
}
}
}
},
"metadata": {
"contentMetadataViewModel": {
"metadataRows": [
{
"metadataParts": [
{
"text": {
"content": "LIVE_CHANNEL_NAME",
"styleRuns": [],
"attachmentRuns": []
}
}
]
}
],
"delimiter": " • "
}
},
"menuButton": {}
}
},
"contentId": "LIVE_VIDEO_ID",
"contentType": "LOCKUP_CONTENT_TYPE_VIDEO"
}
Loading