Skip to content

Commit 827d206

Browse files
TobiGrTobiGr
authored
Merge pull request #1478 from Ecomont/fix/youtube-lockup-duration-live-stream
Fix duration and live stream display in related videos
2 parents 1512cf3 + 4bebf3d commit 827d206

8 files changed

Lines changed: 442 additions & 38 deletions

File tree

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -235,6 +235,10 @@ public static boolean isY2ubeURL(@Nonnull final URL url) {
235235
*/
236236
public static int parseDurationString(@Nonnull final String input)
237237
throws ParsingException, NumberFormatException {
238+
if (!input.matches(".*\\d.*") && !input.equalsIgnoreCase("SHORTS")) {
239+
throw new ParsingException("Error duration string contains no digits: " + input);
240+
}
241+
238242
// If time separator : is not detected, try . instead
239243
final String[] splitInput = input.contains(":")
240244
? input.split(":")

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java‎

Lines changed: 29 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@
4949
import java.time.format.DateTimeFormatter;
5050
import java.util.List;
5151
import java.util.regex.Pattern;
52+
import java.util.stream.Collectors;
5253

5354
public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
5455

@@ -155,19 +156,24 @@ public long getDuration() throws ParsingException {
155156
duration = videoInfo.getString("lengthSeconds");
156157

157158
if (isNullOrEmpty(duration)) {
158-
final JsonObject timeOverlay = videoInfo.getArray("thumbnailOverlays")
159+
final List<String> timeOverlays = videoInfo.getArray("thumbnailOverlays")
159160
.stream()
160161
.filter(JsonObject.class::isInstance)
161162
.map(JsonObject.class::cast)
162163
.filter(thumbnailOverlay ->
163164
thumbnailOverlay.has("thumbnailOverlayTimeStatusRenderer"))
164-
.findFirst()
165-
.orElse(null);
166-
167-
if (timeOverlay != null) {
168-
duration = getTextFromObject(
169-
timeOverlay.getObject("thumbnailOverlayTimeStatusRenderer")
170-
.getObject("text"));
165+
.map(thumbnailOverlay -> getTextFromObject(
166+
thumbnailOverlay.getObject("thumbnailOverlayTimeStatusRenderer")
167+
.getObject("text")))
168+
.filter(text -> !isNullOrEmpty(text))
169+
.collect(Collectors.toList());
170+
171+
for (final String timeOverlayText : timeOverlays) {
172+
try {
173+
return YoutubeParsingHelper.parseDurationString(timeOverlayText);
174+
} catch (final ParsingException ex) {
175+
// try next
176+
}
171177
}
172178
}
173179

@@ -452,24 +458,21 @@ public boolean isShortFormContent() throws ParsingException {
452458
}
453459

454460
if (!isShort) {
455-
final JsonObject thumbnailTimeOverlay = videoInfo.getArray("thumbnailOverlays")
456-
.stream()
457-
.filter(JsonObject.class::isInstance)
458-
.map(JsonObject.class::cast)
459-
.filter(thumbnailOverlay -> thumbnailOverlay.has(
460-
"thumbnailOverlayTimeStatusRenderer"))
461-
.map(thumbnailOverlay -> thumbnailOverlay.getObject(
462-
"thumbnailOverlayTimeStatusRenderer"))
463-
.findFirst()
464-
.orElse(null);
465-
466-
if (!isNullOrEmpty(thumbnailTimeOverlay)) {
467-
isShort = thumbnailTimeOverlay.getString("style", "")
468-
.equalsIgnoreCase("SHORTS")
469-
|| thumbnailTimeOverlay.getObject("icon")
470-
.getString("iconType", "")
471-
.toLowerCase()
472-
.contains("shorts");
461+
if (videoInfo.has("thumbnailOverlays")) {
462+
isShort = videoInfo.getArray("thumbnailOverlays")
463+
.stream()
464+
.filter(JsonObject.class::isInstance)
465+
.map(JsonObject.class::cast)
466+
.filter(thumbnailOverlay -> thumbnailOverlay.has(
467+
"thumbnailOverlayTimeStatusRenderer"))
468+
.map(thumbnailOverlay -> thumbnailOverlay.getObject(
469+
"thumbnailOverlayTimeStatusRenderer"))
470+
.anyMatch(timeOverlay -> timeOverlay.getString("style", "")
471+
.equalsIgnoreCase("SHORTS")
472+
|| timeOverlay.getObject("icon")
473+
.getString("iconType", "")
474+
.toLowerCase()
475+
.contains("shorts"));
473476
}
474477
}
475478

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemLockupExtractor.java‎

Lines changed: 30 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -31,9 +31,9 @@
3131
/**
3232
* Note:
3333
* This extractor is currently (2025-07) only used to extract related video streams.<br>
34-
* The following features are currently not implemented because they have never been observed:
34+
* The following features are currently not implemented:
3535
* <ul>
36-
* <li>Shorts</li>
36+
* <li>Shorts: appear in related videos without a duration badge; getDuration() returns -1</li>
3737
* <li>Paid content (Premium, members first or only)</li>
3838
* </ul>
3939
*/
@@ -77,22 +77,22 @@ public StreamType getStreamType() throws ParsingException {
7777
}
7878

7979
private StreamType determineStreamType() throws ParsingException {
80-
if (JsonUtils.getArray(lockupViewModel, "contentImage.thumbnailViewModel.overlays")
81-
.streamAsJsonObjects()
80+
final JsonArray overlays = JsonUtils.getArray(lockupViewModel,
81+
"contentImage.thumbnailViewModel.overlays");
82+
83+
// thumbnailOverlayBadgeViewModel path (legacy/alternate overlay structure)
84+
if (overlays.streamAsJsonObjects()
8285
.flatMap(overlay -> overlay
8386
.getObject("thumbnailOverlayBadgeViewModel")
8487
.getArray("thumbnailBadges")
8588
.streamAsJsonObjects())
8689
.map(thumbnailBadge -> thumbnailBadge.getObject("thumbnailBadgeViewModel"))
87-
.anyMatch(thumbnailBadgeViewModel -> {
88-
if ("THUMBNAIL_OVERLAY_BADGE_STYLE_LIVE".equals(
89-
thumbnailBadgeViewModel.getString("badgeStyle"))) {
90+
.anyMatch(vm -> {
91+
if ("THUMBNAIL_OVERLAY_BADGE_STYLE_LIVE".equals(vm.getString("badgeStyle"))) {
9092
return true;
9193
}
92-
9394
// Fallback: Check if there is a live icon
94-
return thumbnailBadgeViewModel
95-
.getObject("icon")
95+
return vm.getObject("icon")
9696
.getArray("sources")
9797
.streamAsJsonObjects()
9898
.map(source -> source
@@ -103,6 +103,18 @@ private StreamType determineStreamType() throws ParsingException {
103103
return StreamType.LIVE_STREAM;
104104
}
105105

106+
// thumbnailBottomOverlayViewModel path (used in lockup format for both duration and live)
107+
if (overlays.streamAsJsonObjects()
108+
.flatMap(overlay -> overlay
109+
.getObject("thumbnailBottomOverlayViewModel")
110+
.getArray("badges")
111+
.streamAsJsonObjects())
112+
.map(badge -> badge.getObject("thumbnailBadgeViewModel"))
113+
.anyMatch(vm -> "THUMBNAIL_OVERLAY_BADGE_STYLE_LIVE".equals(
114+
vm.getString("badgeStyle")))) {
115+
return StreamType.LIVE_STREAM;
116+
}
117+
106118
return StreamType.VIDEO_STREAM;
107119
}
108120

@@ -164,18 +176,25 @@ public long getDuration() throws ParsingException {
164176
.collect(Collectors.toList());
165177

166178
if (potentialDurations.isEmpty()) {
167-
throw new ParsingException("Could not get duration: No parsable durations detected");
179+
return -1;
168180
}
169181

170182
ParsingException parsingException = null;
171183
for (final String potentialDuration : potentialDurations) {
184+
if (potentialDuration == null || !potentialDuration.matches(".*\\d.*")) {
185+
continue;
186+
}
172187
try {
173188
return YoutubeParsingHelper.parseDurationString(potentialDuration);
174189
} catch (final ParsingException ex) {
175190
parsingException = ex;
176191
}
177192
}
178193

194+
if (parsingException == null) {
195+
return -1; // e.g. only "SHORTS" or "CC" badge was present, no duration available
196+
}
197+
179198
throw new ParsingException("Could not get duration", parsingException);
180199
}
181200

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamInfoItemTest.java‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,58 @@ void lockupViewModelPremiere()
8686
);
8787
}
8888

89+
@Test
90+
void lockupViewModelVideo()
91+
throws FileNotFoundException, JsonParserException {
92+
final var json = JsonParser.object().from(new FileInputStream(getMockPath(
93+
YoutubeStreamInfoItemTest.class, "lockupViewModelVideo") + ".json"));
94+
final var timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(Localization.DEFAULT);
95+
final var extractor = new YoutubeStreamInfoItemLockupExtractor(json, timeAgoParser);
96+
assertAll(
97+
() -> assertEquals(StreamType.VIDEO_STREAM, extractor.getStreamType()),
98+
() -> assertFalse(extractor.isAd()),
99+
() -> assertEquals("https://www.youtube.com/watch?v=dQw4w9WgXcQ", extractor.getUrl()),
100+
() -> assertEquals("VIDEO_TITLE", extractor.getName()),
101+
() -> assertEquals(974, extractor.getDuration()),
102+
() -> assertFalse(extractor.getThumbnails().isEmpty())
103+
);
104+
}
105+
106+
@Test
107+
void lockupViewModelLiveStream()
108+
throws FileNotFoundException, JsonParserException {
109+
final var json = JsonParser.object().from(new FileInputStream(getMockPath(
110+
YoutubeStreamInfoItemTest.class, "lockupViewModelLiveStream") + ".json"));
111+
final var timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(Localization.DEFAULT);
112+
final var extractor = new YoutubeStreamInfoItemLockupExtractor(json, timeAgoParser);
113+
assertAll(
114+
() -> assertEquals(StreamType.LIVE_STREAM, extractor.getStreamType()),
115+
() -> assertFalse(extractor.isAd()),
116+
() -> assertEquals("https://www.youtube.com/watch?v=LIVE_VIDEO_ID", extractor.getUrl()),
117+
() -> assertEquals("LIVE_VIDEO_TITLE", extractor.getName()),
118+
() -> assertEquals(-1, extractor.getDuration()),
119+
() -> assertNull(extractor.getTextualUploadDate()),
120+
() -> assertNull(extractor.getUploadDate()),
121+
() -> assertEquals(0, extractor.getViewCount()),
122+
() -> assertFalse(extractor.getThumbnails().isEmpty())
123+
);
124+
}
125+
126+
@Test
127+
void lockupViewModelNoDuration()
128+
throws FileNotFoundException, JsonParserException {
129+
final var json = JsonParser.object().from(new FileInputStream(getMockPath(
130+
YoutubeStreamInfoItemTest.class, "lockupViewModelNoDuration") + ".json"));
131+
final var timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(Localization.DEFAULT);
132+
final var extractor = new YoutubeStreamInfoItemLockupExtractor(json, timeAgoParser);
133+
assertAll(
134+
() -> assertEquals(StreamType.VIDEO_STREAM, extractor.getStreamType()),
135+
() -> assertFalse(extractor.isAd()),
136+
() -> assertEquals(-1, extractor.getDuration()),
137+
() -> assertFalse(extractor.getThumbnails().isEmpty())
138+
);
139+
}
140+
89141
@Test
90142
void emptyTitle() throws FileNotFoundException, JsonParserException {
91143
final var json = JsonParser.object().from(new FileInputStream(getMockPath(

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -235,7 +235,8 @@ protected SearchExtractor createExtractor() throws Exception {
235235
Collections.singletonList("Learn more")
236236
));
237237
}
238-
// testMoreRelatedItems is broken because a video has no duration shown
238+
// testMoreRelatedItems: a video in this mock has no duration badge; getDuration() now
239+
// returns -1 instead of throwing, but the mock may need re-recording before re-enabling.
239240
@Test @Override public void testMoreRelatedItems() { }
240241
@Override public StreamingService expectedService() { return YouTube; }
241242
@Override public String expectedName() { return QUERY; }

‎extractor/src/test/resources/mocks/v1/org/schabi/newpipe/extractor/services/youtube/youtubestreaminfoitem/lockupviewmodellivestream.json‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
{
2+
"contentImage": {
3+
"thumbnailViewModel": {
4+
"image": {
5+
"sources": [
6+
{
7+
"url": "https://i.ytimg.com/vi/LIVE_VIDEO_ID/hqdefault.jpg",
8+
"width": 168,
9+
"height": 94
10+
},
11+
{
12+
"url": "https://i.ytimg.com/vi/LIVE_VIDEO_ID/hqdefault.jpg",
13+
"width": 336,
14+
"height": 188
15+
}
16+
]
17+
},
18+
"overlays": [
19+
{
20+
"thumbnailBottomOverlayViewModel": {
21+
"badges": [
22+
{
23+
"thumbnailBadgeViewModel": {
24+
"badgeStyle": "THUMBNAIL_OVERLAY_BADGE_STYLE_LIVE",
25+
"text": "LIVE"
26+
}
27+
}
28+
]
29+
}
30+
},
31+
{
32+
"thumbnailHoverOverlayToggleActionsViewModel": {
33+
"buttons": []
34+
}
35+
}
36+
]
37+
}
38+
},
39+
"metadata": {
40+
"lockupMetadataViewModel": {
41+
"title": {
42+
"content": "LIVE_VIDEO_TITLE"
43+
},
44+
"image": {
45+
"decoratedAvatarViewModel": {
46+
"avatar": {
47+
"avatarViewModel": {
48+
"image": {
49+
"sources": [
50+
{
51+
"url": "https://yt3.ggpht.com/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==",
52+
"width": 68,
53+
"height": 68
54+
}
55+
]
56+
},
57+
"avatarImageSize": "AVATAR_SIZE_M"
58+
}
59+
},
60+
"a11yLabel": "Go to channel",
61+
"rendererContext": {
62+
"commandContext": {
63+
"onTap": {
64+
"innertubeCommand": {
65+
"commandMetadata": {
66+
"webCommandMetadata": {
67+
"url": "/@LIVE_CHANNEL_HANDLE"
68+
}
69+
},
70+
"browseEndpoint": {
71+
"browseId": "UCD_on7-zu7Zuc3zissQvrgw",
72+
"canonicalBaseUrl": "/@LIVE_CHANNEL_HANDLE"
73+
}
74+
}
75+
}
76+
}
77+
}
78+
}
79+
},
80+
"metadata": {
81+
"contentMetadataViewModel": {
82+
"metadataRows": [
83+
{
84+
"metadataParts": [
85+
{
86+
"text": {
87+
"content": "LIVE_CHANNEL_NAME",
88+
"styleRuns": [],
89+
"attachmentRuns": []
90+
}
91+
}
92+
]
93+
}
94+
],
95+
"delimiter": " • "
96+
}
97+
},
98+
"menuButton": {}
99+
}
100+
},
101+
"contentId": "LIVE_VIDEO_ID",
102+
"contentType": "LOCKUP_CONTENT_TYPE_VIDEO"
103+
}

0 commit comments

Comments
 (0)