Skip to content

Commit 251f7f6

Browse files
StypoxStypox
authored
Merge pull request #1443 from Stypox/fix-tests
2 parents ac654a7 + 4a729a7 commit 251f7f6

55 files changed

Lines changed: 3856 additions & 2253 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeDescriptionHelper.java‎

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,16 @@
11
package org.schabi.newpipe.extractor.services.youtube;
22

33
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
4+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isYoutubeServiceURL;
5+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isYoutubeURL;
46
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
57

68
import com.grack.nanojson.JsonObject;
79

810
import org.jsoup.nodes.Entities;
911

12+
import java.net.MalformedURLException;
13+
import java.net.URL;
1014
import java.util.ArrayList;
1115
import java.util.Collections;
1216
import java.util.Comparator;
@@ -242,23 +246,35 @@ private static void addAllCommandRuns(
242246
return;
243247
}
244248

249+
boolean isYoutubeUrl;
250+
try {
251+
final URL parsedUrl = new URL(url);
252+
isYoutubeUrl = isYoutubeURL(parsedUrl) || isYoutubeServiceURL(parsedUrl);
253+
} catch (final MalformedURLException ignored) {
254+
// this should never happen, but just in case, assume this is a generic URL
255+
isYoutubeUrl = false;
256+
}
257+
245258
final String open = "<a href=\"" + Entities.escape(url) + "\">";
246-
final Function<String, String> transformContent = getTransformContentFun(run);
259+
final Function<String, String> transformContent = getTransformContentFun(
260+
run, isYoutubeUrl);
247261

248262
openers.add(new Run(open, LINK_CLOSE, startIndex, transformContent));
249263
closers.add(new Run(open, LINK_CLOSE, startIndex + length, transformContent));
250264
});
251265
}
252266

253-
private static Function<String, String> getTransformContentFun(final JsonObject run) {
267+
private static Function<String, String> getTransformContentFun(final JsonObject run,
268+
final boolean isYoutube) {
254269
final String accessibilityLabel = run.getObject("onTapOptions")
255270
.getObject("accessibilityInfo")
256271
.getString("accessibilityLabel", "")
257272
// accessibility labels are e.g. "Instagram Channel Link: instagram_profile_name"
258273
.replaceFirst(" Channel Link", "");
259274

260275
final Function<String, String> transformContent;
261-
if (accessibilityLabel.isEmpty() || accessibilityLabel.startsWith("YouTube: ")) {
276+
if (isYoutube
277+
|| accessibilityLabel.isEmpty() || accessibilityLabel.startsWith("YouTube: ")) {
262278
// if there is no accessibility label, or the link points to YouTube, cleanup the link
263279
// text, see LINK_CONTENT_CLEANER_REGEX's documentation for more details
264280
transformContent = (content) -> {

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSignatureUtils.java‎

Lines changed: 25 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,16 @@
11
package org.schabi.newpipe.extractor.services.youtube;
22

3-
import static org.schabi.newpipe.extractor.utils.Parser.matchGroup1MultiplePatterns;
3+
import static org.schabi.newpipe.extractor.utils.Parser.matchMultiplePatterns;
44

55
import org.schabi.newpipe.extractor.exceptions.ParsingException;
66
import org.schabi.newpipe.extractor.utils.JavaScript;
7+
import org.schabi.newpipe.extractor.utils.Pair;
78
import org.schabi.newpipe.extractor.utils.Parser;
89
import org.schabi.newpipe.extractor.utils.jsextractor.JavaScriptExtractor;
910

1011
import javax.annotation.Nonnull;
12+
13+
import java.util.regex.Matcher;
1114
import java.util.regex.Pattern;
1215

1316
/**
@@ -24,6 +27,7 @@ final class YoutubeSignatureUtils {
2427

2528
private static final Pattern[] FUNCTION_REGEXES = {
2629
// CHECKSTYLE:OFF
30+
Pattern.compile("\\b(?:[a-zA-Z0-9_$]+)&&\\((?:[a-zA-Z0-9_$]+)=([a-zA-Z0-9_$]{2,})\\((\\d+,)decodeURIComponent\\((?:[a-zA-Z0-9_$]+)\\)\\)"),
2731
Pattern.compile("\\b(?:[a-zA-Z0-9_$]+)&&\\((?:[a-zA-Z0-9_$]+)=([a-zA-Z0-9_$]{2,})\\(decodeURIComponent\\((?:[a-zA-Z0-9_$]+)\\)\\)"),
2832
Pattern.compile("\\bm=([a-zA-Z0-9$]{2,})\\(decodeURIComponent\\(h\\.s\\)\\)"),
2933
Pattern.compile("\\bc&&\\(c=([a-zA-Z0-9$]{2,})\\(decodeURIComponent\\(c\\)\\)"),
@@ -38,8 +42,10 @@ final class YoutubeSignatureUtils {
3842
private static final String DEOBF_FUNC_REGEX_END = "=function\\([a-zA-Z0-9_]+\\)\\{.+?\\})";
3943

4044
// CHECKSTYLE:OFF
41-
private static final String SIG_DEOBF_GLOBAL_ARRAY_REGEX = "(var [A-z]=['\"].*['\"].split\\(\";\"\\))";
42-
private static final String SIG_DEOBF_HELPER_OBJ_NAME_REGEX = ";([A-Za-z0-9_\\$]{2,})\\[..";
45+
private static final Pattern SIG_DEOBF_GLOBAL_ARRAY_REGEX =
46+
Pattern.compile("(var [A-z]=['\"].*['\"].split\\(\"[;{]\"\\))");
47+
private static final Pattern SIG_DEOBF_HELPER_OBJ_NAME_REGEX =
48+
Pattern.compile("[;,]([A-Za-z0-9_$]{2,})\\[..");
4349
private static final String SIG_DEOBF_HELPER_OBJ_REGEX_START = "(var ";
4450
private static final String SIG_DEOBF_HELPER_OBJ_REGEX_END = "=\\{(?>.|\\n)+?\\}\\};)";
4551
// CHECKSTYLE:ON
@@ -76,8 +82,10 @@ static String getSignatureTimestamp(@Nonnull final String javaScriptPlayerCode)
7682
static String getDeobfuscationCode(@Nonnull final String javaScriptPlayerCode)
7783
throws ParsingException {
7884
try {
79-
final String deobfuscationFunctionName = getDeobfuscationFunctionName(
80-
javaScriptPlayerCode);
85+
final Pair<String, String> deobfuscationFunctionNameAndParams =
86+
getDeobfuscationFunctionNameAndParams(javaScriptPlayerCode);
87+
final String deobfuscationFunctionName = deobfuscationFunctionNameAndParams.getFirst();
88+
final String functionAdditionalParams = deobfuscationFunctionNameAndParams.getSecond();
8189

8290
String deobfuscationFunction;
8391
try {
@@ -102,7 +110,7 @@ static String getDeobfuscationCode(@Nonnull final String javaScriptPlayerCode)
102110
final String callerFunction = "function " + DEOBFUSCATION_FUNCTION_NAME
103111
+ "(a){return "
104112
+ deobfuscationFunctionName
105-
+ "(a);}";
113+
+ "(" + functionAdditionalParams + "a);}";
106114

107115
return globalVar + ";" + helperObject + deobfuscationFunction + ";" + callerFunction;
108116
} catch (final Exception e) {
@@ -111,10 +119,18 @@ static String getDeobfuscationCode(@Nonnull final String javaScriptPlayerCode)
111119
}
112120

113121
@Nonnull
114-
private static String getDeobfuscationFunctionName(@Nonnull final String javaScriptPlayerCode)
115-
throws ParsingException {
122+
private static Pair<String, String> getDeobfuscationFunctionNameAndParams(
123+
@Nonnull final String javaScriptPlayerCode) throws ParsingException {
116124
try {
117-
return matchGroup1MultiplePatterns(FUNCTION_REGEXES, javaScriptPlayerCode);
125+
final Matcher m = matchMultiplePatterns(FUNCTION_REGEXES, javaScriptPlayerCode);
126+
final String functionName = m.group(1);
127+
final String functionAdditionalParams;
128+
if (m.groupCount() > 1) {
129+
functionAdditionalParams = m.group(2);
130+
} else {
131+
functionAdditionalParams = "";
132+
}
133+
return new Pair<>(functionName, functionAdditionalParams);
118134
} catch (final Parser.RegexException e) {
119135
throw new ParsingException(
120136
"Could not find deobfuscation function with any of the known patterns", e);

‎extractor/src/test/java/org/schabi/newpipe/downloader/DownloaderFactory.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package org.schabi.newpipe.downloader;
22

33
import org.schabi.newpipe.extractor.downloader.Downloader;
4+
import org.schabi.newpipe.extractor.utils.Utils;
45

56
import java.util.Locale;
67

@@ -23,7 +24,7 @@ private static DownloaderType getDownloaderType() {
2324

2425
private static DownloaderType determineDownloaderType() {
2526
String propValue = System.getProperty("downloader");
26-
if (propValue == null) {
27+
if (Utils.isNullOrEmpty(propValue)) {
2728
return DEFAULT_DOWNLOADER;
2829
}
2930
propValue = propValue.toUpperCase();
@@ -33,8 +34,8 @@ private static DownloaderType determineDownloaderType() {
3334
}
3435
try {
3536
return DownloaderType.valueOf(propValue);
36-
} catch (final Exception e) {
37-
return DEFAULT_DOWNLOADER;
37+
} catch (final IllegalArgumentException e) {
38+
throw new IllegalArgumentException("Unknown downloader name: " + propValue, e);
3839
}
3940
}
4041

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java‎

Lines changed: 5 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,10 @@
3535
*/
3636
public class YoutubeChannelExtractorTest {
3737

38+
/**
39+
* See <a href="https://youtube.fandom.com/wiki/Termination#Ban_Messages">here</a>
40+
* for a list of account termination messages.
41+
*/
3842
public static class NotAvailable implements InitYoutubeTest {
3943
@Test
4044
void deletedFetch() throws Exception {
@@ -67,7 +71,7 @@ void accountTerminatedTOSFetch() throws Exception {
6771
void accountTerminatedCommunityFetch() throws Exception {
6872
// "This account has been terminated for violating YouTube's Community Guidelines."
6973
final ChannelExtractor extractor =
70-
YouTube.getChannelExtractor("https://www.youtube.com/channel/UC0AuOxCr9TZ0TtEgL1zpIgA");
74+
YouTube.getChannelExtractor("https://www.youtube.com/channel/UC-nQp2ewj2Yeg5w7VyoVBwQ");
7175

7276
final AccountTerminatedException ex =
7377
assertThrows(AccountTerminatedException.class, extractor::fetchPage);
@@ -86,18 +90,6 @@ void accountTerminatedHateFetch() throws Exception {
8690
assertEquals(AccountTerminatedException.Reason.VIOLATION, ex.getReason());
8791
}
8892

89-
@Test
90-
void accountTerminatedBullyFetch() throws Exception {
91-
// "This account has been terminated due to multiple or severe violations
92-
// of YouTube's policy prohibiting content designed to harass, bully or threaten."
93-
final ChannelExtractor extractor =
94-
YouTube.getChannelExtractor("https://youtube.com/channel/UCB1o7_gbFp2PLsamWxFenBg");
95-
96-
final AccountTerminatedException ex =
97-
assertThrows(AccountTerminatedException.class, extractor::fetchPage);
98-
assertEquals(AccountTerminatedException.Reason.VIOLATION, ex.getReason());
99-
}
100-
10193
@Test
10294
void accountTerminatedSpamFetch() throws Exception {
10395
// "This account has been terminated due to multiple or severe violations

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -162,7 +162,7 @@ void testGetCommentsAllData() throws IOException, ExtractionException {
162162
}
163163

164164
public static class HeartedByCreator extends Base {
165-
private final static String URL = "https://www.youtube.com/watch?v=tR11b7uh17Y";
165+
private final static String URL = "https://www.youtube.com/watch?v=RwTdoQNVMTY";
166166

167167
@Override
168168
protected String extractorUrl() {

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeKioskExtractorTest.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems;
88

99
import org.junit.jupiter.api.Test;
10+
import org.junit.jupiter.api.Disabled;
1011
import org.schabi.newpipe.extractor.exceptions.ParsingException;
1112
import org.schabi.newpipe.extractor.services.BaseListExtractorTest;
1213
import org.schabi.newpipe.extractor.services.DefaultSimpleExtractorTest;
@@ -295,6 +296,7 @@ public void testMoreRelatedItems() throws Exception {
295296
}
296297

297298
// Deprecated (i.e. removed from the interface of YouTube) since July 21, 2025
299+
@Disabled("Trending section was removed from YouTube")
298300
public static class Trending extends DefaultSimpleExtractorTest<YoutubeTrendingExtractor>
299301
implements BaseListExtractorTest, InitYoutubeTest {
300302

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -306,7 +306,7 @@ void getPlaylistType() throws ParsingException {
306306
}
307307

308308
public static class InvalidPageEmpty extends Base {
309-
private static final String VIDEO_ID = "QMVCAPd5cwBcg";
309+
private static final String VIDEO_ID = "WRz2MxhAdJo";
310310

311311
@Override
312312
protected String extractorUrl() {

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingKioskInfoTest.java‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,14 +20,15 @@
2020
* along with NewPipe Extractor. If not, see <http://www.gnu.org/licenses/>.
2121
*/
2222

23+
import static org.junit.jupiter.api.Assertions.assertEquals;
2324
import static org.junit.jupiter.api.Assertions.assertFalse;
24-
import static org.junit.jupiter.api.Assertions.assertTrue;
2525
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
2626

2727
import org.junit.jupiter.api.BeforeAll;
2828
import org.junit.jupiter.api.Test;
2929
import org.schabi.newpipe.extractor.kiosk.KioskInfo;
3030
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
31+
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeLiveLinkHandlerFactory;
3132

3233
/**
3334
* Test for {@link KioskInfo}
@@ -41,9 +42,11 @@ class YoutubeTrendingKioskInfoTest implements InitYoutubeTest {
4142
public void setUp() throws Exception {
4243
InitYoutubeTest.super.setUp();
4344

44-
final LinkHandlerFactory linkHandlerFactory = YouTube.getKioskList().getListLinkHandlerFactoryByType("Trending");
45+
final LinkHandlerFactory linkHandlerFactory = YouTube.getKioskList()
46+
.getListLinkHandlerFactoryByType(YoutubeLiveLinkHandlerFactory.KIOSK_ID);
4547

46-
kioskInfo = KioskInfo.getInfo(YouTube, linkHandlerFactory.fromId("Trending").getUrl());
48+
kioskInfo = KioskInfo.getInfo(YouTube,
49+
linkHandlerFactory.fromId(YoutubeLiveLinkHandlerFactory.KIOSK_ID).getUrl());
4750
}
4851

4952
@Test
@@ -53,8 +56,7 @@ void getStreams() {
5356

5457
@Test
5558
void getId() {
56-
assertTrue(kioskInfo.getId().equals("Trending")
57-
|| kioskInfo.getId().equals("Trends"));
59+
assertEquals(YoutubeLiveLinkHandlerFactory.KIOSK_ID, kioskInfo.getId());
5860
}
5961

6062
@Test

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,9 +57,9 @@ protected SearchExtractor createExtractor() throws Exception {
5757
}
5858

5959
public static class MusicAlbums extends DefaultSearchExtractorTest implements InitYoutubeTest {
60-
// searching for "scenography" on 28/07/2025 returns some autogenerated albums,
60+
// searching for "axwell" on 27/01/2026 returns some autogenerated albums,
6161
// and we want to test the extraction of those, too
62-
private static final String QUERY = "scenography";
62+
private static final String QUERY = "axwell";
6363

6464
@Override
6565
protected SearchExtractor createExtractor() throws Exception {

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ void nonExistentFetch() throws Exception {
8686

8787
final StreamExtractor extractor =
8888
YouTube.getStreamExtractor(BASE_URL + "don-t-exist");
89-
assertThrows(ContentNotAvailableException.class, extractor::fetchPage);
89+
assertThrows(ParsingException.class, extractor::fetchPage);
9090
}
9191

9292
@Test
@@ -280,7 +280,6 @@ protected StreamExtractor createExtractor() throws Exception {
280280
@Nullable @Override public String expectedTextualUploadDate() { return "2021-03-17T12:56:59-07:00"; }
281281
@Override public long expectedLikeCountAtLeast() { return 2300; }
282282
@Override public long expectedDislikeCountAtLeast() { return -1; }
283-
@Override public boolean expectedHasSubtitles() { return false; }
284283
@Override public int expectedStreamSegmentsCount() { return 13; }
285284
@Override public String expectedLicence() { return YOUTUBE_LICENCE; }
286285
@Override public String expectedCategory() { return "News & Politics"; }

0 commit comments

Comments
 (0)