Skip to content

Commit 3af7326

Browse files
TobiGrTobiGr
authored
Merge pull request #1387 from that1potato/issue#12251
Fix SoundCloud channel tab pagination looping on empty pages
2 parents 8e90305 + 702f1b0 commit 3af7326

18 files changed

Lines changed: 1126 additions & 3 deletions

File tree

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelTabExtractor.java‎

Lines changed: 39 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@
1414

1515
import javax.annotation.Nonnull;
1616
import java.io.IOException;
17+
import java.util.HashSet;
18+
import java.util.Set;
1719

1820
import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL;
1921
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
@@ -22,6 +24,9 @@ public class SoundcloudChannelTabExtractor extends ChannelTabExtractor {
2224

2325
private static final String USERS_ENDPOINT = SOUNDCLOUD_API_V2_URL + "users/";
2426

27+
/** Empty page cap against infinite pagination loops. */
28+
private static final int MAX_EMPTY_PAGES = 3;
29+
2530
private final String userId;
2631

2732
public SoundcloudChannelTabExtractor(final StreamingService service,
@@ -70,9 +75,40 @@ public InfoItemsPage<InfoItem> getPage(final Page page)
7075
}
7176

7277
final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId());
73-
final String nextPageUrl = SoundcloudParsingHelper.getInfoItemsFromApi(
74-
collector, page.getUrl());
78+
final Set<String> visitedPages = new HashSet<>();
79+
80+
String currentPageUrl = page.getUrl();
81+
String nextPageUrl = "";
82+
int emptyPageCount = 0;
83+
84+
while (!isNullOrEmpty(currentPageUrl)) {
85+
if (!visitedPages.add(currentPageUrl)) {
86+
// Prevent infinite loops when the API points back to an already visited page.
87+
nextPageUrl = "";
88+
break;
89+
}
90+
91+
final int itemsBefore = collector.getItems().size();
92+
final String candidateNextPage = SoundcloudParsingHelper
93+
.getInfoItemsFromApi(collector, currentPageUrl);
94+
final boolean hasNewItems = collector.getItems().size() > itemsBefore;
95+
96+
if (hasNewItems) {
97+
nextPageUrl = candidateNextPage;
98+
break;
99+
}
100+
101+
emptyPageCount++;
102+
if (emptyPageCount >= MAX_EMPTY_PAGES || isNullOrEmpty(candidateNextPage)) {
103+
// Give up after too many empty responses or when SoundCloud stops providing tokens.
104+
nextPageUrl = "";
105+
break;
106+
}
107+
108+
currentPageUrl = candidateNextPage;
109+
}
75110

76-
return new InfoItemsPage<>(collector, new Page(nextPageUrl));
111+
final Page nextPage = isNullOrEmpty(nextPageUrl) ? null : new Page(nextPageUrl);
112+
return new InfoItemsPage<>(collector, nextPage);
77113
}
78114
}

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelTabExtractorTest.java‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,21 @@
11
package org.schabi.newpipe.extractor.services.soundcloud;
22

3+
import static org.junit.jupiter.api.Assertions.assertFalse;
4+
import static org.junit.jupiter.api.Assertions.assertTrue;
5+
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmptyErrors;
36
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
47
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestGetPageInNewExtractor;
58

69
import org.junit.jupiter.api.Test;
710
import org.schabi.newpipe.extractor.InfoItem;
11+
import org.schabi.newpipe.extractor.ListExtractor;
812
import org.schabi.newpipe.extractor.StreamingService;
913
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabExtractor;
1014
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
1115
import org.schabi.newpipe.extractor.services.DefaultListExtractorTest;
1216

17+
import java.util.List;
18+
1319
class SoundcloudChannelTabExtractorTest {
1420

1521
static class Tracks extends DefaultListExtractorTest<ChannelTabExtractor> {
@@ -88,4 +94,30 @@ protected ChannelTabExtractor createExtractor() throws Exception {
8894
@Override public String expectedOriginalUrlContains() throws Exception { return "https://soundcloud.com/soreen-735855039/likes"; }
8995
@Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.PLAYLIST; }
9096
}
97+
98+
static class NoEmptyNextPages extends DefaultListExtractorTest<ChannelTabExtractor> {
99+
@Override
100+
protected ChannelTabExtractor createExtractor() throws Exception {
101+
return SoundCloud.getChannelTabExtractorFromId("73637815", ChannelTabs.TRACKS);
102+
}
103+
104+
@Override public StreamingService expectedService() throws Exception { return SoundCloud; }
105+
@Override public String expectedName() throws Exception { return ChannelTabs.TRACKS; }
106+
@Override public String expectedId() throws Exception { return "73637815"; }
107+
@Override public String expectedUrlContains() throws Exception { return "https://soundcloud.com/hurtbox/tracks"; }
108+
@Override public String expectedOriginalUrlContains() throws Exception { return "https://soundcloud.com/hurtbox/tracks"; }
109+
@Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; }
110+
111+
@Test
112+
public void testNextPages() throws Exception {
113+
ListExtractor.InfoItemsPage<InfoItem> page = extractor().getInitialPage();
114+
for (int i = 1; i <= 5; i++) {
115+
assertEmptyErrors("Next page has errors", page.getErrors());
116+
assertFalse(page.getItems().isEmpty(), "Next page is empty");
117+
assertTrue(page.hasNextPage(), "Next page does not have more items");
118+
page = extractor().getPage(page.getNextPage());
119+
}
120+
}
121+
122+
}
91123
}

‎extractor/src/test/resources/mocks/v1/org/schabi/newpipe/extractor/services/soundcloud/soundcloudchanneltabextractor/noemptynextpages/generated_mock_0.json‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
{
2+
"request": {
3+
"httpMethod": "GET",
4+
"url": "https://w.soundcloud.com/player/?url\u003dhttps%3A%2F%2Fapi.soundcloud.com%2Fusers%2F73637815",
5+
"headers": {
6+
"Accept-Language": [
7+
"en-GB, en;q\u003d0.9"
8+
]
9+
},
10+
"localization": {
11+
"languageCode": "en",
12+
"countryCode": "GB"
13+
}
14+
},
15+
"response": {
16+
"responseCode": 200,
17+
"responseMessage": "",
18+
"responseHeaders": {
19+
"age": [
20+
"41"
21+
],
22+
"cache-control": [
23+
"public, max-age\u003d300"
24+
],
25+
"content-type": [
26+
"text/html"
27+
],
28+
"date": [
29+
"Sat, 11 Oct 2025 07:18:56 GMT"
30+
],
31+
"p3p": [
32+
"policyref\u003d\"https://w.soundcloud.com/player/w3c/p3p.xml\", CP\u003d\"NON DSP COR CUR ADM DEV TAI PSAo PSDo OUR STP CNT\""
33+
],
34+
"server": [
35+
"am/2"
36+
],
37+
"strict-transport-security": [
38+
"max-age\u003d63072000"
39+
],
40+
"vary": [
41+
"Accept-Encoding"
42+
],
43+
"via": [
44+
"sssr, 1.1 446f4800ed8a030e10e4a6dd15349224.cloudfront.net (CloudFront)"
45+
],
46+
"x-amz-cf-id": [
47+
"6Yw0AsuW3SIXRNaT7KyQD0Rm1Mtomsc3UiS4gbjS0hGNEDTdE1RW7A\u003d\u003d"
48+
],
49+
"x-amz-cf-pop": [
50+
"TXL50-P5"
51+
],
52+
"x-cache": [
53+
"Hit from cloudfront"
54+
]
55+
},
56+
"responseBody": "\u003c!DOCTYPE html\u003e\n\u003chtml lang\u003d\"en\"\u003e\n\u003chead\u003e\n \u003cmeta charset\u003d\"UTF-8\"\u003e\n\n \n \u003clink rel\u003d\"dns-prefetch\" href\u003d\"//api-widget.soundcloud.com\"\u003e\n \u003clink rel\u003d\"dns-prefetch\" href\u003d\"//sb.scorecardresearch.com\"\u003e\n \u003clink rel\u003d\"dns-prefetch\" href\u003d\"//api.soundcloud.com\"\u003e\n \u003clink rel\u003d\"dns-prefetch\" href\u003d\"//va.sndcdn.com\"\u003e\n \u003clink rel\u003d\"dns-prefetch\" href\u003d\"//i1.sndcdn.com\"\u003e\n \u003clink rel\u003d\"dns-prefetch\" href\u003d\"//i2.sndcdn.com\"\u003e\n \u003clink rel\u003d\"dns-prefetch\" href\u003d\"//i3.sndcdn.com\"\u003e\n \u003clink rel\u003d\"dns-prefetch\" href\u003d\"//i4.sndcdn.com\"\u003e\n \u003clink rel\u003d\"dns-prefetch\" href\u003d\"//w1.sndcdn.com\"\u003e\n \u003clink rel\u003d\"dns-prefetch\" href\u003d\"//wis.sndcdn.com\"\u003e\n \u003clink rel\u003d\"dns-prefetch\" href\u003d\"//style.sndcdn.com\"\u003e\n \n \u003cmeta name\u003d\"robots\" content\u003d\"noindex, follow\"\u003e\n \u003cmeta http-equiv\u003d\"X-UA-Compatible\" content\u003d\"IE\u003dedge,chrome\u003d1\"\u003e\n \u003cmeta name\u003d\"viewport\" content\u003d\"width\u003ddevice-width,minimum-scale\u003d1,maximum-scale\u003d1,user-scalable\u003dno\"\u003e\n \u003clink rel\u003d\"canonical\" href\u003d\"https://soundcloud.com/hurtbox\"\u003e\n\n \u003ctitle\u003eSoundCloud Widget\u003c/title\u003e\n\n \n \u003cbase target\u003d\"_blank\"\u003e\n \n\u003c/head\u003e\n\u003cbody style\u003d\"margin:0\"\u003e\n \u003cdiv id\u003d\"widget\" class\u003d\"widget g-background-default g-shadow-inset\" style\u003d\"height:100%\"\u003e\u003c/div\u003e\n \u003cscript src\u003d\"https://widget.sndcdn.com/widget-4-434ca88dccea.js\" crossorigin\u003e\u003c/script\u003e\n \n \u003cscript crossorigin src\u003d\"https://widget.sndcdn.com/widget-8-8102dadd8b2e.js\"\u003e\u003c/script\u003e\n \n \u003cscript crossorigin src\u003d\"https://widget.sndcdn.com/widget-9-0111f457ff60.js\"\u003e\u003c/script\u003e\n \n \n \u003cscript\u003ewindow.innerHeight\u003c\u003d20\u0026\u0026document.body.classList.add(\"mini-player\")\u003c/script\u003e\n\u003c/body\u003e\n\u003c/html\u003e\n",
57+
"latestUrl": "https://w.soundcloud.com/player/?url\u003dhttps%3A%2F%2Fapi.soundcloud.com%2Fusers%2F73637815"
58+
}
59+
}

‎extractor/src/test/resources/mocks/v1/org/schabi/newpipe/extractor/services/soundcloud/soundcloudchanneltabextractor/noemptynextpages/generated_mock_1.json‎

Lines changed: 68 additions & 0 deletions
Large diffs are not rendered by default.

‎extractor/src/test/resources/mocks/v1/org/schabi/newpipe/extractor/services/soundcloud/soundcloudchanneltabextractor/noemptynextpages/generated_mock_10.json‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
{
2+
"request": {
3+
"httpMethod": "GET",
4+
"url": "https://api-v2.soundcloud.com/users/73637815/tracks?offset\u003d2025-06-06T13%3A44%3A21.000Z%2Ctracks%2C00000000002108853114\u0026limit\u003d20\u0026client_id\u003dMoLbAg35TuqjYwWVtNIKyRPFScQGMOBY",
5+
"headers": {
6+
"Accept-Language": [
7+
"en-GB, en;q\u003d0.9"
8+
]
9+
},
10+
"localization": {
11+
"languageCode": "en",
12+
"countryCode": "GB"
13+
}
14+
},
15+
"response": {
16+
"responseCode": 200,
17+
"responseMessage": "",
18+
"responseHeaders": {
19+
"cache-control": [
20+
"private, max-age\u003d0"
21+
],
22+
"content-type": [
23+
"application/json; charset\u003dutf-8"
24+
],
25+
"date": [
26+
"Sat, 11 Oct 2025 07:19:41 GMT"
27+
],
28+
"referrer-policy": [
29+
"no-referrer"
30+
],
31+
"server": [
32+
"am/2"
33+
],
34+
"strict-transport-security": [
35+
"max-age\u003d63072000"
36+
],
37+
"vary": [
38+
"Origin"
39+
],
40+
"via": [
41+
"1.1 83dede49b3c78b7f99a0399a34264ed6.cloudfront.net (CloudFront)"
42+
],
43+
"x-amz-cf-id": [
44+
"pyfR0l0XQeLpb9zKY_oTxzNyHqBqQwW3DUlsakMBZ_OJXpXCtMuQ2g\u003d\u003d"
45+
],
46+
"x-amz-cf-pop": [
47+
"TXL52-P2"
48+
],
49+
"x-cache": [
50+
"Miss from cloudfront"
51+
],
52+
"x-content-type-options": [
53+
"nosniff"
54+
],
55+
"x-frame-options": [
56+
"DENY"
57+
],
58+
"x-robots-tag": [
59+
"noindex"
60+
]
61+
},
62+
"responseBody": "{\"collection\":[{\"artwork_url\":\"https://i1.sndcdn.com/artworks-6coMhjFIhwNmQn3U-0Ls5Ig-large.jpg\",\"caption\":null,\"commentable\":true,\"comment_count\":137,\"created_at\":\"2025-05-16T04:00:56Z\",\"description\":\"HURTBOX - FUCK LOVE (ft. WATSON)\\n\\nBUY / STREAM: https://createmusic.fm/fklove\\n\\nCONNECT WITH HURTBOX\\nLINKTR.EE/HURTBOX\\nWWW.INSTAGRAM.COM/HURTBOXOFFICIAL\\nWWW.TWITTER.COM/HURTBOXOFFICIAL\\n\\nCONNECT WITH @THEONLYWATSON\\nwww.instagram.com/theonlywatson\\nwww.x.com/theonlywatson\",\"downloadable\":false,\"download_count\":0,\"duration\":184966,\"full_duration\":184947,\"embeddable_by\":\"all\",\"genre\":\"Dubstep\",\"has_downloads_left\":false,\"id\":2096736162,\"kind\":\"track\",\"label_name\":\"\",\"last_modified\":\"2025-05-16T10:08:07Z\",\"license\":\"all-rights-reserved\",\"likes_count\":2795,\"permalink\":\"fucklove\",\"permalink_url\":\"https://soundcloud.com/hurtbox/fucklove\",\"playback_count\":46975,\"public\":true,\"publisher_metadata\":{\"id\":2096736162,\"urn\":\"soundcloud:tracks:2096736162\",\"artist\":\"hurtbox\",\"contains_music\":true,\"isrc\":\"QZTGW2404395\",\"explicit\":false},\"purchase_title\":\"Buy / Stream\",\"purchase_url\":\"https://createmusic.fm/fklove\",\"release_date\":\"2025-05-15T00:00:00Z\",\"reposts_count\":189,\"secret_token\":null,\"sharing\":\"public\",\"state\":\"finished\",\"streamable\":true,\"tag_list\":\"\",\"title\":\"HURTBOX - FUCK LOVE (ft. WATSON)\",\"uri\":\"https://api.soundcloud.com/tracks/soundcloud%3Atracks%3A2096736162\",\"urn\":\"soundcloud:tracks:2096736162\",\"user_id\":73637815,\"visuals\":null,\"waveform_url\":\"https://wave.sndcdn.com/pcfjNOZBWkai_m.json\",\"display_date\":\"2025-05-16T04:00:56Z\",\"media\":{\"transcodings\":[{\"url\":\"https://api-v2.soundcloud.com/media/soundcloud:tracks:2096736162/55a12483-c5f8-471b-b2ac-7e044bc71ce8/stream/hls\",\"preset\":\"aac_160k\",\"duration\":184966,\"snipped\":false,\"format\":{\"protocol\":\"hls\",\"mime_type\":\"audio/mp4; codecs\u003d\\\"mp4a.40.2\\\"\"},\"quality\":\"sq\",\"is_legacy_transcoding\":false},{\"url\":\"https://api-v2.soundcloud.com/media/soundcloud:tracks:2096736162/08ba6930-e6f1-4b36-a6aa-19c32439b311/stream/hls\",\"preset\":\"abr_sq\",\"duration\":184966,\"snipped\":false,\"format\":{\"protocol\":\"hls\",\"mime_type\":\"audio/mpegurl\"},\"quality\":\"sq\",\"is_legacy_transcoding\":false},{\"url\":\"https://api-v2.soundcloud.com/media/soundcloud:tracks:2096736162/9f023231-5e1d-49ae-bda7-1779d7db7628/stream/hls\",\"preset\":\"mp3_1_0\",\"duration\":184947,\"snipped\":false,\"format\":{\"protocol\":\"hls\",\"mime_type\":\"audio/mpeg\"},\"quality\":\"sq\",\"is_legacy_transcoding\":true},{\"url\":\"https://api-v2.soundcloud.com/media/soundcloud:tracks:2096736162/9f023231-5e1d-49ae-bda7-1779d7db7628/stream/progressive\",\"preset\":\"mp3_1_0\",\"duration\":184947,\"snipped\":false,\"format\":{\"protocol\":\"progressive\",\"mime_type\":\"audio/mpeg\"},\"quality\":\"sq\",\"is_legacy_transcoding\":true},{\"url\":\"https://api-v2.soundcloud.com/media/soundcloud:tracks:2096736162/6e004bd1-2895-4975-a95f-bcaba530bb16/stream/hls\",\"preset\":\"opus_0_0\",\"duration\":184926,\"snipped\":false,\"format\":{\"protocol\":\"hls\",\"mime_type\":\"audio/ogg; codecs\u003d\\\"opus\\\"\"},\"quality\":\"sq\",\"is_legacy_transcoding\":true}]},\"station_urn\":\"soundcloud:system-playlists:track-stations:2096736162\",\"station_permalink\":\"track-stations:2096736162\",\"track_authorization\":\"eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJnZW8iOiJERSIsInN1YiI6IiIsInJpZCI6IjkwNDI3NzRmLTRhYjEtNGUzYS1iZTQ1LWQ4NzViOWEwZGNmMyIsImlhdCI6MTc2MDE2NzE4MX0.HJl6wdL5pKQUGa4vISvx8Lxanh8NLPrNUFaGB-qa0Ko\",\"monetization_model\":\"AD_SUPPORTED\",\"policy\":\"MONETIZE\",\"user\":{\"avatar_url\":\"https://i1.sndcdn.com/avatars-ETlfaab3lfH7NuVu-7ckpPg-large.jpg\",\"first_name\":\"Stranger\",\"followers_count\":17408,\"full_name\":\"Stranger Twins\",\"id\":73637815,\"kind\":\"user\",\"last_modified\":\"2025-09-26T12:22:45Z\",\"last_name\":\"Twins\",\"permalink\":\"hurtbox\",\"permalink_url\":\"https://soundcloud.com/hurtbox\",\"uri\":\"https://api.soundcloud.com/users/soundcloud%3Ausers%3A73637815\",\"urn\":\"soundcloud:users:73637815\",\"username\":\"HURTBOX\",\"verified\":true,\"city\":\"\",\"country_code\":\"FR\",\"badges\":{\"pro\":false,\"creator_mid_tier\":false,\"pro_unlimited\":true,\"verified\":true},\"station_urn\":\"soundcloud:system-playlists:artist-stations:73637815\",\"station_permalink\":\"artist-stations:73637815\"}}],\"next_href\":\"https://api-v2.soundcloud.com/users/73637815/tracks?offset\u003d2025-05-04T21%3A58%3A34.000Z%2Ctracks%2C00000000002090038011\u0026limit\u003d20\",\"query_urn\":null}",
63+
"latestUrl": "https://api-v2.soundcloud.com/users/73637815/tracks?offset\u003d2025-06-06T13%3A44%3A21.000Z%2Ctracks%2C00000000002108853114\u0026limit\u003d20\u0026client_id\u003dLAd42S06rwW6N9SO85p7573ak7rH6lMf"
64+
}
65+
}

0 commit comments

Comments
 (0)