Skip to content

Fix SoundCloud channel tab pagination looping on empty pages #1387

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
TobiGr merged 3 commits into from
Oct 15, 2025
Merged

Fix SoundCloud channel tab pagination looping on empty pages #1387

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d6165fa
Fix SoundCloud artist tracks page infinite loading
that1potato Oct 11, 2025
d7a38f2
[SoundCloud] Add test to check that next pages are not empty for tabs
TobiGr Oct 11, 2025
702f1b0
Record mocks for new test
TobiGr Oct 11, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 39 additions & 3 deletions ...chabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelTabExtractor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@

import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.HashSet;
import java.util.Set;

import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
Expand All @@ -22,6 +24,9 @@ public class SoundcloudChannelTabExtractor extends ChannelTabExtractor {

private static final String USERS_ENDPOINT = SOUNDCLOUD_API_V2_URL + "users/";

/** Empty page cap against infinite pagination loops. */
private static final int MAX_EMPTY_PAGES = 3;

private final String userId;

public SoundcloudChannelTabExtractor(final StreamingService service,
Expand Down Expand Up @@ -70,9 +75,40 @@ public InfoItemsPage<InfoItem> getPage(final Page page)
}

final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId());
final String nextPageUrl = SoundcloudParsingHelper.getInfoItemsFromApi(
collector, page.getUrl());
final Set<String> visitedPages = new HashSet<>();

String currentPageUrl = page.getUrl();
String nextPageUrl = "";
int emptyPageCount = 0;

while (!isNullOrEmpty(currentPageUrl)) {
if (!visitedPages.add(currentPageUrl)) {
// Prevent infinite loops when the API points back to an already visited page.
nextPageUrl = "";
break;
}

final int itemsBefore = collector.getItems().size();
final String candidateNextPage = SoundcloudParsingHelper
.getInfoItemsFromApi(collector, currentPageUrl);
final boolean hasNewItems = collector.getItems().size() > itemsBefore;

if (hasNewItems) {
nextPageUrl = candidateNextPage;
break;
}

emptyPageCount++;
if (emptyPageCount >= MAX_EMPTY_PAGES || isNullOrEmpty(candidateNextPage)) {
// Give up after too many empty responses or when SoundCloud stops providing tokens.
nextPageUrl = "";
break;
}

currentPageUrl = candidateNextPage;
}

return new InfoItemsPage<>(collector, new Page(nextPageUrl));
final Page nextPage = isNullOrEmpty(nextPageUrl) ? null : new Page(nextPageUrl);
return new InfoItemsPage<>(collector, nextPage);
}
}
32 changes: 32 additions & 0 deletions ...a/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelTabExtractorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,15 +1,21 @@
package org.schabi.newpipe.extractor.services.soundcloud;

import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmptyErrors;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestGetPageInNewExtractor;

import org.junit.jupiter.api.Test;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabExtractor;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
import org.schabi.newpipe.extractor.services.DefaultListExtractorTest;

import java.util.List;

class SoundcloudChannelTabExtractorTest {

static class Tracks extends DefaultListExtractorTest<ChannelTabExtractor> {
Expand Down Expand Up @@ -88,4 +94,30 @@ protected ChannelTabExtractor createExtractor() throws Exception {
@Override public String expectedOriginalUrlContains() throws Exception { return "https://soundcloud.com/soreen-735855039/likes"; }
@Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.PLAYLIST; }
}

static class NoEmptyNextPages extends DefaultListExtractorTest<ChannelTabExtractor> {
@Override
protected ChannelTabExtractor createExtractor() throws Exception {
return SoundCloud.getChannelTabExtractorFromId("73637815", ChannelTabs.TRACKS);
}

@Override public StreamingService expectedService() throws Exception { return SoundCloud; }
@Override public String expectedName() throws Exception { return ChannelTabs.TRACKS; }
@Override public String expectedId() throws Exception { return "73637815"; }
@Override public String expectedUrlContains() throws Exception { return "https://soundcloud.com/hurtbox/tracks"; }
@Override public String expectedOriginalUrlContains() throws Exception { return "https://soundcloud.com/hurtbox/tracks"; }
@Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; }

@Test
public void testNextPages() throws Exception {
ListExtractor.InfoItemsPage<InfoItem> page = extractor().getInitialPage();
for (int i = 1; i <= 5; i++) {
assertEmptyErrors("Next page has errors", page.getErrors());
assertFalse(page.getItems().isEmpty(), "Next page is empty");
assertTrue(page.hasNextPage(), "Next page does not have more items");
page = extractor().getPage(page.getNextPage());
}
}

}
}
59 changes: 59 additions & 0 deletions .../services/soundcloud/soundcloudchanneltabextractor/noemptynextpages/generated_mock_0.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
{
"request": {
"httpMethod": "GET",
"url": "https://w.soundcloud.com/player/?url\u003dhttps%3A%2F%2Fapi.soundcloud.com%2Fusers%2F73637815",
"headers": {
"Accept-Language": [
"en-GB, en;q\u003d0.9"
]
},
"localization": {
"languageCode": "en",
"countryCode": "GB"
}
},
"response": {
"responseCode": 200,
"responseMessage": "",
"responseHeaders": {
"age": [
"41"
],
"cache-control": [
"public, max-age\u003d300"
],
"content-type": [
"text/html"
],
"date": [
"Sat, 11 Oct 2025 07:18:56 GMT"
],
"p3p": [
"policyref\u003d\"https://w.soundcloud.com/player/w3c/p3p.xml\", CP\u003d\"NON DSP COR CUR ADM DEV TAI PSAo PSDo OUR STP CNT\""
],
"server": [
"am/2"
],
"strict-transport-security": [
"max-age\u003d63072000"
],
"vary": [
"Accept-Encoding"
],
"via": [
"sssr, 1.1 446f4800ed8a030e10e4a6dd15349224.cloudfront.net (CloudFront)"
],
"x-amz-cf-id": [
"6Yw0AsuW3SIXRNaT7KyQD0Rm1Mtomsc3UiS4gbjS0hGNEDTdE1RW7A\u003d\u003d"
],
"x-amz-cf-pop": [
"TXL50-P5"
],
"x-cache": [
"Hit from cloudfront"
]
},
"responseBody": "\u003c!DOCTYPE html\u003e\n\u003chtml lang\u003d\"en\"\u003e\n\u003chead\u003e\n \u003cmeta charset\u003d\"UTF-8\"\u003e\n\n \n \u003clink rel\u003d\"dns-prefetch\" href\u003d\"//api-widget.soundcloud.com\"\u003e\n \u003clink rel\u003d\"dns-prefetch\" href\u003d\"//sb.scorecardresearch.com\"\u003e\n \u003clink rel\u003d\"dns-prefetch\" href\u003d\"//api.soundcloud.com\"\u003e\n \u003clink rel\u003d\"dns-prefetch\" href\u003d\"//va.sndcdn.com\"\u003e\n \u003clink rel\u003d\"dns-prefetch\" href\u003d\"//i1.sndcdn.com\"\u003e\n \u003clink rel\u003d\"dns-prefetch\" href\u003d\"//i2.sndcdn.com\"\u003e\n \u003clink rel\u003d\"dns-prefetch\" href\u003d\"//i3.sndcdn.com\"\u003e\n \u003clink rel\u003d\"dns-prefetch\" href\u003d\"//i4.sndcdn.com\"\u003e\n \u003clink rel\u003d\"dns-prefetch\" href\u003d\"//w1.sndcdn.com\"\u003e\n \u003clink rel\u003d\"dns-prefetch\" href\u003d\"//wis.sndcdn.com\"\u003e\n \u003clink rel\u003d\"dns-prefetch\" href\u003d\"//style.sndcdn.com\"\u003e\n \n \u003cmeta name\u003d\"robots\" content\u003d\"noindex, follow\"\u003e\n \u003cmeta http-equiv\u003d\"X-UA-Compatible\" content\u003d\"IE\u003dedge,chrome\u003d1\"\u003e\n \u003cmeta name\u003d\"viewport\" content\u003d\"width\u003ddevice-width,minimum-scale\u003d1,maximum-scale\u003d1,user-scalable\u003dno\"\u003e\n \u003clink rel\u003d\"canonical\" href\u003d\"https://soundcloud.com/hurtbox\"\u003e\n\n \u003ctitle\u003eSoundCloud Widget\u003c/title\u003e\n\n \n \u003cbase target\u003d\"_blank\"\u003e\n \n\u003c/head\u003e\n\u003cbody style\u003d\"margin:0\"\u003e\n \u003cdiv id\u003d\"widget\" class\u003d\"widget g-background-default g-shadow-inset\" style\u003d\"height:100%\"\u003e\u003c/div\u003e\n \u003cscript src\u003d\"https://widget.sndcdn.com/widget-4-434ca88dccea.js\" crossorigin\u003e\u003c/script\u003e\n \n \u003cscript crossorigin src\u003d\"https://widget.sndcdn.com/widget-8-8102dadd8b2e.js\"\u003e\u003c/script\u003e\n \n \u003cscript crossorigin src\u003d\"https://widget.sndcdn.com/widget-9-0111f457ff60.js\"\u003e\u003c/script\u003e\n \n \n \u003cscript\u003ewindow.innerHeight\u003c\u003d20\u0026\u0026document.body.classList.add(\"mini-player\")\u003c/script\u003e\n\u003c/body\u003e\n\u003c/html\u003e\n",
"latestUrl": "https://w.soundcloud.com/player/?url\u003dhttps%3A%2F%2Fapi.soundcloud.com%2Fusers%2F73637815"
}
}
68 changes: 68 additions & 0 deletions .../services/soundcloud/soundcloudchanneltabextractor/noemptynextpages/generated_mock_1.json
View file
Open in desktop

Large diffs are not rendered by default.

65 changes: 65 additions & 0 deletions ...services/soundcloud/soundcloudchanneltabextractor/noemptynextpages/generated_mock_10.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
{
"request": {
"httpMethod": "GET",
"url": "https://api-v2.soundcloud.com/users/73637815/tracks?offset\u003d2025-06-06T13%3A44%3A21.000Z%2Ctracks%2C00000000002108853114\u0026limit\u003d20\u0026client_id\u003dMoLbAg35TuqjYwWVtNIKyRPFScQGMOBY",
"headers": {
"Accept-Language": [
"en-GB, en;q\u003d0.9"
]
},
"localization": {
"languageCode": "en",
"countryCode": "GB"
}
},
"response": {
"responseCode": 200,
"responseMessage": "",
"responseHeaders": {
"cache-control": [
"private, max-age\u003d0"
],
"content-type": [
"application/json; charset\u003dutf-8"
],
"date": [
"Sat, 11 Oct 2025 07:19:41 GMT"
],
"referrer-policy": [
"no-referrer"
],
"server": [
"am/2"
],
"strict-transport-security": [
"max-age\u003d63072000"
],
"vary": [
"Origin"
],
"via": [
"1.1 83dede49b3c78b7f99a0399a34264ed6.cloudfront.net (CloudFront)"
],
"x-amz-cf-id": [
"pyfR0l0XQeLpb9zKY_oTxzNyHqBqQwW3DUlsakMBZ_OJXpXCtMuQ2g\u003d\u003d"
],
"x-amz-cf-pop": [
"TXL52-P2"
],
"x-cache": [
"Miss from cloudfront"
],
"x-content-type-options": [
"nosniff"
],
"x-frame-options": [
"DENY"
],
"x-robots-tag": [
"noindex"
]
},
"responseBody": "{\"collection\":[{\"artwork_url\":\"https://i1.sndcdn.com/artworks-6coMhjFIhwNmQn3U-0Ls5Ig-large.jpg\",\"caption\":null,\"commentable\":true,\"comment_count\":137,\"created_at\":\"2025-05-16T04:00:56Z\",\"description\":\"HURTBOX - FUCK LOVE (ft. WATSON)\\n\\nBUY / STREAM: https://createmusic.fm/fklove\\n\\nCONNECT WITH HURTBOX\\nLINKTR.EE/HURTBOX\\nWWW.INSTAGRAM.COM/HURTBOXOFFICIAL\\nWWW.TWITTER.COM/HURTBOXOFFICIAL\\n\\nCONNECT WITH @THEONLYWATSON\\nwww.instagram.com/theonlywatson\\nwww.x.com/theonlywatson\",\"downloadable\":false,\"download_count\":0,\"duration\":184966,\"full_duration\":184947,\"embeddable_by\":\"all\",\"genre\":\"Dubstep\",\"has_downloads_left\":false,\"id\":2096736162,\"kind\":\"track\",\"label_name\":\"\",\"last_modified\":\"2025-05-16T10:08:07Z\",\"license\":\"all-rights-reserved\",\"likes_count\":2795,\"permalink\":\"fucklove\",\"permalink_url\":\"https://soundcloud.com/hurtbox/fucklove\",\"playback_count\":46975,\"public\":true,\"publisher_metadata\":{\"id\":2096736162,\"urn\":\"soundcloud:tracks:2096736162\",\"artist\":\"hurtbox\",\"contains_music\":true,\"isrc\":\"QZTGW2404395\",\"explicit\":false},\"purchase_title\":\"Buy / Stream\",\"purchase_url\":\"https://createmusic.fm/fklove\",\"release_date\":\"2025-05-15T00:00:00Z\",\"reposts_count\":189,\"secret_token\":null,\"sharing\":\"public\",\"state\":\"finished\",\"streamable\":true,\"tag_list\":\"\",\"title\":\"HURTBOX - FUCK LOVE (ft. WATSON)\",\"uri\":\"https://api.soundcloud.com/tracks/soundcloud%3Atracks%3A2096736162\",\"urn\":\"soundcloud:tracks:2096736162\",\"user_id\":73637815,\"visuals\":null,\"waveform_url\":\"https://wave.sndcdn.com/pcfjNOZBWkai_m.json\",\"display_date\":\"2025-05-16T04:00:56Z\",\"media\":{\"transcodings\":[{\"url\":\"https://api-v2.soundcloud.com/media/soundcloud:tracks:2096736162/55a12483-c5f8-471b-b2ac-7e044bc71ce8/stream/hls\",\"preset\":\"aac_160k\",\"duration\":184966,\"snipped\":false,\"format\":{\"protocol\":\"hls\",\"mime_type\":\"audio/mp4; codecs\u003d\\\"mp4a.40.2\\\"\"},\"quality\":\"sq\",\"is_legacy_transcoding\":false},{\"url\":\"https://api-v2.soundcloud.com/media/soundcloud:tracks:2096736162/08ba6930-e6f1-4b36-a6aa-19c32439b311/stream/hls\",\"preset\":\"abr_sq\",\"duration\":184966,\"snipped\":false,\"format\":{\"protocol\":\"hls\",\"mime_type\":\"audio/mpegurl\"},\"quality\":\"sq\",\"is_legacy_transcoding\":false},{\"url\":\"https://api-v2.soundcloud.com/media/soundcloud:tracks:2096736162/9f023231-5e1d-49ae-bda7-1779d7db7628/stream/hls\",\"preset\":\"mp3_1_0\",\"duration\":184947,\"snipped\":false,\"format\":{\"protocol\":\"hls\",\"mime_type\":\"audio/mpeg\"},\"quality\":\"sq\",\"is_legacy_transcoding\":true},{\"url\":\"https://api-v2.soundcloud.com/media/soundcloud:tracks:2096736162/9f023231-5e1d-49ae-bda7-1779d7db7628/stream/progressive\",\"preset\":\"mp3_1_0\",\"duration\":184947,\"snipped\":false,\"format\":{\"protocol\":\"progressive\",\"mime_type\":\"audio/mpeg\"},\"quality\":\"sq\",\"is_legacy_transcoding\":true},{\"url\":\"https://api-v2.soundcloud.com/media/soundcloud:tracks:2096736162/6e004bd1-2895-4975-a95f-bcaba530bb16/stream/hls\",\"preset\":\"opus_0_0\",\"duration\":184926,\"snipped\":false,\"format\":{\"protocol\":\"hls\",\"mime_type\":\"audio/ogg; codecs\u003d\\\"opus\\\"\"},\"quality\":\"sq\",\"is_legacy_transcoding\":true}]},\"station_urn\":\"soundcloud:system-playlists:track-stations:2096736162\",\"station_permalink\":\"track-stations:2096736162\",\"track_authorization\":\"eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJnZW8iOiJERSIsInN1YiI6IiIsInJpZCI6IjkwNDI3NzRmLTRhYjEtNGUzYS1iZTQ1LWQ4NzViOWEwZGNmMyIsImlhdCI6MTc2MDE2NzE4MX0.HJl6wdL5pKQUGa4vISvx8Lxanh8NLPrNUFaGB-qa0Ko\",\"monetization_model\":\"AD_SUPPORTED\",\"policy\":\"MONETIZE\",\"user\":{\"avatar_url\":\"https://i1.sndcdn.com/avatars-ETlfaab3lfH7NuVu-7ckpPg-large.jpg\",\"first_name\":\"Stranger\",\"followers_count\":17408,\"full_name\":\"Stranger Twins\",\"id\":73637815,\"kind\":\"user\",\"last_modified\":\"2025-09-26T12:22:45Z\",\"last_name\":\"Twins\",\"permalink\":\"hurtbox\",\"permalink_url\":\"https://soundcloud.com/hurtbox\",\"uri\":\"https://api.soundcloud.com/users/soundcloud%3Ausers%3A73637815\",\"urn\":\"soundcloud:users:73637815\",\"username\":\"HURTBOX\",\"verified\":true,\"city\":\"\",\"country_code\":\"FR\",\"badges\":{\"pro\":false,\"creator_mid_tier\":false,\"pro_unlimited\":true,\"verified\":true},\"station_urn\":\"soundcloud:system-playlists:artist-stations:73637815\",\"station_permalink\":\"artist-stations:73637815\"}}],\"next_href\":\"https://api-v2.soundcloud.com/users/73637815/tracks?offset\u003d2025-05-04T21%3A58%3A34.000Z%2Ctracks%2C00000000002090038011\u0026limit\u003d20\",\"query_urn\":null}",
"latestUrl": "https://api-v2.soundcloud.com/users/73637815/tracks?offset\u003d2025-06-06T13%3A44%3A21.000Z%2Ctracks%2C00000000002108853114\u0026limit\u003d20\u0026client_id\u003dLAd42S06rwW6N9SO85p7573ak7rH6lMf"
}
}
Loading