From 03aae47ce7388e1d002649e02be8d763955ad7c9 Mon Sep 17 00:00:00 2001 From: AbsurdlyLongUsername <22662897+absurdlylongusername@users.noreply.github.com> Date: Sat, 4 Apr 2026 23:56:52 +0100 Subject: [PATCH 1/7] [SoundCloud] Validate http response code in SoundcloudParsingHelper --- .../extractor/downloader/Response.java | 20 +++++++++++ .../exceptions/HttpResponseException.java | 15 ++++++++ .../soundcloud/SoundcloudParsingHelper.java | 34 +++++++++++------- .../newpipe/extractor/utils/HttpUtils.java | 36 +++++++++++++++++++ 4 files changed, 92 insertions(+), 13 deletions(-) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/HttpResponseException.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/utils/HttpUtils.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java b/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java index ac792dc756..87c3577ef4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java @@ -6,6 +6,9 @@ import java.util.List; import java.util.Map; +import org.schabi.newpipe.extractor.exceptions.HttpResponseException; +import org.schabi.newpipe.extractor.utils.HttpUtils; + /** * A Data class used to hold the results from requests made by the Downloader implementation. */ @@ -80,4 +83,21 @@ public String getHeader(final String name) { return null; } + // CHECKSTYLE:OFF + /** + * Helper function simply to make it easier to validate response code inline + * before getting the code/body/latestUrl/etc. + * Validates the response codes for the given {@link Response}, and throws a {@link HttpResponseException} if the code is invalid + * @see HttpUtils#validateResponseCode(Response, int...) + * @param validResponseCodes Expected valid response codes + * @return {@link this} response + * @throws HttpResponseException Thrown when the response code is not in {@code validResponseCodes}, + * or when {@code validResponseCodes} is empty and the code is a 4xx or 5xx error. + */ + // CHECKSTYLE:ON + public Response validateResponseCode(final int... validResponseCodes) + throws HttpResponseException { + HttpUtils.validateResponseCode(this, validResponseCodes); + return this; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/HttpResponseException.java b/extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/HttpResponseException.java new file mode 100644 index 0000000000..c07850a9d3 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/HttpResponseException.java @@ -0,0 +1,15 @@ +package org.schabi.newpipe.extractor.exceptions; + +import java.io.IOException; +import org.schabi.newpipe.extractor.downloader.Response; + +public class HttpResponseException extends IOException { + public HttpResponseException(final Response response) { + this("Error in HTTP Response for " + response.latestUrl() + "\n\t" + + response.responseCode() + " - " + response.responseMessage()); + } + + public HttpResponseException(final String message) { + super(message); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index aeff6bd363..26856fc8c8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -5,6 +5,7 @@ import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; +import static org.schabi.newpipe.extractor.utils.HttpUtils.validateResponseCode; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; @@ -12,7 +13,6 @@ import com.grack.nanojson.JsonParserException; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; import org.jsoup.select.Elements; import org.schabi.newpipe.extractor.MultiInfoItemsCollector; import org.schabi.newpipe.extractor.Image; @@ -105,8 +105,8 @@ public static synchronized String clientId() throws ExtractionException, IOExcep final Downloader dl = NewPipe.getDownloader(); - final Response download = dl.get("https://soundcloud.com"); - final String responseBody = download.responseBody(); + final Response downloadResponse = dl.get("https://soundcloud.com").validateResponseCode(); + final String responseBody = downloadResponse.responseBody(); final String clientIdPattern = ",client_id:\"(.*?)\""; final Document doc = Jsoup.parse(responseBody); @@ -117,11 +117,12 @@ public static synchronized String clientId() throws ExtractionException, IOExcep final var headers = Map.of("Range", List.of("bytes=0-50000")); - for (final Element element : possibleScripts) { + for (final var element : possibleScripts) { final String srcUrl = element.attr("src"); if (!isNullOrEmpty(srcUrl)) { try { clientId = Parser.matchGroup1(clientIdPattern, dl.get(srcUrl, headers) + .validateResponseCode() .responseBody()); return clientId; } catch (final RegexException ignored) { @@ -149,11 +150,13 @@ public static DateWrapper parseDate(final String uploadDate) throws ParsingExcep } } + // CHECKSTYLE:OFF /** - * Call the endpoint "/resolve" of the API.
+ * Call the endpoint "/resolve" of the API. *
- * See https://developers.soundcloud.com/docs/api/reference#resolve
+ * See https://web.archive.org/web/20170804051146/https://developers.soundcloud.com/docs/api/reference#resolve
*/
+ // CHECKSTYLE:ON
public static JsonObject resolveFor(@Nonnull final Downloader downloader, final String url)
throws IOException, ExtractionException {
final String apiUrl = SOUNDCLOUD_API_V2_URL + "resolve"
@@ -178,10 +181,11 @@ public static JsonObject resolveFor(@Nonnull final Downloader downloader, final
public static String resolveUrlWithEmbedPlayer(final String apiUrl) throws IOException,
ReCaptchaException {
- final String response = NewPipe.getDownloader().get("https://w.soundcloud.com/player/?url="
- + Utils.encodeUrlUtf8(apiUrl), SoundCloud.getLocalization()).responseBody();
-
- return Jsoup.parse(response).select("link[rel=\"canonical\"]").first()
+ final var response = NewPipe.getDownloader().get("https://w.soundcloud.com/player/?url="
+ + Utils.encodeUrlUtf8(apiUrl), SoundCloud.getLocalization());
+ validateResponseCode(response);
+ final var responseBody = response.responseBody();
+ return Jsoup.parse(responseBody).select("link[rel=\"canonical\"]").first()
.attr("abs:href");
}
@@ -190,6 +194,7 @@ public static String resolveUrlWithEmbedPlayer(final String apiUrl) throws IOExc
*
* @return the resolved id
*/
+ // TODO: what makes this method different from the others? Don' they all return the same?
public static String resolveIdWithWidgetApi(final String urlString) throws IOException,
ParsingException {
String fixedUrl = urlString;
@@ -225,9 +230,12 @@ public static String resolveIdWithWidgetApi(final String urlString) throws IOExc
final String widgetUrl = "https://api-widget.soundcloud.com/resolve?url="
+ Utils.encodeUrlUtf8(url.toString())
+ "&format=json&client_id=" + SoundcloudParsingHelper.clientId();
- final String response = NewPipe.getDownloader().get(widgetUrl,
- SoundCloud.getLocalization()).responseBody();
- final JsonObject o = JsonParser.object().from(response);
+
+ final var response = NewPipe.getDownloader().get(widgetUrl,
+ SoundCloud.getLocalization());
+
+ final var responseBody = response.validateResponseCode().responseBody();
+ final JsonObject o = JsonParser.object().from(responseBody);
return String.valueOf(JsonUtils.getValue(o, "id"));
} catch (final JsonParserException e) {
throw new ParsingException("Could not parse JSON response", e);
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/HttpUtils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/HttpUtils.java
new file mode 100644
index 0000000000..31c937ea09
--- /dev/null
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/HttpUtils.java
@@ -0,0 +1,36 @@
+package org.schabi.newpipe.extractor.utils;
+
+import java.util.Arrays;
+
+import org.schabi.newpipe.extractor.downloader.Response;
+import org.schabi.newpipe.extractor.exceptions.HttpResponseException;
+
+public final class HttpUtils {
+
+ private HttpUtils() {
+ // Utility class, no instances allowed
+ }
+
+ // CHECKSTYLE:OFF
+ /**
+ * Validates the response codes for the given {@link Response}, and throws
+ * a {@link HttpResponseException} if the code is invalid
+ * @param response The response to validate
+ * @param validResponseCodes Expected valid response codes
+ * @throws HttpResponseException Thrown when the response code is not in {@code validResponseCodes},
+ * or when {@code validResponseCodes} is empty and the code is a 4xx or 5xx error.
+ */
+ // CHECKSTYLE:ON
+ public static void validateResponseCode(final Response response,
+ final int... validResponseCodes)
+ throws HttpResponseException {
+ final int code = response.responseCode();
+ final var throwError = (validResponseCodes == null || validResponseCodes.length == 0)
+ ? code >= 400 && code <= 599
+ : Arrays.stream(validResponseCodes).noneMatch(c -> c == code);
+
+ if (throwError) {
+ throw new HttpResponseException(response);
+ }
+ }
+}
From 59e94ee627ff20dcef43923bd89c8a946243685e Mon Sep 17 00:00:00 2001
From: AbsurdlyLongUsername
<22662897+absurdlylongusername@users.noreply.github.com>
Date: Sat, 4 Apr 2026 23:57:04 +0100
Subject: [PATCH 2/7] Move validateResponseCode to Response Make
HttpResponseException extend ExtractionException Remove HttpUtils
---
.../extractor/downloader/Response.java | 29 +++++++++++++--
.../exceptions/HttpResponseException.java | 3 +-
.../soundcloud/SoundcloudParsingHelper.java | 5 +--
.../newpipe/extractor/utils/HttpUtils.java | 36 -------------------
4 files changed, 30 insertions(+), 43 deletions(-)
delete mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/utils/HttpUtils.java
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java b/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java
index 87c3577ef4..593b8a78e5 100644
--- a/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java
@@ -2,12 +2,12 @@
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
+import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import org.schabi.newpipe.extractor.exceptions.HttpResponseException;
-import org.schabi.newpipe.extractor.utils.HttpUtils;
/**
* A Data class used to hold the results from requests made by the Downloader implementation.
@@ -33,6 +33,28 @@ public Response(final int responseCode,
this.latestUrl = latestUrl;
}
+ /**
+ * Validates the response codes for the given {@link Response}, and throws
+ * a {@link HttpResponseException} if the code is invalid
+ * @param response The response to validate
+ * @param validResponseCodes Expected valid response codes
+ * @throws HttpResponseException Thrown when the response code is not in {@code validResponseCodes},
+ * or when {@code validResponseCodes} is empty and the code is a 4xx or 5xx error.
+ */
+ // CHECKSTYLE:ON
+ public static void validateResponseCode(final Response response,
+ final int... validResponseCodes)
+ throws HttpResponseException {
+ final int code = response.responseCode();
+ final var throwError = (validResponseCodes == null || validResponseCodes.length == 0)
+ ? code >= 400 && code <= 599
+ : Arrays.stream(validResponseCodes).noneMatch(c -> c == code);
+
+ if (throwError) {
+ throw new HttpResponseException(response);
+ }
+ }
+
public int responseCode() {
return responseCode;
}
@@ -83,12 +105,13 @@ public String getHeader(final String name) {
return null;
}
+
// CHECKSTYLE:OFF
/**
* Helper function simply to make it easier to validate response code inline
* before getting the code/body/latestUrl/etc.
* Validates the response codes for the given {@link Response}, and throws a {@link HttpResponseException} if the code is invalid
- * @see HttpUtils#validateResponseCode(Response, int...)
+ * @see Response#validateResponseCode(Response, int...)
* @param validResponseCodes Expected valid response codes
* @return {@link this} response
* @throws HttpResponseException Thrown when the response code is not in {@code validResponseCodes},
@@ -97,7 +120,7 @@ public String getHeader(final String name) {
// CHECKSTYLE:ON
public Response validateResponseCode(final int... validResponseCodes)
throws HttpResponseException {
- HttpUtils.validateResponseCode(this, validResponseCodes);
+ validateResponseCode(this, validResponseCodes);
return this;
}
}
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/HttpResponseException.java b/extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/HttpResponseException.java
index c07850a9d3..51b1f77c66 100644
--- a/extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/HttpResponseException.java
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/HttpResponseException.java
@@ -1,9 +1,8 @@
package org.schabi.newpipe.extractor.exceptions;
-import java.io.IOException;
import org.schabi.newpipe.extractor.downloader.Response;
-public class HttpResponseException extends IOException {
+public class HttpResponseException extends ExtractionException {
public HttpResponseException(final Response response) {
this("Error in HTTP Response for " + response.latestUrl() + "\n\t"
+ response.responseCode() + " - " + response.responseMessage());
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java
index 26856fc8c8..17f3a49edf 100644
--- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java
@@ -5,7 +5,7 @@
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
-import static org.schabi.newpipe.extractor.utils.HttpUtils.validateResponseCode;
+import static org.schabi.newpipe.extractor.downloader.Response.validateResponseCode;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
@@ -21,6 +21,7 @@
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.exceptions.HttpResponseException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.localization.DateWrapper;
@@ -179,7 +180,7 @@ public static JsonObject resolveFor(@Nonnull final Downloader downloader, final
* @return the url resolved
*/
public static String resolveUrlWithEmbedPlayer(final String apiUrl) throws IOException,
- ReCaptchaException {
+ ReCaptchaException, HttpResponseException {
final var response = NewPipe.getDownloader().get("https://w.soundcloud.com/player/?url="
+ Utils.encodeUrlUtf8(apiUrl), SoundCloud.getLocalization());
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/HttpUtils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/HttpUtils.java
deleted file mode 100644
index 31c937ea09..0000000000
--- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/HttpUtils.java
+++ /dev/null
@@ -1,36 +0,0 @@
-package org.schabi.newpipe.extractor.utils;
-
-import java.util.Arrays;
-
-import org.schabi.newpipe.extractor.downloader.Response;
-import org.schabi.newpipe.extractor.exceptions.HttpResponseException;
-
-public final class HttpUtils {
-
- private HttpUtils() {
- // Utility class, no instances allowed
- }
-
- // CHECKSTYLE:OFF
- /**
- * Validates the response codes for the given {@link Response}, and throws
- * a {@link HttpResponseException} if the code is invalid
- * @param response The response to validate
- * @param validResponseCodes Expected valid response codes
- * @throws HttpResponseException Thrown when the response code is not in {@code validResponseCodes},
- * or when {@code validResponseCodes} is empty and the code is a 4xx or 5xx error.
- */
- // CHECKSTYLE:ON
- public static void validateResponseCode(final Response response,
- final int... validResponseCodes)
- throws HttpResponseException {
- final int code = response.responseCode();
- final var throwError = (validResponseCodes == null || validResponseCodes.length == 0)
- ? code >= 400 && code <= 599
- : Arrays.stream(validResponseCodes).noneMatch(c -> c == code);
-
- if (throwError) {
- throw new HttpResponseException(response);
- }
- }
-}
From 6831369cec2d3184ed4aa0f465a1b447f5919ea2 Mon Sep 17 00:00:00 2001
From: AbsurdlyLongUsername
<22662897+absurdlylongusername@users.noreply.github.com>
Date: Sat, 4 Apr 2026 23:57:11 +0100
Subject: [PATCH 3/7] Fix checkstyle off
---
.../java/org/schabi/newpipe/extractor/downloader/Response.java | 1 +
1 file changed, 1 insertion(+)
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java b/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java
index 593b8a78e5..b759fac2cc 100644
--- a/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java
@@ -33,6 +33,7 @@ public Response(final int responseCode,
this.latestUrl = latestUrl;
}
+ // CHECKSTYLE:OFF
/**
* Validates the response codes for the given {@link Response}, and throws
* a {@link HttpResponseException} if the code is invalid
From ad9f4f48c42e41c96405477e7a77e9a3d8f86861 Mon Sep 17 00:00:00 2001
From: AbsurdlyLongUsername
<22662897+absurdlylongusername@users.noreply.github.com>
Date: Tue, 7 Apr 2026 05:41:50 +0100
Subject: [PATCH 4/7] Remove validateResponseCodes Add various helper methods
for validating response codes instead
---
.../extractor/downloader/Response.java | 157 ++++++++++++++----
1 file changed, 121 insertions(+), 36 deletions(-)
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java b/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java
index b759fac2cc..03f22ecfda 100644
--- a/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java
@@ -2,16 +2,17 @@
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
-import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
+import java.util.function.Function;
import org.schabi.newpipe.extractor.exceptions.HttpResponseException;
/**
* A Data class used to hold the results from requests made by the Downloader implementation.
*/
+@SuppressWarnings("checkstyle:NeedBraces")
public class Response {
private final int responseCode;
private final String responseMessage;
@@ -33,29 +34,6 @@ public Response(final int responseCode,
this.latestUrl = latestUrl;
}
- // CHECKSTYLE:OFF
- /**
- * Validates the response codes for the given {@link Response}, and throws
- * a {@link HttpResponseException} if the code is invalid
- * @param response The response to validate
- * @param validResponseCodes Expected valid response codes
- * @throws HttpResponseException Thrown when the response code is not in {@code validResponseCodes},
- * or when {@code validResponseCodes} is empty and the code is a 4xx or 5xx error.
- */
- // CHECKSTYLE:ON
- public static void validateResponseCode(final Response response,
- final int... validResponseCodes)
- throws HttpResponseException {
- final int code = response.responseCode();
- final var throwError = (validResponseCodes == null || validResponseCodes.length == 0)
- ? code >= 400 && code <= 599
- : Arrays.stream(validResponseCodes).noneMatch(c -> c == code);
-
- if (throwError) {
- throw new HttpResponseException(response);
- }
- }
-
public int responseCode() {
return responseCode;
}
@@ -107,21 +85,128 @@ public String getHeader(final String name) {
return null;
}
- // CHECKSTYLE:OFF
/**
- * Helper function simply to make it easier to validate response code inline
- * before getting the code/body/latestUrl/etc.
- * Validates the response codes for the given {@link Response}, and throws a {@link HttpResponseException} if the code is invalid
- * @see Response#validateResponseCode(Response, int...)
- * @param validResponseCodes Expected valid response codes
- * @return {@link this} response
- * @throws HttpResponseException Thrown when the response code is not in {@code validResponseCodes},
- * or when {@code validResponseCodes} is empty and the code is a 4xx or 5xx error.
+ * Ensure the response code is 2xx
+ * @return this {@code Response}
+ * @throws HttpResponseException if the response code is not 2xx
+ */
+ public Response ensureSuccessResponseCode() throws HttpResponseException {
+ return ensureResponseCodeInRange(200, 299);
+ }
+
+ /**
+ * Ensure the response code is 3xx
+ * @return this {@code Response}
+ * @throws HttpResponseException if the response code is not 3xx
+ */
+ public Response ensureRedirectResponseCode() throws HttpResponseException {
+ return ensureResponseCodeInRange(300, 399);
+ }
+
+ /**
+ * Ensure the response code is not 4xx or 5xx
+ * @return this {@code Response}
+ * @throws HttpResponseException if the response code is client or server error
+ */
+ public Response ensureResponseCodeIsNotError() throws HttpResponseException {
+ return ensureResponseCodeNotInRange(400, 599);
+ }
+
+ /**
+ * Ensure the HTTP response code is within range of min and max inclusive
+ * @return this Response
+ * @throws HttpResponseException if the code is outside the range
+ */
+ public Response ensureResponseCodeInRange(final int min, final int max)
+ throws HttpResponseException {
+ if (responseCode() < min || responseCode() > max) {
+ throw new HttpResponseException(this);
+ }
+ return this;
+ }
+
+ public Response ensureResponseCodeNotInRange(
+ final int min,
+ final int max,
+ final Function
- * See https://web.archive.org/web/20170804051146/https://developers.soundcloud.com/docs/api/reference#resolve
+ * See docs
*/
// CHECKSTYLE:ON
public static JsonObject resolveFor(@Nonnull final Downloader downloader, final String url)
@@ -166,7 +165,8 @@ public static JsonObject resolveFor(@Nonnull final Downloader downloader, final
try {
final String response = downloader.get(apiUrl, SoundCloud.getLocalization())
- .responseBody();
+ .ensureSuccessResponseCode()
+ .responseBody();
return JsonParser.object().from(response);
} catch (final JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
@@ -184,7 +184,7 @@ public static String resolveUrlWithEmbedPlayer(final String apiUrl) throws IOExc
final var response = NewPipe.getDownloader().get("https://w.soundcloud.com/player/?url="
+ Utils.encodeUrlUtf8(apiUrl), SoundCloud.getLocalization());
- validateResponseCode(response);
+ response.ensureSuccessResponseCode();
final var responseBody = response.responseBody();
return Jsoup.parse(responseBody).select("link[rel=\"canonical\"]").first()
.attr("abs:href");
@@ -195,7 +195,6 @@ public static String resolveUrlWithEmbedPlayer(final String apiUrl) throws IOExc
*
* @return the resolved id
*/
- // TODO: what makes this method different from the others? Don' they all return the same?
public static String resolveIdWithWidgetApi(final String urlString) throws IOException,
ParsingException {
String fixedUrl = urlString;
@@ -204,7 +203,14 @@ public static String resolveIdWithWidgetApi(final String urlString) throws IOExc
if (ON_URL_PATTERN.matcher(fixedUrl).find()) {
try {
- fixedUrl = NewPipe.getDownloader().head(fixedUrl).latestUrl();
+ fixedUrl = NewPipe.getDownloader()
+ .head(fixedUrl)
+ .throwIfServerError()
+ .latestUrl();
+ // Soundcloud returns HTTP 403 but it still redirects,
+ // so can get the redirected url via latestUrl regardless.
+ // This why only throw for 5xx and not 4xx.
+
// remove tracking params which are in the query string
fixedUrl = fixedUrl.split("\\?")[0];
} catch (final ExtractionException e) {
@@ -232,10 +238,11 @@ public static String resolveIdWithWidgetApi(final String urlString) throws IOExc
+ Utils.encodeUrlUtf8(url.toString())
+ "&format=json&client_id=" + SoundcloudParsingHelper.clientId();
- final var response = NewPipe.getDownloader().get(widgetUrl,
- SoundCloud.getLocalization());
+ final var responseBody = NewPipe.getDownloader()
+ .get(widgetUrl, SoundCloud.getLocalization())
+ .ensureSuccessResponseCode()
+ .responseBody();
- final var responseBody = response.validateResponseCode().responseBody();
final JsonObject o = JsonParser.object().from(responseBody);
return String.valueOf(JsonUtils.getValue(o, "id"));
} catch (final JsonParserException e) {
@@ -249,16 +256,16 @@ public static String resolveIdWithWidgetApi(final String urlString) throws IOExc
/**
* Fetch the users from the given API and commit each of them to the collector.
*
- * This differ from {@link #getUsersFromApi(ChannelInfoItemsCollector, String)} in the sense
+ * This differs from {@link #getUsersFromApi(ChannelInfoItemsCollector, String)} in the sense
* that they will always get MIN_ITEMS or more.
*
- * @param minItems the method will return only when it have extracted that many items
+ * @param minItems the method will return only when it has extracted this many items
* (equal or more)
*/
public static String getUsersFromApiMinItems(final int minItems,
final ChannelInfoItemsCollector collector,
final String apiUrl) throws IOException,
- ReCaptchaException, ParsingException {
+ ReCaptchaException, ParsingException, HttpResponseException {
String nextPageUrl = SoundcloudParsingHelper.getUsersFromApi(collector, apiUrl);
while (!nextPageUrl.isEmpty() && collector.getItems().size() < minItems) {
@@ -271,14 +278,16 @@ public static String getUsersFromApiMinItems(final int minItems,
/**
* Fetch the user items from the given API and commit each of them to the collector.
*
- * @return the next streams url, empty if don't have
+ * @return the next stream's url, empty if don't have
*/
@Nonnull
public static String getUsersFromApi(final ChannelInfoItemsCollector collector,
- final String apiUrl) throws IOException,
- ReCaptchaException, ParsingException {
- final String response = NewPipe.getDownloader().get(apiUrl, SoundCloud.getLocalization())
- .responseBody();
+ final String apiUrl)
+ throws IOException, ReCaptchaException, ParsingException, HttpResponseException {
+ final String response = NewPipe.getDownloader()
+ .get(apiUrl, SoundCloud.getLocalization())
+ .ensureSuccessResponseCode()
+ .responseBody();
final JsonObject responseObject;
try {
@@ -301,16 +310,16 @@ public static String getUsersFromApi(final ChannelInfoItemsCollector collector,
/**
* Fetch the streams from the given API and commit each of them to the collector.
*
- * This differ from {@link #getStreamsFromApi(StreamInfoItemsCollector, String)} in the sense
+ * This differs from {@link #getStreamsFromApi(StreamInfoItemsCollector, String)} in the sense
* that they will always get MIN_ITEMS or more items.
*
- * @param minItems the method will return only when it have extracted that many items
+ * @param minItems the method will return only when it has extracted this many items
* (equal or more)
*/
public static String getStreamsFromApiMinItems(final int minItems,
final StreamInfoItemsCollector collector,
- final String apiUrl) throws IOException,
- ReCaptchaException, ParsingException {
+ final String apiUrl)
+ throws IOException, ReCaptchaException, ParsingException, HttpResponseException {
String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl);
while (!nextPageUrl.isEmpty() && collector.getItems().size() < minItems) {
@@ -328,18 +337,16 @@ public static String getStreamsFromApiMinItems(final int minItems,
@Nonnull
public static String getStreamsFromApi(final StreamInfoItemsCollector collector,
final String apiUrl,
- final boolean charts) throws IOException,
- ReCaptchaException, ParsingException {
- final Response response = NewPipe.getDownloader().get(apiUrl, SoundCloud
- .getLocalization());
- if (response.responseCode() >= 400) {
- throw new IOException("Could not get streams from API, HTTP " + response
- .responseCode());
- }
+ final boolean charts)
+ throws IOException, ReCaptchaException, ParsingException, HttpResponseException {
+ final String responseBody = NewPipe.getDownloader()
+ .get(apiUrl, SoundCloud.getLocalization())
+ .ensureResponseCodeIsNotError()
+ .responseBody();
final JsonObject responseObject;
try {
- responseObject = JsonParser.object().from(response.responseBody());
+ responseObject = JsonParser.object().from(responseBody);
} catch (final JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}
@@ -370,23 +377,22 @@ private static String getNextPageUrl(@Nonnull final JsonObject response) {
}
public static String getStreamsFromApi(final StreamInfoItemsCollector collector,
- final String apiUrl) throws ReCaptchaException,
- ParsingException, IOException {
+ final String apiUrl)
+ throws ReCaptchaException, ParsingException, IOException, HttpResponseException {
return getStreamsFromApi(collector, apiUrl, false);
}
public static String getInfoItemsFromApi(final MultiInfoItemsCollector collector,
- final String apiUrl) throws ReCaptchaException,
- ParsingException, IOException {
- final Response response = NewPipe.getDownloader().get(apiUrl, SoundCloud.getLocalization());
- if (response.responseCode() >= 400) {
- throw new IOException("Could not get streams from API, HTTP "
- + response.responseCode());
- }
+ final String apiUrl)
+ throws ReCaptchaException, ParsingException, IOException, HttpResponseException {
+ final String responseBody = NewPipe.getDownloader()
+ .get(apiUrl, SoundCloud.getLocalization())
+ .ensureResponseCodeIsNotError()
+ .responseBody();
final JsonObject responseObject;
try {
- responseObject = JsonParser.object().from(response.responseBody());
+ responseObject = JsonParser.object().from(responseBody);
} catch (final JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}
From 3c8b3ab677db757f5233072a99d7a17abd5bebd3 Mon Sep 17 00:00:00 2001
From: AbsurdlyLongUsername
<22662897+absurdlylongusername@users.noreply.github.com>
Date: Tue, 7 Apr 2026 05:53:29 +0100
Subject: [PATCH 6/7] Catch HttpResponseException in to fix
SoundcloudChartsExtractorTest.testRelatedItems and testMoreRelatedItems
---
.../soundcloud/extractors/SoundcloudChartsExtractor.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChartsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChartsExtractor.java
index 3e2fb67433..cad45e946e 100644
--- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChartsExtractor.java
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChartsExtractor.java
@@ -8,6 +8,7 @@
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.exceptions.HttpResponseException;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.localization.ContentCountry;
@@ -56,7 +57,7 @@ public void onFetchPage(@Nonnull final Downloader downloader)
initialFetchNextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(
initialFetchCollector,
apiUrlWithRegion == null ? apiUrl : apiUrlWithRegion, true);
- } catch (final IOException e) {
+ } catch (final IOException | HttpResponseException e) {
// Request to other region may be geo-restricted.
// See https://github.com/TeamNewPipe/NewPipeExtractor/issues/537.
// We retry without the specified region.
From b2299dc4ad488d26c8deabcc3314e872d310f810 Mon Sep 17 00:00:00 2001
From: AbsurdlyLongUsername
<22662897+absurdlylongusername@users.noreply.github.com>
Date: Fri, 10 Apr 2026 16:14:57 +0100
Subject: [PATCH 7/7] Fix typo Use IllegalArgumentException instead of Runtime
Add bounds checking to all range methods Add doc to another method
---
.../schabi/newpipe/extractor/downloader/Response.java | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java b/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java
index 03f22ecfda..322be9f407 100644
--- a/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java
@@ -119,6 +119,7 @@ public Response ensureResponseCodeIsNotError() throws HttpResponseException {
*/
public Response ensureResponseCodeInRange(final int min, final int max)
throws HttpResponseException {
+ if (min > max) throw new IllegalArgumentException("min must be less than max");
if (responseCode() < min || responseCode() > max) {
throw new HttpResponseException(this);
}
@@ -131,7 +132,7 @@ public Response ensureResponseCodeNotInRange(
final Function