From 03aae47ce7388e1d002649e02be8d763955ad7c9 Mon Sep 17 00:00:00 2001 From: AbsurdlyLongUsername <22662897+absurdlylongusername@users.noreply.github.com> Date: Sat, 4 Apr 2026 23:56:52 +0100 Subject: [PATCH 1/7] [SoundCloud] Validate http response code in SoundcloudParsingHelper --- .../extractor/downloader/Response.java | 20 +++++++++++ .../exceptions/HttpResponseException.java | 15 ++++++++ .../soundcloud/SoundcloudParsingHelper.java | 34 +++++++++++------- .../newpipe/extractor/utils/HttpUtils.java | 36 +++++++++++++++++++ 4 files changed, 92 insertions(+), 13 deletions(-) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/HttpResponseException.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/utils/HttpUtils.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java b/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java index ac792dc756..87c3577ef4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java @@ -6,6 +6,9 @@ import java.util.List; import java.util.Map; +import org.schabi.newpipe.extractor.exceptions.HttpResponseException; +import org.schabi.newpipe.extractor.utils.HttpUtils; + /** * A Data class used to hold the results from requests made by the Downloader implementation. */ @@ -80,4 +83,21 @@ public String getHeader(final String name) { return null; } + // CHECKSTYLE:OFF + /** + * Helper function simply to make it easier to validate response code inline + * before getting the code/body/latestUrl/etc. + * Validates the response codes for the given {@link Response}, and throws a {@link HttpResponseException} if the code is invalid + * @see HttpUtils#validateResponseCode(Response, int...) + * @param validResponseCodes Expected valid response codes + * @return {@link this} response + * @throws HttpResponseException Thrown when the response code is not in {@code validResponseCodes}, + * or when {@code validResponseCodes} is empty and the code is a 4xx or 5xx error. + */ + // CHECKSTYLE:ON + public Response validateResponseCode(final int... validResponseCodes) + throws HttpResponseException { + HttpUtils.validateResponseCode(this, validResponseCodes); + return this; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/HttpResponseException.java b/extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/HttpResponseException.java new file mode 100644 index 0000000000..c07850a9d3 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/HttpResponseException.java @@ -0,0 +1,15 @@ +package org.schabi.newpipe.extractor.exceptions; + +import java.io.IOException; +import org.schabi.newpipe.extractor.downloader.Response; + +public class HttpResponseException extends IOException { + public HttpResponseException(final Response response) { + this("Error in HTTP Response for " + response.latestUrl() + "\n\t" + + response.responseCode() + " - " + response.responseMessage()); + } + + public HttpResponseException(final String message) { + super(message); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index aeff6bd363..26856fc8c8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -5,6 +5,7 @@ import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; +import static org.schabi.newpipe.extractor.utils.HttpUtils.validateResponseCode; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; @@ -12,7 +13,6 @@ import com.grack.nanojson.JsonParserException; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; import org.jsoup.select.Elements; import org.schabi.newpipe.extractor.MultiInfoItemsCollector; import org.schabi.newpipe.extractor.Image; @@ -105,8 +105,8 @@ public static synchronized String clientId() throws ExtractionException, IOExcep final Downloader dl = NewPipe.getDownloader(); - final Response download = dl.get("https://soundcloud.com"); - final String responseBody = download.responseBody(); + final Response downloadResponse = dl.get("https://soundcloud.com").validateResponseCode(); + final String responseBody = downloadResponse.responseBody(); final String clientIdPattern = ",client_id:\"(.*?)\""; final Document doc = Jsoup.parse(responseBody); @@ -117,11 +117,12 @@ public static synchronized String clientId() throws ExtractionException, IOExcep final var headers = Map.of("Range", List.of("bytes=0-50000")); - for (final Element element : possibleScripts) { + for (final var element : possibleScripts) { final String srcUrl = element.attr("src"); if (!isNullOrEmpty(srcUrl)) { try { clientId = Parser.matchGroup1(clientIdPattern, dl.get(srcUrl, headers) + .validateResponseCode() .responseBody()); return clientId; } catch (final RegexException ignored) { @@ -149,11 +150,13 @@ public static DateWrapper parseDate(final String uploadDate) throws ParsingExcep } } + // CHECKSTYLE:OFF /** - * Call the endpoint "/resolve" of the API.

+ * Call the endpoint "/resolve" of the API. *

- * See https://developers.soundcloud.com/docs/api/reference#resolve + * See https://web.archive.org/web/20170804051146/https://developers.soundcloud.com/docs/api/reference#resolve */ + // CHECKSTYLE:ON public static JsonObject resolveFor(@Nonnull final Downloader downloader, final String url) throws IOException, ExtractionException { final String apiUrl = SOUNDCLOUD_API_V2_URL + "resolve" @@ -178,10 +181,11 @@ public static JsonObject resolveFor(@Nonnull final Downloader downloader, final public static String resolveUrlWithEmbedPlayer(final String apiUrl) throws IOException, ReCaptchaException { - final String response = NewPipe.getDownloader().get("https://w.soundcloud.com/player/?url=" - + Utils.encodeUrlUtf8(apiUrl), SoundCloud.getLocalization()).responseBody(); - - return Jsoup.parse(response).select("link[rel=\"canonical\"]").first() + final var response = NewPipe.getDownloader().get("https://w.soundcloud.com/player/?url=" + + Utils.encodeUrlUtf8(apiUrl), SoundCloud.getLocalization()); + validateResponseCode(response); + final var responseBody = response.responseBody(); + return Jsoup.parse(responseBody).select("link[rel=\"canonical\"]").first() .attr("abs:href"); } @@ -190,6 +194,7 @@ public static String resolveUrlWithEmbedPlayer(final String apiUrl) throws IOExc * * @return the resolved id */ + // TODO: what makes this method different from the others? Don' they all return the same? public static String resolveIdWithWidgetApi(final String urlString) throws IOException, ParsingException { String fixedUrl = urlString; @@ -225,9 +230,12 @@ public static String resolveIdWithWidgetApi(final String urlString) throws IOExc final String widgetUrl = "https://api-widget.soundcloud.com/resolve?url=" + Utils.encodeUrlUtf8(url.toString()) + "&format=json&client_id=" + SoundcloudParsingHelper.clientId(); - final String response = NewPipe.getDownloader().get(widgetUrl, - SoundCloud.getLocalization()).responseBody(); - final JsonObject o = JsonParser.object().from(response); + + final var response = NewPipe.getDownloader().get(widgetUrl, + SoundCloud.getLocalization()); + + final var responseBody = response.validateResponseCode().responseBody(); + final JsonObject o = JsonParser.object().from(responseBody); return String.valueOf(JsonUtils.getValue(o, "id")); } catch (final JsonParserException e) { throw new ParsingException("Could not parse JSON response", e); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/HttpUtils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/HttpUtils.java new file mode 100644 index 0000000000..31c937ea09 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/HttpUtils.java @@ -0,0 +1,36 @@ +package org.schabi.newpipe.extractor.utils; + +import java.util.Arrays; + +import org.schabi.newpipe.extractor.downloader.Response; +import org.schabi.newpipe.extractor.exceptions.HttpResponseException; + +public final class HttpUtils { + + private HttpUtils() { + // Utility class, no instances allowed + } + + // CHECKSTYLE:OFF + /** + * Validates the response codes for the given {@link Response}, and throws + * a {@link HttpResponseException} if the code is invalid + * @param response The response to validate + * @param validResponseCodes Expected valid response codes + * @throws HttpResponseException Thrown when the response code is not in {@code validResponseCodes}, + * or when {@code validResponseCodes} is empty and the code is a 4xx or 5xx error. + */ + // CHECKSTYLE:ON + public static void validateResponseCode(final Response response, + final int... validResponseCodes) + throws HttpResponseException { + final int code = response.responseCode(); + final var throwError = (validResponseCodes == null || validResponseCodes.length == 0) + ? code >= 400 && code <= 599 + : Arrays.stream(validResponseCodes).noneMatch(c -> c == code); + + if (throwError) { + throw new HttpResponseException(response); + } + } +} From 59e94ee627ff20dcef43923bd89c8a946243685e Mon Sep 17 00:00:00 2001 From: AbsurdlyLongUsername <22662897+absurdlylongusername@users.noreply.github.com> Date: Sat, 4 Apr 2026 23:57:04 +0100 Subject: [PATCH 2/7] Move validateResponseCode to Response Make HttpResponseException extend ExtractionException Remove HttpUtils --- .../extractor/downloader/Response.java | 29 +++++++++++++-- .../exceptions/HttpResponseException.java | 3 +- .../soundcloud/SoundcloudParsingHelper.java | 5 +-- .../newpipe/extractor/utils/HttpUtils.java | 36 ------------------- 4 files changed, 30 insertions(+), 43 deletions(-) delete mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/utils/HttpUtils.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java b/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java index 87c3577ef4..593b8a78e5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java @@ -2,12 +2,12 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; import org.schabi.newpipe.extractor.exceptions.HttpResponseException; -import org.schabi.newpipe.extractor.utils.HttpUtils; /** * A Data class used to hold the results from requests made by the Downloader implementation. @@ -33,6 +33,28 @@ public Response(final int responseCode, this.latestUrl = latestUrl; } + /** + * Validates the response codes for the given {@link Response}, and throws + * a {@link HttpResponseException} if the code is invalid + * @param response The response to validate + * @param validResponseCodes Expected valid response codes + * @throws HttpResponseException Thrown when the response code is not in {@code validResponseCodes}, + * or when {@code validResponseCodes} is empty and the code is a 4xx or 5xx error. + */ + // CHECKSTYLE:ON + public static void validateResponseCode(final Response response, + final int... validResponseCodes) + throws HttpResponseException { + final int code = response.responseCode(); + final var throwError = (validResponseCodes == null || validResponseCodes.length == 0) + ? code >= 400 && code <= 599 + : Arrays.stream(validResponseCodes).noneMatch(c -> c == code); + + if (throwError) { + throw new HttpResponseException(response); + } + } + public int responseCode() { return responseCode; } @@ -83,12 +105,13 @@ public String getHeader(final String name) { return null; } + // CHECKSTYLE:OFF /** * Helper function simply to make it easier to validate response code inline * before getting the code/body/latestUrl/etc. * Validates the response codes for the given {@link Response}, and throws a {@link HttpResponseException} if the code is invalid - * @see HttpUtils#validateResponseCode(Response, int...) + * @see Response#validateResponseCode(Response, int...) * @param validResponseCodes Expected valid response codes * @return {@link this} response * @throws HttpResponseException Thrown when the response code is not in {@code validResponseCodes}, @@ -97,7 +120,7 @@ public String getHeader(final String name) { // CHECKSTYLE:ON public Response validateResponseCode(final int... validResponseCodes) throws HttpResponseException { - HttpUtils.validateResponseCode(this, validResponseCodes); + validateResponseCode(this, validResponseCodes); return this; } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/HttpResponseException.java b/extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/HttpResponseException.java index c07850a9d3..51b1f77c66 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/HttpResponseException.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/HttpResponseException.java @@ -1,9 +1,8 @@ package org.schabi.newpipe.extractor.exceptions; -import java.io.IOException; import org.schabi.newpipe.extractor.downloader.Response; -public class HttpResponseException extends IOException { +public class HttpResponseException extends ExtractionException { public HttpResponseException(final Response response) { this("Error in HTTP Response for " + response.latestUrl() + "\n\t" + response.responseCode() + " - " + response.responseMessage()); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index 26856fc8c8..17f3a49edf 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -5,7 +5,7 @@ import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; -import static org.schabi.newpipe.extractor.utils.HttpUtils.validateResponseCode; +import static org.schabi.newpipe.extractor.downloader.Response.validateResponseCode; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; @@ -21,6 +21,7 @@ import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.HttpResponseException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.localization.DateWrapper; @@ -179,7 +180,7 @@ public static JsonObject resolveFor(@Nonnull final Downloader downloader, final * @return the url resolved */ public static String resolveUrlWithEmbedPlayer(final String apiUrl) throws IOException, - ReCaptchaException { + ReCaptchaException, HttpResponseException { final var response = NewPipe.getDownloader().get("https://w.soundcloud.com/player/?url=" + Utils.encodeUrlUtf8(apiUrl), SoundCloud.getLocalization()); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/HttpUtils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/HttpUtils.java deleted file mode 100644 index 31c937ea09..0000000000 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/HttpUtils.java +++ /dev/null @@ -1,36 +0,0 @@ -package org.schabi.newpipe.extractor.utils; - -import java.util.Arrays; - -import org.schabi.newpipe.extractor.downloader.Response; -import org.schabi.newpipe.extractor.exceptions.HttpResponseException; - -public final class HttpUtils { - - private HttpUtils() { - // Utility class, no instances allowed - } - - // CHECKSTYLE:OFF - /** - * Validates the response codes for the given {@link Response}, and throws - * a {@link HttpResponseException} if the code is invalid - * @param response The response to validate - * @param validResponseCodes Expected valid response codes - * @throws HttpResponseException Thrown when the response code is not in {@code validResponseCodes}, - * or when {@code validResponseCodes} is empty and the code is a 4xx or 5xx error. - */ - // CHECKSTYLE:ON - public static void validateResponseCode(final Response response, - final int... validResponseCodes) - throws HttpResponseException { - final int code = response.responseCode(); - final var throwError = (validResponseCodes == null || validResponseCodes.length == 0) - ? code >= 400 && code <= 599 - : Arrays.stream(validResponseCodes).noneMatch(c -> c == code); - - if (throwError) { - throw new HttpResponseException(response); - } - } -} From 6831369cec2d3184ed4aa0f465a1b447f5919ea2 Mon Sep 17 00:00:00 2001 From: AbsurdlyLongUsername <22662897+absurdlylongusername@users.noreply.github.com> Date: Sat, 4 Apr 2026 23:57:11 +0100 Subject: [PATCH 3/7] Fix checkstyle off --- .../java/org/schabi/newpipe/extractor/downloader/Response.java | 1 + 1 file changed, 1 insertion(+) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java b/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java index 593b8a78e5..b759fac2cc 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java @@ -33,6 +33,7 @@ public Response(final int responseCode, this.latestUrl = latestUrl; } + // CHECKSTYLE:OFF /** * Validates the response codes for the given {@link Response}, and throws * a {@link HttpResponseException} if the code is invalid From ad9f4f48c42e41c96405477e7a77e9a3d8f86861 Mon Sep 17 00:00:00 2001 From: AbsurdlyLongUsername <22662897+absurdlylongusername@users.noreply.github.com> Date: Tue, 7 Apr 2026 05:41:50 +0100 Subject: [PATCH 4/7] Remove validateResponseCodes Add various helper methods for validating response codes instead --- .../extractor/downloader/Response.java | 157 ++++++++++++++---- 1 file changed, 121 insertions(+), 36 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java b/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java index b759fac2cc..03f22ecfda 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java @@ -2,16 +2,17 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; -import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.function.Function; import org.schabi.newpipe.extractor.exceptions.HttpResponseException; /** * A Data class used to hold the results from requests made by the Downloader implementation. */ +@SuppressWarnings("checkstyle:NeedBraces") public class Response { private final int responseCode; private final String responseMessage; @@ -33,29 +34,6 @@ public Response(final int responseCode, this.latestUrl = latestUrl; } - // CHECKSTYLE:OFF - /** - * Validates the response codes for the given {@link Response}, and throws - * a {@link HttpResponseException} if the code is invalid - * @param response The response to validate - * @param validResponseCodes Expected valid response codes - * @throws HttpResponseException Thrown when the response code is not in {@code validResponseCodes}, - * or when {@code validResponseCodes} is empty and the code is a 4xx or 5xx error. - */ - // CHECKSTYLE:ON - public static void validateResponseCode(final Response response, - final int... validResponseCodes) - throws HttpResponseException { - final int code = response.responseCode(); - final var throwError = (validResponseCodes == null || validResponseCodes.length == 0) - ? code >= 400 && code <= 599 - : Arrays.stream(validResponseCodes).noneMatch(c -> c == code); - - if (throwError) { - throw new HttpResponseException(response); - } - } - public int responseCode() { return responseCode; } @@ -107,21 +85,128 @@ public String getHeader(final String name) { return null; } - // CHECKSTYLE:OFF /** - * Helper function simply to make it easier to validate response code inline - * before getting the code/body/latestUrl/etc. - * Validates the response codes for the given {@link Response}, and throws a {@link HttpResponseException} if the code is invalid - * @see Response#validateResponseCode(Response, int...) - * @param validResponseCodes Expected valid response codes - * @return {@link this} response - * @throws HttpResponseException Thrown when the response code is not in {@code validResponseCodes}, - * or when {@code validResponseCodes} is empty and the code is a 4xx or 5xx error. + * Ensure the response code is 2xx + * @return this {@code Response} + * @throws HttpResponseException if the response code is not 2xx + */ + public Response ensureSuccessResponseCode() throws HttpResponseException { + return ensureResponseCodeInRange(200, 299); + } + + /** + * Ensure the response code is 3xx + * @return this {@code Response} + * @throws HttpResponseException if the response code is not 3xx + */ + public Response ensureRedirectResponseCode() throws HttpResponseException { + return ensureResponseCodeInRange(300, 399); + } + + /** + * Ensure the response code is not 4xx or 5xx + * @return this {@code Response} + * @throws HttpResponseException if the response code is client or server error + */ + public Response ensureResponseCodeIsNotError() throws HttpResponseException { + return ensureResponseCodeNotInRange(400, 599); + } + + /** + * Ensure the HTTP response code is within range of min and max inclusive + * @return this Response + * @throws HttpResponseException if the code is outside the range + */ + public Response ensureResponseCodeInRange(final int min, final int max) + throws HttpResponseException { + if (responseCode() < min || responseCode() > max) { + throw new HttpResponseException(this); + } + return this; + } + + public Response ensureResponseCodeNotInRange( + final int min, + final int max, + final Function errorSupplier + ) + throws HttpResponseException { + if (min > max) throw new RuntimeException("min must be less than max"); + if (responseCode() >= min && responseCode() <= max) { + throw errorSupplier.apply(this); + } + return this; + } + + public Response ensureResponseCodeNotInRange(final int min, final int max) + throws HttpResponseException { + return ensureResponseCodeNotInRange(min, max, HttpResponseException::new); + } + + /** + * Throw exception if response code is a 4xx client error + * @return this {@code Response} + * @throws HttpResponseException if the response code is 4xx + */ + public Response throwIfClientError( + final Function errorSupplier + ) + throws HttpResponseException { + return throwIfResponseCodeInRange(400, 499, errorSupplier); + } + + /** + * Throw exception if response code is a 4xx client error + * @return this {@code Response} + * @throws HttpResponseException if the response code is 4xx + */ + public Response throwIfClientError() + throws HttpResponseException { + return throwIfClientError(HttpResponseException::new); + } + + /** + * Throw exception if response code is a 5xx server error + * @return this {@code Response} + * @throws HttpResponseException if the response code is 4xx */ - // CHECKSTYLE:ON - public Response validateResponseCode(final int... validResponseCodes) + public Response throwIfServerError( + final Function errorSupplier + ) throws HttpResponseException { - validateResponseCode(this, validResponseCodes); + return throwIfResponseCodeInRange(500, 599, errorSupplier); + } + + public Response throwIfServerError() + throws HttpResponseException { + return throwIfServerError(HttpResponseException::new); + } + + public Response throwIfResponseCode( + final int errorCode, + final Function errorSupplier + ) + throws HttpResponseException { + if (responseCode() == errorCode) { + throw errorSupplier.apply(this); + } return this; } + + public Response throwIfResponseCode(final int errorCode) throws HttpResponseException { + return throwIfResponseCode(errorCode, HttpResponseException::new); + } + + public Response throwIfResponseCodeInRange(final int min, + final int max, + final Function errorSupplier) + throws HttpResponseException { + return ensureResponseCodeNotInRange(min, max, errorSupplier); + } + + public Response throwIfResponseCodeInRange(final int min, + final int max) throws HttpResponseException { + return throwIfResponseCodeInRange(min, max, HttpResponseException::new); + } } From a437a5dad355b0c4b21665e2ec59d5a75876cd18 Mon Sep 17 00:00:00 2001 From: AbsurdlyLongUsername <22662897+absurdlylongusername@users.noreply.github.com> Date: Tue, 7 Apr 2026 05:48:11 +0100 Subject: [PATCH 5/7] Add validation to all http calls in SoundcloudParsingHelper --- .../soundcloud/SoundcloudParsingHelper.java | 94 ++++++++++--------- 1 file changed, 50 insertions(+), 44 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index 17f3a49edf..9ba84e8ccb 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -5,7 +5,6 @@ import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; -import static org.schabi.newpipe.extractor.downloader.Response.validateResponseCode; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; @@ -19,7 +18,6 @@ import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.channel.ChannelInfoItemsCollector; import org.schabi.newpipe.extractor.downloader.Downloader; -import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.HttpResponseException; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -106,8 +104,9 @@ public static synchronized String clientId() throws ExtractionException, IOExcep final Downloader dl = NewPipe.getDownloader(); - final Response downloadResponse = dl.get("https://soundcloud.com").validateResponseCode(); - final String responseBody = downloadResponse.responseBody(); + final String responseBody = dl.get("https://soundcloud.com") + .ensureSuccessResponseCode() + .responseBody(); final String clientIdPattern = ",client_id:\"(.*?)\""; final Document doc = Jsoup.parse(responseBody); @@ -123,7 +122,7 @@ public static synchronized String clientId() throws ExtractionException, IOExcep if (!isNullOrEmpty(srcUrl)) { try { clientId = Parser.matchGroup1(clientIdPattern, dl.get(srcUrl, headers) - .validateResponseCode() + .ensureSuccessResponseCode() .responseBody()); return clientId; } catch (final RegexException ignored) { @@ -155,7 +154,7 @@ public static DateWrapper parseDate(final String uploadDate) throws ParsingExcep /** * Call the endpoint "/resolve" of the API. *

- * See https://web.archive.org/web/20170804051146/https://developers.soundcloud.com/docs/api/reference#resolve + * See docs */ // CHECKSTYLE:ON public static JsonObject resolveFor(@Nonnull final Downloader downloader, final String url) @@ -166,7 +165,8 @@ public static JsonObject resolveFor(@Nonnull final Downloader downloader, final try { final String response = downloader.get(apiUrl, SoundCloud.getLocalization()) - .responseBody(); + .ensureSuccessResponseCode() + .responseBody(); return JsonParser.object().from(response); } catch (final JsonParserException e) { throw new ParsingException("Could not parse json response", e); @@ -184,7 +184,7 @@ public static String resolveUrlWithEmbedPlayer(final String apiUrl) throws IOExc final var response = NewPipe.getDownloader().get("https://w.soundcloud.com/player/?url=" + Utils.encodeUrlUtf8(apiUrl), SoundCloud.getLocalization()); - validateResponseCode(response); + response.ensureSuccessResponseCode(); final var responseBody = response.responseBody(); return Jsoup.parse(responseBody).select("link[rel=\"canonical\"]").first() .attr("abs:href"); @@ -195,7 +195,6 @@ public static String resolveUrlWithEmbedPlayer(final String apiUrl) throws IOExc * * @return the resolved id */ - // TODO: what makes this method different from the others? Don' they all return the same? public static String resolveIdWithWidgetApi(final String urlString) throws IOException, ParsingException { String fixedUrl = urlString; @@ -204,7 +203,14 @@ public static String resolveIdWithWidgetApi(final String urlString) throws IOExc if (ON_URL_PATTERN.matcher(fixedUrl).find()) { try { - fixedUrl = NewPipe.getDownloader().head(fixedUrl).latestUrl(); + fixedUrl = NewPipe.getDownloader() + .head(fixedUrl) + .throwIfServerError() + .latestUrl(); + // Soundcloud returns HTTP 403 but it still redirects, + // so can get the redirected url via latestUrl regardless. + // This why only throw for 5xx and not 4xx. + // remove tracking params which are in the query string fixedUrl = fixedUrl.split("\\?")[0]; } catch (final ExtractionException e) { @@ -232,10 +238,11 @@ public static String resolveIdWithWidgetApi(final String urlString) throws IOExc + Utils.encodeUrlUtf8(url.toString()) + "&format=json&client_id=" + SoundcloudParsingHelper.clientId(); - final var response = NewPipe.getDownloader().get(widgetUrl, - SoundCloud.getLocalization()); + final var responseBody = NewPipe.getDownloader() + .get(widgetUrl, SoundCloud.getLocalization()) + .ensureSuccessResponseCode() + .responseBody(); - final var responseBody = response.validateResponseCode().responseBody(); final JsonObject o = JsonParser.object().from(responseBody); return String.valueOf(JsonUtils.getValue(o, "id")); } catch (final JsonParserException e) { @@ -249,16 +256,16 @@ public static String resolveIdWithWidgetApi(final String urlString) throws IOExc /** * Fetch the users from the given API and commit each of them to the collector. *

- * This differ from {@link #getUsersFromApi(ChannelInfoItemsCollector, String)} in the sense + * This differs from {@link #getUsersFromApi(ChannelInfoItemsCollector, String)} in the sense * that they will always get MIN_ITEMS or more. * - * @param minItems the method will return only when it have extracted that many items + * @param minItems the method will return only when it has extracted this many items * (equal or more) */ public static String getUsersFromApiMinItems(final int minItems, final ChannelInfoItemsCollector collector, final String apiUrl) throws IOException, - ReCaptchaException, ParsingException { + ReCaptchaException, ParsingException, HttpResponseException { String nextPageUrl = SoundcloudParsingHelper.getUsersFromApi(collector, apiUrl); while (!nextPageUrl.isEmpty() && collector.getItems().size() < minItems) { @@ -271,14 +278,16 @@ public static String getUsersFromApiMinItems(final int minItems, /** * Fetch the user items from the given API and commit each of them to the collector. * - * @return the next streams url, empty if don't have + * @return the next stream's url, empty if don't have */ @Nonnull public static String getUsersFromApi(final ChannelInfoItemsCollector collector, - final String apiUrl) throws IOException, - ReCaptchaException, ParsingException { - final String response = NewPipe.getDownloader().get(apiUrl, SoundCloud.getLocalization()) - .responseBody(); + final String apiUrl) + throws IOException, ReCaptchaException, ParsingException, HttpResponseException { + final String response = NewPipe.getDownloader() + .get(apiUrl, SoundCloud.getLocalization()) + .ensureSuccessResponseCode() + .responseBody(); final JsonObject responseObject; try { @@ -301,16 +310,16 @@ public static String getUsersFromApi(final ChannelInfoItemsCollector collector, /** * Fetch the streams from the given API and commit each of them to the collector. *

- * This differ from {@link #getStreamsFromApi(StreamInfoItemsCollector, String)} in the sense + * This differs from {@link #getStreamsFromApi(StreamInfoItemsCollector, String)} in the sense * that they will always get MIN_ITEMS or more items. * - * @param minItems the method will return only when it have extracted that many items + * @param minItems the method will return only when it has extracted this many items * (equal or more) */ public static String getStreamsFromApiMinItems(final int minItems, final StreamInfoItemsCollector collector, - final String apiUrl) throws IOException, - ReCaptchaException, ParsingException { + final String apiUrl) + throws IOException, ReCaptchaException, ParsingException, HttpResponseException { String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl); while (!nextPageUrl.isEmpty() && collector.getItems().size() < minItems) { @@ -328,18 +337,16 @@ public static String getStreamsFromApiMinItems(final int minItems, @Nonnull public static String getStreamsFromApi(final StreamInfoItemsCollector collector, final String apiUrl, - final boolean charts) throws IOException, - ReCaptchaException, ParsingException { - final Response response = NewPipe.getDownloader().get(apiUrl, SoundCloud - .getLocalization()); - if (response.responseCode() >= 400) { - throw new IOException("Could not get streams from API, HTTP " + response - .responseCode()); - } + final boolean charts) + throws IOException, ReCaptchaException, ParsingException, HttpResponseException { + final String responseBody = NewPipe.getDownloader() + .get(apiUrl, SoundCloud.getLocalization()) + .ensureResponseCodeIsNotError() + .responseBody(); final JsonObject responseObject; try { - responseObject = JsonParser.object().from(response.responseBody()); + responseObject = JsonParser.object().from(responseBody); } catch (final JsonParserException e) { throw new ParsingException("Could not parse json response", e); } @@ -370,23 +377,22 @@ private static String getNextPageUrl(@Nonnull final JsonObject response) { } public static String getStreamsFromApi(final StreamInfoItemsCollector collector, - final String apiUrl) throws ReCaptchaException, - ParsingException, IOException { + final String apiUrl) + throws ReCaptchaException, ParsingException, IOException, HttpResponseException { return getStreamsFromApi(collector, apiUrl, false); } public static String getInfoItemsFromApi(final MultiInfoItemsCollector collector, - final String apiUrl) throws ReCaptchaException, - ParsingException, IOException { - final Response response = NewPipe.getDownloader().get(apiUrl, SoundCloud.getLocalization()); - if (response.responseCode() >= 400) { - throw new IOException("Could not get streams from API, HTTP " - + response.responseCode()); - } + final String apiUrl) + throws ReCaptchaException, ParsingException, IOException, HttpResponseException { + final String responseBody = NewPipe.getDownloader() + .get(apiUrl, SoundCloud.getLocalization()) + .ensureResponseCodeIsNotError() + .responseBody(); final JsonObject responseObject; try { - responseObject = JsonParser.object().from(response.responseBody()); + responseObject = JsonParser.object().from(responseBody); } catch (final JsonParserException e) { throw new ParsingException("Could not parse json response", e); } From 3c8b3ab677db757f5233072a99d7a17abd5bebd3 Mon Sep 17 00:00:00 2001 From: AbsurdlyLongUsername <22662897+absurdlylongusername@users.noreply.github.com> Date: Tue, 7 Apr 2026 05:53:29 +0100 Subject: [PATCH 6/7] Catch HttpResponseException in to fix SoundcloudChartsExtractorTest.testRelatedItems and testMoreRelatedItems --- .../soundcloud/extractors/SoundcloudChartsExtractor.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChartsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChartsExtractor.java index 3e2fb67433..cad45e946e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChartsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChartsExtractor.java @@ -8,6 +8,7 @@ import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.HttpResponseException; import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.localization.ContentCountry; @@ -56,7 +57,7 @@ public void onFetchPage(@Nonnull final Downloader downloader) initialFetchNextPageUrl = SoundcloudParsingHelper.getStreamsFromApi( initialFetchCollector, apiUrlWithRegion == null ? apiUrl : apiUrlWithRegion, true); - } catch (final IOException e) { + } catch (final IOException | HttpResponseException e) { // Request to other region may be geo-restricted. // See https://github.com/TeamNewPipe/NewPipeExtractor/issues/537. // We retry without the specified region. From b2299dc4ad488d26c8deabcc3314e872d310f810 Mon Sep 17 00:00:00 2001 From: AbsurdlyLongUsername <22662897+absurdlylongusername@users.noreply.github.com> Date: Fri, 10 Apr 2026 16:14:57 +0100 Subject: [PATCH 7/7] Fix typo Use IllegalArgumentException instead of Runtime Add bounds checking to all range methods Add doc to another method --- .../schabi/newpipe/extractor/downloader/Response.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java b/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java index 03f22ecfda..322be9f407 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java @@ -119,6 +119,7 @@ public Response ensureResponseCodeIsNotError() throws HttpResponseException { */ public Response ensureResponseCodeInRange(final int min, final int max) throws HttpResponseException { + if (min > max) throw new IllegalArgumentException("min must be less than max"); if (responseCode() < min || responseCode() > max) { throw new HttpResponseException(this); } @@ -131,7 +132,7 @@ public Response ensureResponseCodeNotInRange( final Function errorSupplier ) throws HttpResponseException { - if (min > max) throw new RuntimeException("min must be less than max"); + if (min > max) throw new IllegalArgumentException("min must be less than max"); if (responseCode() >= min && responseCode() <= max) { throw errorSupplier.apply(this); } @@ -168,7 +169,7 @@ public Response throwIfClientError() /** * Throw exception if response code is a 5xx server error * @return this {@code Response} - * @throws HttpResponseException if the response code is 4xx + * @throws HttpResponseException if the response code is 5xx */ public Response throwIfServerError( final Function errorSupplier @@ -177,6 +178,11 @@ public Response throwIfServerError( return throwIfResponseCodeInRange(500, 599, errorSupplier); } + /** + * Throw exception if response code is a 5xx server error + * @return this {@code Response} + * @throws HttpResponseException if the response code is 5xx + */ public Response throwIfServerError() throws HttpResponseException { return throwIfServerError(HttpResponseException::new);