diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java b/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java index ac792dc756..322be9f407 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java @@ -5,10 +5,14 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.function.Function; + +import org.schabi.newpipe.extractor.exceptions.HttpResponseException; /** * A Data class used to hold the results from requests made by the Downloader implementation. */ +@SuppressWarnings("checkstyle:NeedBraces") public class Response { private final int responseCode; private final String responseMessage; @@ -80,4 +84,135 @@ public String getHeader(final String name) { return null; } + + /** + * Ensure the response code is 2xx + * @return this {@code Response} + * @throws HttpResponseException if the response code is not 2xx + */ + public Response ensureSuccessResponseCode() throws HttpResponseException { + return ensureResponseCodeInRange(200, 299); + } + + /** + * Ensure the response code is 3xx + * @return this {@code Response} + * @throws HttpResponseException if the response code is not 3xx + */ + public Response ensureRedirectResponseCode() throws HttpResponseException { + return ensureResponseCodeInRange(300, 399); + } + + /** + * Ensure the response code is not 4xx or 5xx + * @return this {@code Response} + * @throws HttpResponseException if the response code is client or server error + */ + public Response ensureResponseCodeIsNotError() throws HttpResponseException { + return ensureResponseCodeNotInRange(400, 599); + } + + /** + * Ensure the HTTP response code is within range of min and max inclusive + * @return this Response + * @throws HttpResponseException if the code is outside the range + */ + public Response ensureResponseCodeInRange(final int min, final int max) + throws HttpResponseException { + if (min > max) throw new IllegalArgumentException("min must be less than max"); + if (responseCode() < min || responseCode() > max) { + throw new HttpResponseException(this); + } + return this; + } + + public Response ensureResponseCodeNotInRange( + final int min, + final int max, + final Function errorSupplier + ) + throws HttpResponseException { + if (min > max) throw new IllegalArgumentException("min must be less than max"); + if (responseCode() >= min && responseCode() <= max) { + throw errorSupplier.apply(this); + } + return this; + } + + public Response ensureResponseCodeNotInRange(final int min, final int max) + throws HttpResponseException { + return ensureResponseCodeNotInRange(min, max, HttpResponseException::new); + } + + /** + * Throw exception if response code is a 4xx client error + * @return this {@code Response} + * @throws HttpResponseException if the response code is 4xx + */ + public Response throwIfClientError( + final Function errorSupplier + ) + throws HttpResponseException { + return throwIfResponseCodeInRange(400, 499, errorSupplier); + } + + /** + * Throw exception if response code is a 4xx client error + * @return this {@code Response} + * @throws HttpResponseException if the response code is 4xx + */ + public Response throwIfClientError() + throws HttpResponseException { + return throwIfClientError(HttpResponseException::new); + } + + /** + * Throw exception if response code is a 5xx server error + * @return this {@code Response} + * @throws HttpResponseException if the response code is 5xx + */ + public Response throwIfServerError( + final Function errorSupplier + ) + throws HttpResponseException { + return throwIfResponseCodeInRange(500, 599, errorSupplier); + } + + /** + * Throw exception if response code is a 5xx server error + * @return this {@code Response} + * @throws HttpResponseException if the response code is 5xx + */ + public Response throwIfServerError() + throws HttpResponseException { + return throwIfServerError(HttpResponseException::new); + } + + public Response throwIfResponseCode( + final int errorCode, + final Function errorSupplier + ) + throws HttpResponseException { + if (responseCode() == errorCode) { + throw errorSupplier.apply(this); + } + return this; + } + + public Response throwIfResponseCode(final int errorCode) throws HttpResponseException { + return throwIfResponseCode(errorCode, HttpResponseException::new); + } + + public Response throwIfResponseCodeInRange(final int min, + final int max, + final Function errorSupplier) + throws HttpResponseException { + return ensureResponseCodeNotInRange(min, max, errorSupplier); + } + + public Response throwIfResponseCodeInRange(final int min, + final int max) throws HttpResponseException { + return throwIfResponseCodeInRange(min, max, HttpResponseException::new); + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/HttpResponseException.java b/extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/HttpResponseException.java new file mode 100644 index 0000000000..51b1f77c66 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/HttpResponseException.java @@ -0,0 +1,14 @@ +package org.schabi.newpipe.extractor.exceptions; + +import org.schabi.newpipe.extractor.downloader.Response; + +public class HttpResponseException extends ExtractionException { + public HttpResponseException(final Response response) { + this("Error in HTTP Response for " + response.latestUrl() + "\n\t" + + response.responseCode() + " - " + response.responseMessage()); + } + + public HttpResponseException(final String message) { + super(message); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index aeff6bd363..9ba84e8ccb 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -12,15 +12,14 @@ import com.grack.nanojson.JsonParserException; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; import org.jsoup.select.Elements; import org.schabi.newpipe.extractor.MultiInfoItemsCollector; import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.channel.ChannelInfoItemsCollector; import org.schabi.newpipe.extractor.downloader.Downloader; -import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.HttpResponseException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.localization.DateWrapper; @@ -105,8 +104,9 @@ public static synchronized String clientId() throws ExtractionException, IOExcep final Downloader dl = NewPipe.getDownloader(); - final Response download = dl.get("https://soundcloud.com"); - final String responseBody = download.responseBody(); + final String responseBody = dl.get("https://soundcloud.com") + .ensureSuccessResponseCode() + .responseBody(); final String clientIdPattern = ",client_id:\"(.*?)\""; final Document doc = Jsoup.parse(responseBody); @@ -117,11 +117,12 @@ public static synchronized String clientId() throws ExtractionException, IOExcep final var headers = Map.of("Range", List.of("bytes=0-50000")); - for (final Element element : possibleScripts) { + for (final var element : possibleScripts) { final String srcUrl = element.attr("src"); if (!isNullOrEmpty(srcUrl)) { try { clientId = Parser.matchGroup1(clientIdPattern, dl.get(srcUrl, headers) + .ensureSuccessResponseCode() .responseBody()); return clientId; } catch (final RegexException ignored) { @@ -149,11 +150,13 @@ public static DateWrapper parseDate(final String uploadDate) throws ParsingExcep } } + // CHECKSTYLE:OFF /** - * Call the endpoint "/resolve" of the API.

+ * Call the endpoint "/resolve" of the API. *

- * See https://developers.soundcloud.com/docs/api/reference#resolve + * See docs */ + // CHECKSTYLE:ON public static JsonObject resolveFor(@Nonnull final Downloader downloader, final String url) throws IOException, ExtractionException { final String apiUrl = SOUNDCLOUD_API_V2_URL + "resolve" @@ -162,7 +165,8 @@ public static JsonObject resolveFor(@Nonnull final Downloader downloader, final try { final String response = downloader.get(apiUrl, SoundCloud.getLocalization()) - .responseBody(); + .ensureSuccessResponseCode() + .responseBody(); return JsonParser.object().from(response); } catch (final JsonParserException e) { throw new ParsingException("Could not parse json response", e); @@ -176,12 +180,13 @@ public static JsonObject resolveFor(@Nonnull final Downloader downloader, final * @return the url resolved */ public static String resolveUrlWithEmbedPlayer(final String apiUrl) throws IOException, - ReCaptchaException { + ReCaptchaException, HttpResponseException { - final String response = NewPipe.getDownloader().get("https://w.soundcloud.com/player/?url=" - + Utils.encodeUrlUtf8(apiUrl), SoundCloud.getLocalization()).responseBody(); - - return Jsoup.parse(response).select("link[rel=\"canonical\"]").first() + final var response = NewPipe.getDownloader().get("https://w.soundcloud.com/player/?url=" + + Utils.encodeUrlUtf8(apiUrl), SoundCloud.getLocalization()); + response.ensureSuccessResponseCode(); + final var responseBody = response.responseBody(); + return Jsoup.parse(responseBody).select("link[rel=\"canonical\"]").first() .attr("abs:href"); } @@ -198,7 +203,14 @@ public static String resolveIdWithWidgetApi(final String urlString) throws IOExc if (ON_URL_PATTERN.matcher(fixedUrl).find()) { try { - fixedUrl = NewPipe.getDownloader().head(fixedUrl).latestUrl(); + fixedUrl = NewPipe.getDownloader() + .head(fixedUrl) + .throwIfServerError() + .latestUrl(); + // Soundcloud returns HTTP 403 but it still redirects, + // so can get the redirected url via latestUrl regardless. + // This why only throw for 5xx and not 4xx. + // remove tracking params which are in the query string fixedUrl = fixedUrl.split("\\?")[0]; } catch (final ExtractionException e) { @@ -225,9 +237,13 @@ public static String resolveIdWithWidgetApi(final String urlString) throws IOExc final String widgetUrl = "https://api-widget.soundcloud.com/resolve?url=" + Utils.encodeUrlUtf8(url.toString()) + "&format=json&client_id=" + SoundcloudParsingHelper.clientId(); - final String response = NewPipe.getDownloader().get(widgetUrl, - SoundCloud.getLocalization()).responseBody(); - final JsonObject o = JsonParser.object().from(response); + + final var responseBody = NewPipe.getDownloader() + .get(widgetUrl, SoundCloud.getLocalization()) + .ensureSuccessResponseCode() + .responseBody(); + + final JsonObject o = JsonParser.object().from(responseBody); return String.valueOf(JsonUtils.getValue(o, "id")); } catch (final JsonParserException e) { throw new ParsingException("Could not parse JSON response", e); @@ -240,16 +256,16 @@ public static String resolveIdWithWidgetApi(final String urlString) throws IOExc /** * Fetch the users from the given API and commit each of them to the collector. *

- * This differ from {@link #getUsersFromApi(ChannelInfoItemsCollector, String)} in the sense + * This differs from {@link #getUsersFromApi(ChannelInfoItemsCollector, String)} in the sense * that they will always get MIN_ITEMS or more. * - * @param minItems the method will return only when it have extracted that many items + * @param minItems the method will return only when it has extracted this many items * (equal or more) */ public static String getUsersFromApiMinItems(final int minItems, final ChannelInfoItemsCollector collector, final String apiUrl) throws IOException, - ReCaptchaException, ParsingException { + ReCaptchaException, ParsingException, HttpResponseException { String nextPageUrl = SoundcloudParsingHelper.getUsersFromApi(collector, apiUrl); while (!nextPageUrl.isEmpty() && collector.getItems().size() < minItems) { @@ -262,14 +278,16 @@ public static String getUsersFromApiMinItems(final int minItems, /** * Fetch the user items from the given API and commit each of them to the collector. * - * @return the next streams url, empty if don't have + * @return the next stream's url, empty if don't have */ @Nonnull public static String getUsersFromApi(final ChannelInfoItemsCollector collector, - final String apiUrl) throws IOException, - ReCaptchaException, ParsingException { - final String response = NewPipe.getDownloader().get(apiUrl, SoundCloud.getLocalization()) - .responseBody(); + final String apiUrl) + throws IOException, ReCaptchaException, ParsingException, HttpResponseException { + final String response = NewPipe.getDownloader() + .get(apiUrl, SoundCloud.getLocalization()) + .ensureSuccessResponseCode() + .responseBody(); final JsonObject responseObject; try { @@ -292,16 +310,16 @@ public static String getUsersFromApi(final ChannelInfoItemsCollector collector, /** * Fetch the streams from the given API and commit each of them to the collector. *

- * This differ from {@link #getStreamsFromApi(StreamInfoItemsCollector, String)} in the sense + * This differs from {@link #getStreamsFromApi(StreamInfoItemsCollector, String)} in the sense * that they will always get MIN_ITEMS or more items. * - * @param minItems the method will return only when it have extracted that many items + * @param minItems the method will return only when it has extracted this many items * (equal or more) */ public static String getStreamsFromApiMinItems(final int minItems, final StreamInfoItemsCollector collector, - final String apiUrl) throws IOException, - ReCaptchaException, ParsingException { + final String apiUrl) + throws IOException, ReCaptchaException, ParsingException, HttpResponseException { String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl); while (!nextPageUrl.isEmpty() && collector.getItems().size() < minItems) { @@ -319,18 +337,16 @@ public static String getStreamsFromApiMinItems(final int minItems, @Nonnull public static String getStreamsFromApi(final StreamInfoItemsCollector collector, final String apiUrl, - final boolean charts) throws IOException, - ReCaptchaException, ParsingException { - final Response response = NewPipe.getDownloader().get(apiUrl, SoundCloud - .getLocalization()); - if (response.responseCode() >= 400) { - throw new IOException("Could not get streams from API, HTTP " + response - .responseCode()); - } + final boolean charts) + throws IOException, ReCaptchaException, ParsingException, HttpResponseException { + final String responseBody = NewPipe.getDownloader() + .get(apiUrl, SoundCloud.getLocalization()) + .ensureResponseCodeIsNotError() + .responseBody(); final JsonObject responseObject; try { - responseObject = JsonParser.object().from(response.responseBody()); + responseObject = JsonParser.object().from(responseBody); } catch (final JsonParserException e) { throw new ParsingException("Could not parse json response", e); } @@ -361,23 +377,22 @@ private static String getNextPageUrl(@Nonnull final JsonObject response) { } public static String getStreamsFromApi(final StreamInfoItemsCollector collector, - final String apiUrl) throws ReCaptchaException, - ParsingException, IOException { + final String apiUrl) + throws ReCaptchaException, ParsingException, IOException, HttpResponseException { return getStreamsFromApi(collector, apiUrl, false); } public static String getInfoItemsFromApi(final MultiInfoItemsCollector collector, - final String apiUrl) throws ReCaptchaException, - ParsingException, IOException { - final Response response = NewPipe.getDownloader().get(apiUrl, SoundCloud.getLocalization()); - if (response.responseCode() >= 400) { - throw new IOException("Could not get streams from API, HTTP " - + response.responseCode()); - } + final String apiUrl) + throws ReCaptchaException, ParsingException, IOException, HttpResponseException { + final String responseBody = NewPipe.getDownloader() + .get(apiUrl, SoundCloud.getLocalization()) + .ensureResponseCodeIsNotError() + .responseBody(); final JsonObject responseObject; try { - responseObject = JsonParser.object().from(response.responseBody()); + responseObject = JsonParser.object().from(responseBody); } catch (final JsonParserException e) { throw new ParsingException("Could not parse json response", e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChartsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChartsExtractor.java index 3e2fb67433..cad45e946e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChartsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChartsExtractor.java @@ -8,6 +8,7 @@ import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.HttpResponseException; import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.localization.ContentCountry; @@ -56,7 +57,7 @@ public void onFetchPage(@Nonnull final Downloader downloader) initialFetchNextPageUrl = SoundcloudParsingHelper.getStreamsFromApi( initialFetchCollector, apiUrlWithRegion == null ? apiUrl : apiUrlWithRegion, true); - } catch (final IOException e) { + } catch (final IOException | HttpResponseException e) { // Request to other region may be geo-restricted. // See https://github.com/TeamNewPipe/NewPipeExtractor/issues/537. // We retry without the specified region.