Add support for Google search redirect url

Author: Stypox

extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java

@@ -16,6 +16,7 @@
 import org.schabi.newpipe.extractor.stream.StreamExtractor;
 import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
 import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
+import org.schabi.newpipe.extractor.utils.Utils;
 
 import javax.annotation.Nullable;
 import java.util.Collections;
@@ -277,12 +278,13 @@ public CommentsExtractor getCommentsExtractor(String url) throws ExtractionExcep
      * Figures out where the link is pointing to (a channel, a video, a playlist, etc.)
      * @param url the url on which it should be decided of which link type it is
      * @return the link type of url
-     * @throws ParsingException
      */
     public final LinkType getLinkTypeByUrl(String url) throws ParsingException {
-        LinkHandlerFactory sH = getStreamLHFactory();
-        LinkHandlerFactory cH = getChannelLHFactory();
-        LinkHandlerFactory pH = getPlaylistLHFactory();
+        url = Utils.followGoogleRedirectIfNeeded(url);
+
+        final LinkHandlerFactory sH = getStreamLHFactory();
+        final LinkHandlerFactory cH = getChannelLHFactory();
+        final LinkHandlerFactory pH = getPlaylistLHFactory();
 
         if (sH != null && sH.acceptUrl(url)) {
             return LinkType.STREAM;

extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandlerFactory.java

@@ -42,12 +42,30 @@ public String getUrl(String id, String baseUrl) throws ParsingException {
     // Logic
     ///////////////////////////////////
 
+    /**
+     * Builds a {@link LinkHandler} from a url.<br>
+     * Be sure to call {@link Utils#followGoogleRedirectIfNeeded(String)} on the url if overriding
+     * this function.
+     * @param url the url to extract path and id from
+     * @return a {@link LinkHandler} complete with information
+     */
     public LinkHandler fromUrl(String url) throws ParsingException {
         if (url == null) throw new IllegalArgumentException("url can not be null");
+        url = Utils.followGoogleRedirectIfNeeded(url);
         final String baseUrl = Utils.getBaseUrl(url);
         return fromUrl(url, baseUrl);
     }
 
+    /**
+     * Builds a {@link LinkHandler} from a url and a base url. The url is expected to be already
+     * polished from google search redirects (otherwise how could {@code baseUrl} have been
+     * extracted?).<br>
+     * So do not call {@link Utils#followGoogleRedirectIfNeeded(String)} on the url if overriding
+     * this function, since that should be done in {@link #fromUrl(String)}.
+     * @param url the url without google search redirects to extract id from
+     * @param baseUrl the base url
+     * @return a {@link LinkHandler} complete with information
+     */
     public LinkHandler fromUrl(String url, String baseUrl) throws ParsingException {
         if (url == null) throw new IllegalArgumentException("url can not be null");
         if (!acceptUrl(url)) {

extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ListLinkHandlerFactory.java

@@ -32,7 +32,8 @@ public String getUrl(String id, List<String> contentFilter, String sortFilter, S
 
     @Override
     public ListLinkHandler fromUrl(String url) throws ParsingException {
-        String baseUrl = Utils.getBaseUrl(url);
+        url = Utils.followGoogleRedirectIfNeeded(url);
+        final String baseUrl = Utils.getBaseUrl(url);
         return fromUrl(url, baseUrl);
     }
 

extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java

@@ -196,6 +196,26 @@ public static String getBaseUrl(String url) throws ParsingException {
         }
     }
 
+    /**
+     * If the provided url is a Google search redirect, then the actual url is extracted from the
+     * {@code url=} query value and returned, otherwise the original url is returned.
+     * @param url the url which can possibly be a Google search redirect
+     * @return an url with no Google search redirects
+     */
+    public static String followGoogleRedirectIfNeeded(final String url) {
+        // if the url is a redirect from a Google search, extract the actual url
+        try {
+            final URL decoded = Utils.stringToURL(url);
+            if (decoded.getHost().contains("google") && decoded.getPath().equals("/url")) {
+                return URLDecoder.decode(Parser.matchGroup1("&url=([^&]+)(?:&|$)", url), "UTF-8");
+            }
+        } catch (final Exception ignored) {
+        }
+
+        // url is not a google search redirect
+        return url;
+    }
+
     public static boolean isNullOrEmpty(final String str) {
         return str == null || str.isEmpty();
     }

extractor/src/test/java/org/schabi/newpipe/extractor/NewPipeTest.java

@@ -6,6 +6,7 @@
 
 import static org.junit.Assert.*;
 import static org.schabi.newpipe.extractor.NewPipe.getServiceByUrl;
+import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
 import static org.schabi.newpipe.extractor.ServiceList.YouTube;
 
 public class NewPipeTest {
@@ -39,8 +40,10 @@ public void getServiceWithUrl() throws Exception {
         assertEquals(getServiceByUrl("https://www.youtube.com/watch?v=_r6CgaFNAGg"), YouTube);
         assertEquals(getServiceByUrl("https://www.youtube.com/channel/UCi2bIyFtz-JdI-ou8kaqsqg"), YouTube);
         assertEquals(getServiceByUrl("https://www.youtube.com/playlist?list=PLRqwX-V7Uu6ZiZxtDDRCi6uhfTH4FilpH"), YouTube);
+        assertEquals(getServiceByUrl("https://www.google.it/url?sa=t&rct=j&q=&esrc=s&cd=&cad=rja&uact=8&url=https%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DHu80uDzh8RY&source=video"), YouTube);
 
-        assertNotEquals(getServiceByUrl("https://soundcloud.com/pegboardnerds"), YouTube);
+        assertEquals(getServiceByUrl("https://soundcloud.com/pegboardnerds"), SoundCloud);
+        assertEquals(getServiceByUrl("https://www.google.com/url?sa=t&url=https%3A%2F%2Fsoundcloud.com%2Fciaoproduction&rct=j&q=&esrc=s&source=web&cd="), SoundCloud);
     }
 
     @Test

extractor/src/test/java/org/schabi/newpipe/extractor/utils/UtilsTest.java

@@ -30,4 +30,19 @@ public void testGetBaseUrl() throws ParsingException {
         assertEquals("vnd.youtube", Utils.getBaseUrl("vnd.youtube://n8X9_MgEdCg"));
         assertEquals("https://music.youtube.com", Utils.getBaseUrl("https://music.youtube.com/watch?v=O0EDx9WAelc"));
     }
+
+    @Test
+    public void testFollowGoogleRedirect() {
+        assertEquals("https://www.youtube.com/watch?v=Hu80uDzh8RY",
+                Utils.followGoogleRedirectIfNeeded("https://www.google.it/url?sa=t&rct=j&q=&esrc=s&cd=&cad=rja&uact=8&url=https%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DHu80uDzh8RY&source=video"));
+        assertEquals("https://www.youtube.com/watch?v=0b6cFWG45kA",
+                Utils.followGoogleRedirectIfNeeded("https://www.google.com/url?sa=t&rct=j&q=&esrc=s&source=video&cd=&cad=rja&uact=8&url=https%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3D0b6cFWG45kA"));
+        assertEquals("https://soundcloud.com/ciaoproduction",
+                Utils.followGoogleRedirectIfNeeded("https://www.google.com/url?sa=t&url=https%3A%2F%2Fsoundcloud.com%2Fciaoproduction&rct=j&q=&esrc=s&source=web&cd="));
+
+        assertEquals("https://www.youtube.com/watch?v=Hu80uDzh8RY&param=xyz",
+                Utils.followGoogleRedirectIfNeeded("https://www.youtube.com/watch?v=Hu80uDzh8RY&param=xyz"));
+        assertEquals("https://www.youtube.com/watch?v=Hu80uDzh8RY&url=hello",
+                Utils.followGoogleRedirectIfNeeded("https://www.youtube.com/watch?v=Hu80uDzh8RY&url=hello"));
+    }
 }