Merge pull request #78 from TeamNewPipe/donation

theScrabi · web-flow · commit e5125c15f19c · 2018-05-07T10:08:36.000+02:00
enable Donations for streams
diff --git a/extractor/build.gradle b/extractor/build.gradle
@@ -5,6 +5,7 @@ dependencies {
     implementation 'org.jsoup:jsoup:1.9.2'
     implementation 'org.mozilla:rhino:1.7.7.1'
     implementation 'com.github.spotbugs:spotbugs-annotations:3.1.0'
+    implementation 'org.nibor.autolink:autolink:0.8.0'
 
     testImplementation 'junit:junit:4.12'
 }
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java b/extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java
@@ -35,7 +35,8 @@ public enum MediaFormat {
     // audio formats
     M4A         (0x3,   "m4a",   "m4a",  "audio/mp4"),
     WEBMA       (0x4,   "WebM",  "webm", "audio/webm"),
-    MP3         (0x5,   "MP3",   "mp3",  "audio/mpeg");
+    MP3         (0x5,   "MP3",   "mp3",  "audio/mpeg"),
+    OPUS        (0x6,   "opus",  "opus", "audio/opus");
 
     public final int id;
     public final String name;
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java
@@ -45,4 +45,5 @@ protected UrlIdHandler getUrlIdHandler() {
     public abstract String getFeedUrl() throws ParsingException;
     public abstract long getSubscriberCount() throws ParsingException;
     public abstract String getDescription() throws ParsingException;
+    public abstract String[] getDonationLinks() throws ParsingException;
 }
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java
@@ -92,6 +92,11 @@ public static ChannelInfo getInfo(ChannelExtractor extractor) throws IOException
         } catch (Exception e) {
             info.addError(e);
         }
+        try {
+            info.setDonationLinks(extractor.getDonationLinks());
+        } catch (Exception e) {
+            info.addError(e);
+        }
 
         return info;
     }
@@ -101,6 +106,7 @@ public static ChannelInfo getInfo(ChannelExtractor extractor) throws IOException
     private String feedUrl;
     private long subscriberCount = -1;
     private String description;
+    private String[] donationLinks;
 
     public String getAvatarUrl() {
         return avatarUrl;
@@ -141,4 +147,12 @@ public String getDescription() {
     public void setDescription(String description) {
         this.description = description;
     }
+
+    public String[] getDonationLinks() {
+        return donationLinks;
+    }
+
+    public void setDonationLinks(String[] donationLinks) {
+        this.donationLinks = donationLinks;
+    }
 }
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java
@@ -133,4 +133,9 @@ public InfoItemsPage<StreamInfoItem> getPage(final String pageUrl) throws IOExce
 
         return new InfoItemsPage<>(collector, nextPageUrl);
     }
+
+    @Override
+    public String[] getDonationLinks() {
+        return new String[0];
+    }
 }
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java
@@ -209,6 +209,10 @@ public StreamInfoItemsCollector getRelatedVideos() throws IOException, Extractio
         return collector;
     }
 
+    @Override
+    public String[] getDonationLinks() {
+        return new String[0];
+    }
 
     @Override
     public String getErrorMessage() {
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java
@@ -4,22 +4,26 @@
 import com.grack.nanojson.JsonObject;
 import com.grack.nanojson.JsonParser;
 import com.grack.nanojson.JsonParserException;
+import com.sun.org.apache.xerces.internal.xs.StringList;
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
 import org.schabi.newpipe.extractor.Downloader;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.StreamingService;
+import org.schabi.newpipe.extractor.UrlIdHandler;
 import org.schabi.newpipe.extractor.channel.ChannelExtractor;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
+import org.schabi.newpipe.extractor.utils.DonationLinkHelper;
 import org.schabi.newpipe.extractor.utils.Parser;
 import org.schabi.newpipe.extractor.utils.Utils;
 
 import javax.annotation.Nonnull;
 import java.io.IOException;
+import java.util.ArrayList;
 
 /*
  * Created by Christian Schabesberger on 25.07.16.
@@ -181,6 +185,29 @@ public InfoItemsPage<StreamInfoItem> getPage(String pageUrl) throws IOException,
         return new InfoItemsPage<>(collector, getNextPageUrlFromAjaxPage(ajaxJson, pageUrl));
     }
 
+    @Override
+    public String[] getDonationLinks() throws ParsingException {
+        try {
+            ArrayList<String> links = new ArrayList<>();
+            Element linkHolder = doc.select("div[id=\"header-links\"]").first();
+            if(linkHolder == null) {
+                // this occures if no links are embeded into the channel
+                return new String[0];
+            }
+            for(Element a : linkHolder.select("a")) {
+                String link = a.attr("abs:href");
+                if(DonationLinkHelper.getServiceByLink(link) != DonationLinkHelper.DonationService.NO_DONATION) {
+                    links.add(link);
+                }
+            }
+            String[] retLinks = new String[links.size()];
+            retLinks = links.toArray(retLinks);
+            return retLinks;
+        } catch (Exception e) {
+            throw new ParsingException("Could not get donation links", e);
+        }
+    }
+
     private String getNextPageUrlFromAjaxPage(final JsonObject ajaxJson, final String pageUrl)
         throws ParsingException {
         String loadMoreHtmlDataRaw = ajaxJson.getString("load_more_widget_html");
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java
@@ -19,6 +19,7 @@
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
 import org.schabi.newpipe.extractor.stream.*;
+import org.schabi.newpipe.extractor.utils.DonationLinkHelper;
 import org.schabi.newpipe.extractor.utils.Parser;
 import org.schabi.newpipe.extractor.utils.Utils;
 
@@ -528,6 +529,24 @@ public String getErrorMessage() {
         return errorReason != null ? errorReason.toString() : null;
     }
 
+    @Override
+    public String[] getDonationLinks() throws ParsingException {
+        try {
+            ArrayList<String> donationLinks = new ArrayList<>();
+            for (String s : Parser.getLinksFromString(getDescription())) {
+                if (DonationLinkHelper.getServiceByLink(s) != DonationLinkHelper.DonationService.NO_DONATION) {
+                    donationLinks.add(s);
+                }
+            }
+            String[] donlret = new String[donationLinks.size()];
+            donlret = donationLinks.toArray(donlret);
+            return donlret;
+        } catch (Exception e) {
+            throw new ParsingException("Could not get donation links", e);
+        }
+    }
+
+
     /*//////////////////////////////////////////////////////////////////////////
     // Fetch page
     //////////////////////////////////////////////////////////////////////////*/
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java
@@ -141,6 +141,8 @@ protected long getTimestampSeconds(String regexPattern) throws ParsingException
     public abstract StreamInfoItem getNextVideo() throws IOException, ExtractionException;
     public abstract StreamInfoItemsCollector getRelatedVideos() throws IOException, ExtractionException;
 
+    public abstract String[] getDonationLinks() throws ExtractionException;
+
     /**
      * Analyses the webpage's document and extracts any error message there might be.
      *
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DonationLinkHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DonationLinkHelper.java
@@ -0,0 +1,31 @@
+package org.schabi.newpipe.extractor.utils;
+
+import java.net.MalformedURLException;
+import java.net.URL;
+
+public class DonationLinkHelper {
+    public enum DonationService {
+        NO_DONATION,
+        PATREON,
+        PAYPAL
+    }
+
+
+    public static DonationService getServiceByLink(String link) throws MalformedURLException {
+        URL url = new URL(link);
+        switch (url.getHost()) {
+            case "www.patreon.com":
+                return DonationService.PATREON;
+            case "patreon.com":
+                return DonationService.PATREON;
+            case "paypal.me":
+                return DonationService.PAYPAL;
+            case "www.paypal.me":
+                return DonationService.PAYPAL;
+            default:
+                return DonationService.NO_DONATION;
+        }
+    }
+
+
+}
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java
@@ -1,9 +1,14 @@
 package org.schabi.newpipe.extractor.utils;
 
+import org.nibor.autolink.LinkExtractor;
+import org.nibor.autolink.LinkSpan;
+import org.nibor.autolink.LinkType;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 
 import java.io.UnsupportedEncodingException;
 import java.net.URLDecoder;
+import java.util.ArrayList;
+import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.regex.Matcher;
@@ -80,4 +85,23 @@ public static Map<String, String> compatParseMap(final String input) throws Unsu
         }
         return map;
     }
+
+    public static String[] getLinksFromString(final String txt) throws ParsingException {
+        try {
+            ArrayList<String> links = new ArrayList<>();
+            LinkExtractor linkExtractor = LinkExtractor.builder()
+                    .linkTypes(EnumSet.of(LinkType.URL, LinkType.WWW))
+                    .build();
+            Iterable<LinkSpan> linkss = linkExtractor.extractLinks(txt);
+            for(LinkSpan ls : linkss) {
+                links.add(txt.substring(ls.getBeginIndex(), ls.getEndIndex()));
+            }
+
+            String[] linksarray = new String[links.size()];
+            linksarray = links.toArray(linksarray);
+            return linksarray;
+        } catch (Exception e) {
+            throw new ParsingException("Could not get links from string", e);
+        }
+    }
 }
diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java
@@ -1,6 +1,7 @@
 package org.schabi.newpipe.extractor.services.youtube;
 
 import org.junit.BeforeClass;
+import org.junit.Ignore;
 import org.junit.Test;
 import org.schabi.newpipe.Downloader;
 import org.schabi.newpipe.extractor.NewPipe;
@@ -103,6 +104,13 @@ public void testFeedUrl() throws Exception {
         public void testSubscriberCount() throws Exception {
             assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 0);
         }
+
+        @Test
+        public void testChannelDonation() throws Exception {
+            // this needs to be ignored since wed have to upgrade channel extractor to the new yt interface
+            // in order to make this work
+            assertTrue(extractor.getDonationLinks().length == 0);
+        }
     }
 
     public static class Kurzgesagt implements BaseChannelExtractorTest {
@@ -205,6 +213,11 @@ public void testFeedUrl() throws Exception {
         public void testSubscriberCount() throws Exception {
             assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 5e6);
         }
+
+        @Test
+        public void testChannelDonation() throws Exception {
+            assertTrue(extractor.getDonationLinks().length == 1);
+        }
     }
 
     public static class CaptainDisillusion implements BaseChannelExtractorTest {
@@ -389,6 +402,11 @@ public void testFeedUrl() throws Exception {
         public void testSubscriberCount() throws Exception {
             assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 50);
         }
+
+        @Test
+        public void testChannelDonation() throws Exception {
+            assertTrue(extractor.getDonationLinks().length == 0);
+        }
     }
 };
 
diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDonationTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDonationTest.java
@@ -0,0 +1,52 @@
+package org.schabi.newpipe.extractor.services.youtube;
+
+
+/*
+ * Created by Christian Schabesberger on 30.12.15.
+ *
+ * Copyright (C) Christian Schabesberger 2018 <chris.schabesberger@mailbox.org>
+ * YoutubeStreamExtractorDonationTest.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.schabi.newpipe.Downloader;
+import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.stream.StreamExtractor;
+
+import static org.junit.Assert.assertTrue;
+import static org.schabi.newpipe.extractor.ServiceList.YouTube;
+
+/**
+ * Test for {@link StreamExtractor}
+ */
+public class YoutubeStreamExtractorDonationTest {
+    private static YoutubeStreamExtractor extractor;
+
+    @BeforeClass
+    public static void setUp() throws Exception {
+        NewPipe.init(Downloader.getInstance());
+        extractor = (YoutubeStreamExtractor) YouTube
+                .getStreamExtractor("https://www.youtube.com/watch?v=pXb3jERMoI0");
+        extractor.fetchPage();
+    }
+
+    @Test
+    public void getDonationLinksTest() throws Exception {
+        assertTrue(String.valueOf(extractor.getDonationLinks().length),
+                extractor.getDonationLinks().length == 2);
+    }
+}

Original file line number	Diff line number	Diff line change
`@@ -5,6 +5,7 @@ dependencies {`
`5`	`5`	`implementation 'org.jsoup:jsoup:1.9.2'`
`6`	`6`	`implementation 'org.mozilla:rhino:1.7.7.1'`
`7`	`7`	`implementation 'com.github.spotbugs:spotbugs-annotations:3.1.0'`
	`8`	`+ implementation 'org.nibor.autolink:autolink:0.8.0'`
`8`	`9`
`9`	`10`	`testImplementation 'junit:junit:4.12'`
`10`	`11`	`}`
Original file line number	Diff line number	Diff line change
`@@ -45,4 +45,5 @@ protected UrlIdHandler getUrlIdHandler() {`
`45`	`45`	`public abstract String getFeedUrl() throws ParsingException;`
`46`	`46`	`public abstract long getSubscriberCount() throws ParsingException;`
`47`	`47`	`public abstract String getDescription() throws ParsingException;`
	`48`	`+ public abstract String[] getDonationLinks() throws ParsingException;`
`48`	`49`	`}`
Original file line number	Diff line number	Diff line change
`@@ -133,4 +133,9 @@ public InfoItemsPage<StreamInfoItem> getPage(final String pageUrl) throws IOExce`
`133`	`133`
`134`	`134`	`return new InfoItemsPage<>(collector, nextPageUrl);`
`135`	`135`	`}`
	`136`	`+`
	`137`	`+ @Override`
	`138`	`+ public String[] getDonationLinks() {`
	`139`	`+ return new String[0];`
	`140`	`+ }`
`136`	`141`	`}`
Original file line number	Diff line number	Diff line change
`@@ -209,6 +209,10 @@ public StreamInfoItemsCollector getRelatedVideos() throws IOException, Extractio`
`209`	`209`	`return collector;`
`210`	`210`	`}`
`211`	`211`
	`212`	`+ @Override`
	`213`	`+ public String[] getDonationLinks() {`
	`214`	`+ return new String[0];`
	`215`	`+ }`
`212`	`216`
`213`	`217`	`@Override`
`214`	`218`	`public String getErrorMessage() {`
Original file line number	Diff line number	Diff line change
`@@ -141,6 +141,8 @@ protected long getTimestampSeconds(String regexPattern) throws ParsingException`
`141`	`141`	`public abstract StreamInfoItem getNextVideo() throws IOException, ExtractionException;`
`142`	`142`	`public abstract StreamInfoItemsCollector getRelatedVideos() throws IOException, ExtractionException;`
`143`	`143`
	`144`	`+ public abstract String[] getDonationLinks() throws ExtractionException;`
	`145`	`+`
`144`	`146`	`/**`
`145`	`147`	`* Analyses the webpage's document and extracts any error message there might be.`
`146`	`148`	`*`