Skip to content

Commit 7b7f6d2

Browse files
theScrabitheScrabi
committed
made YoutubeTrendingExtractor work
1 parent 88d2fff commit 7b7f6d2

5 files changed

Lines changed: 112 additions & 13 deletions

File tree

‎src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ public KioskList getKioskList() throws ExtractionException {
8787
// add kiosks here e.g.:
8888
YoutubeTrendingUrlIdHandler h = new YoutubeTrendingUrlIdHandler();
8989
try {
90-
list.addKioskEntry(new YoutubeTrendingExtractor(this, h.getUrl(""), h.getUrl("")), h);
90+
list.addKioskEntry(new YoutubeTrendingExtractor(this, h.getUrl(""), null), h);
9191
} catch (Exception e) {
9292
throw new ExtractionException(e);
9393
}

‎src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java‎

Lines changed: 72 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -20,30 +20,38 @@
2020
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
2121
*/
2222

23-
import org.schabi.newpipe.extractor.ListExtractor;
24-
import org.schabi.newpipe.extractor.StreamingService;
25-
import org.schabi.newpipe.extractor.UrlIdHandler;
23+
import org.jsoup.Jsoup;
24+
import org.jsoup.nodes.Document;
25+
import org.jsoup.nodes.Element;
26+
import org.schabi.newpipe.extractor.*;
2627
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
28+
import org.schabi.newpipe.extractor.exceptions.ParsingException;
2729
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
2830
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
31+
2932
import java.io.IOException;
3033

3134
public class YoutubeTrendingExtractor extends KioskExtractor {
3235

36+
private Document doc;
37+
3338
public YoutubeTrendingExtractor(StreamingService service, String url, String nextStreamsUrl)
3439
throws IOException, ExtractionException {
3540
super(service, url, nextStreamsUrl);
3641
}
3742

3843
@Override
39-
public void fetchPage()
40-
throws IOException, ExtractionException {
44+
public void fetchPage() throws IOException, ExtractionException {
45+
Downloader downloader = NewPipe.getDownloader();
4146

47+
String channelUrl = getCleanUrl();
48+
String pageContent = downloader.download(channelUrl);
49+
doc = Jsoup.parse(pageContent, channelUrl);
4250
}
4351

4452
@Override
4553
public String getType() {
46-
return "Treinding";
54+
return "Trending";
4755
}
4856

4957
@Override
@@ -57,7 +65,63 @@ public ListExtractor.NextItemsResult getNextStreams() {
5765
}
5866

5967
@Override
60-
public StreamInfoItemCollector getStreams() {
61-
return null;
68+
public StreamInfoItemCollector getStreams() throws ParsingException {
69+
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
70+
Element ul = doc.select("ul[class*=\"expanded-shelf-content-list\"]").first();
71+
for(final Element li : ul.children()) {
72+
final Element el = li.select("div[class*=\"yt-lockup-dismissable\"]").first();
73+
collector.commit(new YoutubeStreamInfoItemExtractor(li) {
74+
@Override
75+
public String getUrl() throws ParsingException {
76+
try {
77+
Element dl = el.select("h3").first().select("a").first();
78+
return dl.attr("abs:href");
79+
} catch (Exception e) {
80+
throw new ParsingException("Could not get web page url for the video", e);
81+
}
82+
}
83+
84+
@Override
85+
public String getName() throws ParsingException {
86+
try {
87+
Element dl = el.select("h3").first().select("a").first();
88+
return dl.text();
89+
} catch (Exception e) {
90+
throw new ParsingException("Could not get web page url for the video", e);
91+
}
92+
}
93+
94+
@Override
95+
public String getUploaderName() throws ParsingException {
96+
try {
97+
Element uploaderEl = el.select("div[class*=\"yt-lockup-byline \"]").first();
98+
return uploaderEl.select("a").text();
99+
} catch (Exception e) {
100+
throw new ParsingException("Could not get Uploader name");
101+
}
102+
}
103+
104+
@Override
105+
public String getThumbnailUrl() throws ParsingException {
106+
try {
107+
String url;
108+
Element te = li.select("span[class=\"yt-thumb-simple\"]").first()
109+
.select("img").first();
110+
url = te.attr("abs:src");
111+
// Sometimes youtube sends links to gif files which somehow seem to not exist
112+
// anymore. Items with such gif also offer a secondary image source. So we are going
113+
// to use that if we've caught such an item.
114+
if (url.contains(".gif")) {
115+
url = te.attr("abs:data-thumb");
116+
}
117+
return url;
118+
} catch (Exception e) {
119+
throw new ParsingException("Could not get thumbnail url", e);
120+
}
121+
}
122+
});
123+
}
124+
125+
return collector;
62126
}
63127
}

‎src/main/java/org/schabi/newpipe/extractor/stream/StreamInfoItemCollector.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,13 @@
11
package org.schabi.newpipe.extractor.stream;
22

3+
import org.schabi.newpipe.extractor.InfoItem;
34
import org.schabi.newpipe.extractor.InfoItemCollector;
45
import org.schabi.newpipe.extractor.exceptions.FoundAdException;
56
import org.schabi.newpipe.extractor.exceptions.ParsingException;
67

8+
import java.util.List;
9+
import java.util.Vector;
10+
711
/*
812
* Created by Christian Schabesberger on 28.02.16.
913
*
@@ -80,4 +84,14 @@ public void commit(StreamInfoItemExtractor extractor) throws ParsingException {
8084
addError(e);
8185
}
8286
}
87+
88+
public List<StreamInfoItem> getStreamInfoItemList() {
89+
List<StreamInfoItem> siiList = new Vector<>();
90+
for(InfoItem ii : super.getItemList()) {
91+
if(ii instanceof StreamInfoItem) {
92+
siiList.add((StreamInfoItem) ii);
93+
}
94+
}
95+
return siiList;
96+
}
8397
}

‎src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java‎

Lines changed: 24 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,13 @@
2323
import org.junit.Before;
2424
import org.junit.Test;
2525
import org.schabi.newpipe.Downloader;
26+
import org.schabi.newpipe.extractor.InfoItem;
27+
import org.schabi.newpipe.extractor.InfoItemCollector;
2628
import org.schabi.newpipe.extractor.NewPipe;
2729
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
2830

31+
import java.util.List;
32+
2933
import static junit.framework.TestCase.assertFalse;
3034
import static org.junit.Assert.assertEquals;
3135
import static org.junit.Assert.assertNotNull;
@@ -46,6 +50,7 @@ public void setUp() throws Exception {
4650
extractor = YouTube.getService()
4751
.getKioskList()
4852
.getExtractorByType("Trending");
53+
extractor.fetchPage();
4954
}
5055

5156
@Test
@@ -65,7 +70,17 @@ public void testId() throws Exception {
6570

6671
@Test
6772
public void testGetStreams() throws Exception {
68-
assertTrue("no streams are received", !extractor.getStreams().getItemList().isEmpty());
73+
InfoItemCollector collector = extractor.getStreams();
74+
if(!collector.getErrors().isEmpty()) {
75+
System.err.println("----------");
76+
for(Throwable e : collector.getErrors()) {
77+
e.printStackTrace();
78+
System.err.println("----------");
79+
}
80+
}
81+
assertTrue("no streams are received",
82+
!collector.getItemList().isEmpty()
83+
&& collector.getErrors().isEmpty());
6984
}
7085

7186
@Test
@@ -77,12 +92,17 @@ public void testGetStreamsErrors() throws Exception {
7792
public void testHasMoreStreams() throws Exception {
7893
// Setup the streams
7994
extractor.getStreams();
80-
assertTrue("don't have more streams", extractor.hasMoreStreams());
95+
assertFalse("has more streams", extractor.hasMoreStreams());
8196
}
8297

8398
@Test
8499
public void testGetNextStreams() throws Exception {
85-
assertFalse("extractor has next streams", !extractor.getNextStreams().nextItemsList.isEmpty());
86-
assertFalse("extractor has more streams after getNextStreams", extractor.hasMoreStreams());
100+
assertTrue("extractor has next streams", extractor.getNextStreams() == null
101+
|| extractor.getNextStreams().nextItemsList.isEmpty());
102+
}
103+
104+
@Test
105+
public void testGetCleanUrl() throws Exception {
106+
assertEquals(extractor.getCleanUrl(), extractor.getCleanUrl(), "https://www.youtube.com/feed/trending");
87107
}
88108
}

‎src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingUrlIdHandlerTest.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,7 @@ public void acceptUrl() {
7171
assertFalse(urlIdHandler.acceptUrl("youtube.com/feed/trending askjkf"));
7272
assertFalse(urlIdHandler.acceptUrl("askdjfi youtube.com/feed/trending askjkf"));
7373
assertFalse(urlIdHandler.acceptUrl(" youtube.com/feed/trending"));
74+
assertFalse(urlIdHandler.acceptUrl("https://www.youtube.com/feed/trending.html"));
7475
assertFalse(urlIdHandler.acceptUrl(""));
7576
}
7677
}

0 commit comments

Comments
 (0)