Skip to content

Commit 5f2d0cf

Browse files
theScrabitheScrabi
committed
fix yt trending contains to less items
1 parent a0e5c88 commit 5f2d0cf

2 files changed

Lines changed: 69 additions & 66 deletions

File tree

‎src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java‎

Lines changed: 67 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import org.jsoup.Jsoup;
2424
import org.jsoup.nodes.Document;
2525
import org.jsoup.nodes.Element;
26+
import org.jsoup.select.Elements;
2627
import org.schabi.newpipe.extractor.*;
2728
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
2829
import org.schabi.newpipe.extractor.exceptions.ParsingException;
@@ -37,7 +38,7 @@ public class YoutubeTrendingExtractor extends KioskExtractor {
3738
private Document doc;
3839

3940
public YoutubeTrendingExtractor(StreamingService service, String url, String nextStreamsUrl, String kioskId)
40-
throws IOException, ExtractionException {
41+
throws IOException, ExtractionException {
4142
super(service, url, nextStreamsUrl, kioskId);
4243
}
4344

@@ -81,76 +82,78 @@ public String getName() throws ParsingException {
8182
@Override
8283
public StreamInfoItemCollector getStreams() throws ParsingException {
8384
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
84-
Element ul = doc.select("ul[class*=\"expanded-shelf-content-list\"]").first();
85-
for(final Element li : ul.children()) {
86-
final Element el = li.select("div[class*=\"yt-lockup-dismissable\"]").first();
87-
collector.commit(new YoutubeStreamInfoItemExtractor(li) {
88-
@Override
89-
public String getUrl() throws ParsingException {
90-
try {
91-
Element dl = el.select("h3").first().select("a").first();
92-
return dl.attr("abs:href");
93-
} catch (Exception e) {
94-
throw new ParsingException("Could not get web page url for the video", e);
85+
Elements uls = doc.select("ul[class*=\"expanded-shelf-content-list\"]");
86+
for(Element ul : uls) {
87+
for(final Element li : ul.children()) {
88+
final Element el = li.select("div[class*=\"yt-lockup-dismissable\"]").first();
89+
collector.commit(new YoutubeStreamInfoItemExtractor(li) {
90+
@Override
91+
public String getUrl() throws ParsingException {
92+
try {
93+
Element dl = el.select("h3").first().select("a").first();
94+
return dl.attr("abs:href");
95+
} catch (Exception e) {
96+
throw new ParsingException("Could not get web page url for the video", e);
97+
}
9598
}
96-
}
97-
98-
@Override
99-
public String getName() throws ParsingException {
100-
try {
101-
Element dl = el.select("h3").first().select("a").first();
102-
return dl.text();
103-
} catch (Exception e) {
104-
throw new ParsingException("Could not get web page url for the video", e);
99+
100+
@Override
101+
public String getName() throws ParsingException {
102+
try {
103+
Element dl = el.select("h3").first().select("a").first();
104+
return dl.text();
105+
} catch (Exception e) {
106+
throw new ParsingException("Could not get web page url for the video", e);
107+
}
105108
}
106-
}
107-
108-
@Override
109-
public String getUploaderUrl() throws ParsingException {
110-
try {
111-
String link = getUploaderLink().attr("href");
112-
if(link.isEmpty()) {
113-
throw new IllegalArgumentException("is empty");
109+
110+
@Override
111+
public String getUploaderUrl() throws ParsingException {
112+
try {
113+
String link = getUploaderLink().attr("href");
114+
if (link.isEmpty()) {
115+
throw new IllegalArgumentException("is empty");
116+
}
117+
return link;
118+
} catch (Exception e) {
119+
throw new ParsingException("Could not get Uploader name");
114120
}
115-
return link;
116-
} catch (Exception e) {
117-
throw new ParsingException("Could not get Uploader name");
118121
}
119-
}
120-
121-
private Element getUploaderLink() {
122-
Element uploaderEl = el.select("div[class*=\"yt-lockup-byline \"]").first();
123-
return uploaderEl.select("a").first();
124-
}
125-
126-
@Override
127-
public String getUploaderName() throws ParsingException {
128-
try {
129-
return getUploaderLink().text();
130-
} catch (Exception e) {
131-
throw new ParsingException("Could not get Uploader name");
122+
123+
private Element getUploaderLink() {
124+
Element uploaderEl = el.select("div[class*=\"yt-lockup-byline \"]").first();
125+
return uploaderEl.select("a").first();
132126
}
133-
}
134-
135-
@Override
136-
public String getThumbnailUrl() throws ParsingException {
137-
try {
138-
String url;
139-
Element te = li.select("span[class=\"yt-thumb-simple\"]").first()
140-
.select("img").first();
141-
url = te.attr("abs:src");
142-
// Sometimes youtube sends links to gif files which somehow seem to not exist
143-
// anymore. Items with such gif also offer a secondary image source. So we are going
144-
// to use that if we've caught such an item.
145-
if (url.contains(".gif")) {
146-
url = te.attr("abs:data-thumb");
127+
128+
@Override
129+
public String getUploaderName() throws ParsingException {
130+
try {
131+
return getUploaderLink().text();
132+
} catch (Exception e) {
133+
throw new ParsingException("Could not get Uploader name");
134+
}
135+
}
136+
137+
@Override
138+
public String getThumbnailUrl() throws ParsingException {
139+
try {
140+
String url;
141+
Element te = li.select("span[class=\"yt-thumb-simple\"]").first()
142+
.select("img").first();
143+
url = te.attr("abs:src");
144+
// Sometimes youtube sends links to gif files which somehow seem to not exist
145+
// anymore. Items with such gif also offer a secondary image source. So we are going
146+
// to use that if we've caught such an item.
147+
if (url.contains(".gif")) {
148+
url = te.attr("abs:data-thumb");
149+
}
150+
return url;
151+
} catch (Exception e) {
152+
throw new ParsingException("Could not get thumbnail url", e);
147153
}
148-
return url;
149-
} catch (Exception e) {
150-
throw new ParsingException("Could not get thumbnail url", e);
151154
}
152-
}
153-
});
155+
});
156+
}
154157
}
155158

156159
return collector;

‎src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -67,10 +67,10 @@ public void testId() throws Exception {
6767
}
6868

6969
@Test
70-
public void testGetStreams() throws Exception {
70+
public void testGetStreamsQuantity() throws Exception {
7171
StreamInfoItemCollector collector = extractor.getStreams();
7272
Utils.printErrors(collector);
73-
assertFalse("no streams are received", collector.getItemList().isEmpty());
73+
assertTrue("no streams are received", collector.getItemList().size() >= 20);
7474
}
7575

7676
@Test

0 commit comments

Comments
 (0)