Skip to content

Commit aff595c

Browse files
mauriciocollimauriciocolli
authored
Merge pull request #25 from karyogamy/playlist_info
Playlist info extraction for search engine
2 parents 9184fc5 + 645ee5a commit aff595c

9 files changed

Lines changed: 378 additions & 8 deletions

File tree

‎src/main/java/org/schabi/newpipe/extractor/search/InfoItemSearchCollector.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@
55
import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
66
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
77
import org.schabi.newpipe.extractor.exceptions.FoundAdException;
8+
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemCollector;
9+
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor;
810
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
911
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
1012

@@ -32,13 +34,15 @@ public class InfoItemSearchCollector extends InfoItemCollector {
3234
private String suggestion = "";
3335
private StreamInfoItemCollector streamCollector;
3436
private ChannelInfoItemCollector userCollector;
37+
private PlaylistInfoItemCollector playlistCollector;
3538

3639
private SearchResult result = new SearchResult();
3740

3841
InfoItemSearchCollector(int serviceId) {
3942
super(serviceId);
4043
streamCollector = new StreamInfoItemCollector(serviceId);
4144
userCollector = new ChannelInfoItemCollector(serviceId);
45+
playlistCollector = new PlaylistInfoItemCollector(serviceId);
4246
}
4347

4448
public void setSuggestion(String suggestion) {
@@ -49,6 +53,7 @@ public SearchResult getSearchResult() throws ExtractionException {
4953

5054
addFromCollector(userCollector);
5155
addFromCollector(streamCollector);
56+
addFromCollector(playlistCollector);
5257

5358
result.suggestion = suggestion;
5459
result.errors = getErrors();
@@ -74,4 +79,14 @@ public void commit(ChannelInfoItemExtractor extractor) {
7479
addError(e);
7580
}
7681
}
82+
83+
public void commit(PlaylistInfoItemExtractor extractor) {
84+
try {
85+
result.resultList.add(playlistCollector.extract(extractor));
86+
} catch (FoundAdException ae) {
87+
System.err.println("Found ad");
88+
} catch (Exception e) {
89+
addError(e);
90+
}
91+
}
7792
}

‎src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistInfoItemExtractor.java‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
package org.schabi.newpipe.extractor.services.soundcloud;
2+
3+
import com.github.openjson.JSONArray;
4+
import com.github.openjson.JSONObject;
5+
import org.schabi.newpipe.extractor.exceptions.ParsingException;
6+
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor;
7+
8+
public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtractor {
9+
private static final String USER_KEY = "user";
10+
private static final String AVATAR_URL_KEY = "avatar_url";
11+
private static final String ARTWORK_URL_KEY = "artwork_url";
12+
private static final String NULL_VALUE = "null";
13+
14+
private JSONObject searchResult;
15+
16+
public SoundcloudPlaylistInfoItemExtractor(JSONObject searchResult) {
17+
this.searchResult = searchResult;
18+
}
19+
20+
@Override
21+
public String getName() throws ParsingException {
22+
try {
23+
return searchResult.getString("title");
24+
} catch (Exception e) {
25+
throw new ParsingException("Failed to extract playlist name", e);
26+
}
27+
}
28+
29+
@Override
30+
public String getUrl() throws ParsingException {
31+
try {
32+
return searchResult.getString("permalink_url");
33+
} catch (Exception e) {
34+
throw new ParsingException("Failed to extract playlist name", e);
35+
}
36+
}
37+
38+
@Override
39+
public String getThumbnailUrl() throws ParsingException {
40+
// Over-engineering at its finest
41+
try {
42+
final String artworkUrl = searchResult.optString(ARTWORK_URL_KEY);
43+
if (!artworkUrl.isEmpty() && !artworkUrl.equals(NULL_VALUE)) return artworkUrl;
44+
45+
// Look for artwork url inside the track list
46+
final JSONArray tracks = searchResult.optJSONArray("tracks");
47+
if (tracks == null) return null;
48+
for (int i = 0; i < tracks.length(); i++) {
49+
if (tracks.isNull(i)) continue;
50+
final JSONObject track = tracks.optJSONObject(i);
51+
if (track == null) continue;
52+
53+
// First look for track artwork url
54+
final String url = track.optString(ARTWORK_URL_KEY);
55+
if (!url.isEmpty() && !url.equals(NULL_VALUE)) return url;
56+
57+
// Then look for track creator avatar url
58+
final JSONObject creator = track.getJSONObject(USER_KEY);
59+
final String creatorAvatar = creator.optString(AVATAR_URL_KEY);
60+
if (!creatorAvatar.isEmpty() && !creatorAvatar.equals(NULL_VALUE)) return creatorAvatar;
61+
}
62+
63+
// Last resort, use user avatar url. If still not found, then throw exception.
64+
final JSONObject user = searchResult.getJSONObject(USER_KEY);
65+
return user.getString(AVATAR_URL_KEY);
66+
} catch (Exception e) {
67+
throw new ParsingException("Failed to extract playlist thumbnail url", e);
68+
}
69+
}
70+
71+
@Override
72+
public String getUploaderName() throws ParsingException {
73+
try {
74+
final JSONObject user = searchResult.getJSONObject(USER_KEY);
75+
return user.optString("username");
76+
} catch (Exception e) {
77+
throw new ParsingException("Failed to extract playlist uploader", e);
78+
}
79+
}
80+
81+
@Override
82+
public long getStreamCount() throws ParsingException {
83+
try {
84+
return Long.parseLong(searchResult.optString("track_count"));
85+
} catch (Exception e) {
86+
throw new ParsingException("Failed to extract playlist stream count", e);
87+
}
88+
}
89+
}

‎src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEngine.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,10 +27,12 @@ public InfoItemSearchCollector search(String query, int page, String languageCod
2727

2828
String url = "https://api-v2.soundcloud.com/search";
2929

30-
if (filter.contains(Filter.STREAM) && !filter.contains(Filter.CHANNEL)) {
30+
if (filter.contains(Filter.STREAM) && filter.size() == 1) {
3131
url += "/tracks";
32-
} else if (!filter.contains(Filter.STREAM) && filter.contains(Filter.CHANNEL)) {
32+
} else if (filter.contains(Filter.CHANNEL) && filter.size() == 1) {
3333
url += "/users";
34+
} else if (filter.contains(Filter.PLAYLIST) && filter.size() == 1) {
35+
url += "/playlists";
3436
}
3537

3638
url += "?q=" + URLEncoder.encode(query, CHARSET_UTF_8)
@@ -53,6 +55,8 @@ public InfoItemSearchCollector search(String query, int page, String languageCod
5355
collector.commit(new SoundcloudChannelInfoItemExtractor(searchResult));
5456
} else if (kind.equals("track")) {
5557
collector.commit(new SoundcloudStreamInfoItemExtractor(searchResult));
58+
} else if (kind.equals("playlist")) {
59+
collector.commit(new SoundcloudPlaylistInfoItemExtractor(searchResult));
5660
}
5761
}
5862

‎src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistInfoItemExtractor.java‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
package org.schabi.newpipe.extractor.services.youtube;
2+
3+
import org.jsoup.nodes.Element;
4+
import org.schabi.newpipe.extractor.exceptions.ParsingException;
5+
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor;
6+
import org.schabi.newpipe.extractor.utils.Utils;
7+
8+
public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtractor {
9+
private Element el;
10+
11+
public YoutubePlaylistInfoItemExtractor(Element el) {
12+
this.el = el;
13+
}
14+
15+
@Override
16+
public String getThumbnailUrl() throws ParsingException {
17+
String url;
18+
19+
try {
20+
Element te = el.select("div[class=\"yt-thumb video-thumb\"]").first()
21+
.select("img").first();
22+
url = te.attr("abs:src");
23+
24+
if (url.contains(".gif")) {
25+
url = te.attr("abs:data-thumb");
26+
}
27+
} catch (Exception e) {
28+
throw new ParsingException("Failed to extract playlist thumbnail url", e);
29+
}
30+
31+
return url;
32+
}
33+
34+
@Override
35+
public String getName() throws ParsingException {
36+
String name;
37+
try {
38+
final Element title = el.select("[class=\"yt-lockup-title\"]").first()
39+
.select("a").first();
40+
41+
name = title == null ? "" : title.text();
42+
} catch (Exception e) {
43+
throw new ParsingException("Failed to extract playlist name", e);
44+
}
45+
46+
return name;
47+
}
48+
49+
@Override
50+
public String getUrl() throws ParsingException {
51+
String url;
52+
53+
try {
54+
final Element href = el.select("div[class=\"yt-lockup-meta\"]").first()
55+
.select("a").first();
56+
57+
url = href.attr("abs:href");
58+
} catch (Exception e) {
59+
throw new ParsingException("Failed to extract playlist url", e);
60+
}
61+
62+
return url;
63+
}
64+
65+
@Override
66+
public String getUploaderName() throws ParsingException {
67+
String name;
68+
69+
try {
70+
final Element div = el.select("div[class=\"yt-lockup-byline\"]").first()
71+
.select("a").first();
72+
73+
name = div.text();
74+
} catch (Exception e) {
75+
throw new ParsingException("Failed to extract playlist uploader", e);
76+
}
77+
78+
return name;
79+
}
80+
81+
@Override
82+
public long getStreamCount() throws ParsingException {
83+
try {
84+
final Element count = el.select("span[class=\"formatted-video-count-label\"]").first()
85+
.select("b").first();
86+
87+
return count == null ? 0 : Long.parseLong(Utils.removeNonDigitCharacters(count.text()));
88+
} catch (Exception e) {
89+
throw new ParsingException("Failed to extract playlist stream count", e);
90+
}
91+
}
92+
}

‎src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngine.java‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -57,10 +57,13 @@ public InfoItemSearchCollector search(String query,
5757
String url = "https://www.youtube.com/results"
5858
+ "?q=" + URLEncoder.encode(query, CHARSET_UTF_8)
5959
+ "&page=" + Integer.toString(page + 1);
60-
if (filter.contains(Filter.STREAM) && !filter.contains(Filter.CHANNEL)) {
61-
url += "&sp=EgIQAQ%253D%253D";
62-
} else if (!filter.contains(Filter.STREAM) && filter.contains(Filter.CHANNEL)) {
63-
url += "&sp=EgIQAg%253D%253D";
60+
61+
if (filter.contains(Filter.STREAM) && filter.size() == 1) {
62+
url += "&sp=EgIQAVAU";
63+
} else if (filter.contains(Filter.CHANNEL) && filter.size() == 1) {
64+
url += "&sp=EgIQAlAU"; //EgIQA( lowercase L )AU
65+
} else if (filter.contains(Filter.PLAYLIST) && filter.size() == 1) {
66+
url += "&sp=EgIQA1AU"; //EgIQA( one )AU
6467
}
6568

6669
String site;
@@ -105,6 +108,8 @@ public InfoItemSearchCollector search(String query,
105108
collector.commit(new YoutubeStreamInfoItemExtractor(el));
106109
} else if ((el = item.select("div[class*=\"yt-lockup-channel\"]").first()) != null) {
107110
collector.commit(new YoutubeChannelInfoItemExtractor(el));
111+
} else if ((el = item.select("div[class*=\"yt-lockup-playlist\"]").first()) != null) {
112+
collector.commit(new YoutubePlaylistInfoItemExtractor(el));
108113
} else {
109114
// noinspection ConstantConditions
110115
// simply ignore not known items

‎src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamInfoItemExtractor.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,10 @@ public long getDuration() throws ParsingException {
7474
try {
7575
if (getStreamType() == StreamType.LIVE_STREAM) return -1;
7676

77-
return YoutubeParsingHelper.parseDurationString(item.select("span[class*=\"video-time\"]").first().text());
77+
final Element duration = item.select("span[class*=\"video-time\"]").first();
78+
// apparently on youtube, video-time element will not show up if the video has a duration of 00:00
79+
// see: https://www.youtube.com/results?sp=EgIQAVAU&q=asdfgf
80+
return duration == null ? 0 : YoutubeParsingHelper.parseDurationString(duration.text());
7881
} catch (Exception e) {
7982
throw new ParsingException("Could not get Duration: " + getUrl(), e);
8083
}

‎src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEnginePlaylistTest.java‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
package org.schabi.newpipe.extractor.services.soundcloud;
2+
3+
import org.junit.Before;
4+
import org.junit.Ignore;
5+
import org.junit.Test;
6+
import org.schabi.newpipe.Downloader;
7+
import org.schabi.newpipe.extractor.InfoItem;
8+
import org.schabi.newpipe.extractor.NewPipe;
9+
import org.schabi.newpipe.extractor.search.SearchEngine;
10+
import org.schabi.newpipe.extractor.search.SearchResult;
11+
12+
import java.util.EnumSet;
13+
14+
import static org.junit.Assert.assertEquals;
15+
import static org.junit.Assert.assertFalse;
16+
import static org.junit.Assert.assertTrue;
17+
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
18+
19+
20+
/*
21+
* Created by Christian Schabesberger on 29.12.15.
22+
*
23+
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
24+
* YoutubeSearchEngineStreamTest.java is part of NewPipe.
25+
*
26+
* NewPipe is free software: you can redistribute it and/or modify
27+
* it under the terms of the GNU General Public License as published by
28+
* the Free Software Foundation, either version 3 of the License, or
29+
* (at your option) any later version.
30+
*
31+
* NewPipe is distributed in the hope that it will be useful,
32+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
33+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
34+
* GNU General Public License for more details.
35+
*
36+
* You should have received a copy of the GNU General Public License
37+
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
38+
*/
39+
40+
/**
41+
* Test for {@link SearchEngine}
42+
*/
43+
public class SoundcloudSearchEnginePlaylistTest {
44+
private SearchResult result;
45+
46+
@Before
47+
public void setUp() throws Exception {
48+
NewPipe.init(Downloader.getInstance());
49+
SearchEngine engine = SoundCloud.getService().getSearchEngine();
50+
51+
// Search by country not yet implemented
52+
result = engine.search("parkmemme", 0, "", EnumSet.of(SearchEngine.Filter.PLAYLIST))
53+
.getSearchResult();
54+
}
55+
56+
@Test
57+
public void testResultList() {
58+
assertFalse(result.resultList.isEmpty());
59+
}
60+
61+
@Test
62+
public void testUserItemType() {
63+
for (InfoItem infoItem : result.resultList) {
64+
assertEquals(InfoItem.InfoType.PLAYLIST, infoItem.info_type);
65+
}
66+
}
67+
68+
@Test
69+
public void testResultErrors() {
70+
if (!result.errors.isEmpty()) for (Throwable error : result.errors) error.printStackTrace();
71+
assertTrue(result.errors == null || result.errors.isEmpty());
72+
}
73+
74+
@Ignore
75+
@Test
76+
public void testSuggestion() {
77+
//todo write a real test
78+
assertTrue(result.suggestion != null);
79+
}
80+
}

‎src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineAllTest.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,8 @@ public void setUp() throws Exception {
5050
// keep in mind that the suggestions can change by country (the parameter "de")
5151
result = engine.search("asdgff", 0, "de",
5252
EnumSet.of(SearchEngine.Filter.CHANNEL,
53-
SearchEngine.Filter.STREAM)).getSearchResult();
53+
SearchEngine.Filter.STREAM,
54+
SearchEngine.Filter.PLAYLIST)).getSearchResult();
5455
}
5556

5657
@Test

0 commit comments

Comments
 (0)