Skip to content

Commit f551e6b

Browse files
StypoxStypox
committed
searchfilters: convert youtube to new framework
 Available content filters: Youtube - all - videos - channels - playlists Youtube Music - Songs - Videos - Albums - Playlists - Artists Available sort filters: - 'Sort by' - Upload Date - Duration - Features
1 parent 8568f19 commit f551e6b

12 files changed

Lines changed: 874 additions & 127 deletions

‎extractor/build.gradle‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
plugins {
22
id 'checkstyle'
3+
id 'com.squareup.wire' version '4.4.1'
34
}
45

56
test {
@@ -18,6 +19,15 @@ checkstyle {
1819
toolVersion checkstyleVersion
1920
}
2021

22+
checkstyleMain
23+
// exclude the wire generated youtube protobuf files
24+
.exclude ('org/schabi/newpipe/extractor/services/youtube/search/filter/protobuf/')
25+
26+
wire {
27+
java {
28+
}
29+
}
30+
2131
checkstyleTest {
2232
enabled false // do not checkstyle test files
2333
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
package org.schabi.newpipe.extractor.services.youtube;
22

3+
import org.schabi.newpipe.extractor.search.filter.FilterItem;
4+
import org.schabi.newpipe.extractor.services.youtube.search.filter.YoutubeFilters;
5+
36
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO;
47
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.COMMENTS;
58
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.LIVE;
@@ -46,6 +49,7 @@
4649
import org.schabi.newpipe.extractor.stream.StreamExtractor;
4750
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
4851
import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
52+
import org.schabi.newpipe.extractor.utils.Utils;
4953

5054
import java.util.List;
5155

@@ -137,9 +141,10 @@ public PlaylistExtractor getPlaylistExtractor(final ListLinkHandler linkHandler)
137141

138142
@Override
139143
public SearchExtractor getSearchExtractor(final SearchQueryHandler query) {
140-
final List<String> contentFilters = query.getContentFilters();
144+
final FilterItem filterItem =
145+
Utils.getFirstContentFilterItem(query);
141146

142-
if (!contentFilters.isEmpty() && contentFilters.get(0).startsWith("music_")) {
147+
if (filterItem instanceof YoutubeFilters.MusicYoutubeContentFilterItem) {
143148
return new YoutubeMusicSearchExtractor(this, query);
144149
} else {
145150
return new YoutubeSearchExtractor(this, query);

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java‎

Lines changed: 19 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -29,14 +29,16 @@
2929
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
3030
import org.schabi.newpipe.extractor.search.SearchExtractor;
3131
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
32+
import org.schabi.newpipe.extractor.services.youtube.search.filter.YoutubeFilters;
3233
import org.schabi.newpipe.extractor.utils.JsonUtils;
34+
import org.schabi.newpipe.extractor.utils.Utils;
3335

3436
import java.io.IOException;
3537
import java.nio.charset.StandardCharsets;
3638
import java.util.Collections;
3739
import java.util.List;
38-
import java.util.stream.Collectors;
3940
import java.util.Objects;
41+
import java.util.stream.Collectors;
4042

4143
import javax.annotation.Nonnull;
4244
import javax.annotation.Nullable;
@@ -49,6 +51,15 @@ public YoutubeMusicSearchExtractor(final StreamingService service,
4951
super(service, linkHandler);
5052
}
5153

54+
private String getSearchType() {
55+
final YoutubeFilters.MusicYoutubeContentFilterItem contentFilterItem =
56+
Utils.getFirstContentFilterItem(getLinkHandler());
57+
if (contentFilterItem != null && contentFilterItem.getName() != null) {
58+
return contentFilterItem.getName();
59+
}
60+
return "";
61+
}
62+
5263
@Override
5364
public void onFetchPage(@Nonnull final Downloader downloader)
5465
throws IOException, ExtractionException {
@@ -57,28 +68,12 @@ public void onFetchPage(@Nonnull final Downloader downloader)
5768
final String url = "https://music.youtube.com/youtubei/v1/search?key="
5869
+ youtubeMusicKeys[0] + DISABLE_PRETTY_PRINT_PARAMETER;
5970

60-
final String params;
61-
62-
switch (getLinkHandler().getContentFilters().get(0)) {
63-
case MUSIC_SONGS:
64-
params = "Eg-KAQwIARAAGAAgACgAMABqChAEEAUQAxAKEAk%3D";
65-
break;
66-
case MUSIC_VIDEOS:
67-
params = "Eg-KAQwIABABGAAgACgAMABqChAEEAUQAxAKEAk%3D";
68-
break;
69-
case MUSIC_ALBUMS:
70-
params = "Eg-KAQwIABAAGAEgACgAMABqChAEEAUQAxAKEAk%3D";
71-
break;
72-
case MUSIC_PLAYLISTS:
73-
params = "Eg-KAQwIABAAGAAgACgBMABqChAEEAUQAxAKEAk%3D";
74-
break;
75-
case MUSIC_ARTISTS:
76-
params = "Eg-KAQwIABAAGAAgASgAMABqChAEEAUQAxAKEAk%3D";
77-
break;
78-
default:
79-
params = null;
80-
break;
81-
}
71+
72+
final YoutubeFilters.MusicYoutubeContentFilterItem contentFilterItem =
73+
Utils.getFirstContentFilterItem(getLinkHandler());
74+
// Get the search parameter for the request. If getParams() be null
75+
// (which should never happen - only in test cases), JsonWriter.string() can handle it
76+
final String params = (contentFilterItem != null) ? contentFilterItem.getParams() : null;
8277

8378
// @formatter:off
8479
final byte[] json = JsonWriter.string()
@@ -256,7 +251,7 @@ public InfoItemsPage<InfoItem> getPage(final Page page)
256251

257252
private void collectMusicStreamsFrom(final MultiInfoItemsCollector collector,
258253
@Nonnull final JsonArray videos) {
259-
final String searchType = getLinkHandler().getContentFilters().get(0);
254+
final String searchType = getSearchType();
260255
videos.stream()
261256
.filter(JsonObject.class::isInstance)
262257
.map(JsonObject.class::cast)

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java‎

Lines changed: 19 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,6 @@
66
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
77
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
88
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
9-
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.ALL;
10-
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.CHANNELS;
11-
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.PLAYLISTS;
12-
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.VIDEOS;
13-
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.getSearchParameter;
149
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
1510

1611
import com.grack.nanojson.JsonArray;
@@ -30,8 +25,11 @@
3025
import org.schabi.newpipe.extractor.localization.Localization;
3126
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
3227
import org.schabi.newpipe.extractor.search.SearchExtractor;
28+
import org.schabi.newpipe.extractor.search.filter.FilterItem;
3329
import org.schabi.newpipe.extractor.services.youtube.YoutubeMetaInfoHelper;
30+
import org.schabi.newpipe.extractor.services.youtube.search.filter.YoutubeFilters;
3431
import org.schabi.newpipe.extractor.utils.JsonUtils;
32+
import org.schabi.newpipe.extractor.utils.Utils;
3533

3634
import java.io.IOException;
3735
import java.nio.charset.StandardCharsets;
@@ -63,8 +61,6 @@
6361

6462
public class YoutubeSearchExtractor extends SearchExtractor {
6563

66-
@Nullable
67-
private final String searchType;
6864
private final boolean extractVideoResults;
6965
private final boolean extractChannelResults;
7066
private final boolean extractPlaylistResults;
@@ -74,25 +70,32 @@ public class YoutubeSearchExtractor extends SearchExtractor {
7470
public YoutubeSearchExtractor(final StreamingService service,
7571
final SearchQueryHandler linkHandler) {
7672
super(service, linkHandler);
77-
final List<String> contentFilters = linkHandler.getContentFilters();
78-
searchType = isNullOrEmpty(contentFilters) ? null : contentFilters.get(0);
73+
final List<FilterItem> contentFilters = linkHandler.getContentFilters();
74+
7975
// Save whether we should extract video, channel and playlist results depending on the
8076
// requested search type, as YouTube returns sometimes videos inside channel search results
8177
// If no search type is provided or ALL filter is requested, extract everything
82-
extractVideoResults = searchType == null || ALL.equals(searchType)
83-
|| VIDEOS.equals(searchType);
84-
extractChannelResults = searchType == null || ALL.equals(searchType)
85-
|| CHANNELS.equals(searchType);
86-
extractPlaylistResults = searchType == null || ALL.equals(searchType)
87-
|| PLAYLISTS.equals(searchType);
78+
final boolean nullOrAll = contentFilters == null || contentFilters.stream()
79+
.anyMatch(item -> item.getIdentifier() == YoutubeFilters.ID_CF_MAIN_ALL);
80+
extractVideoResults = nullOrAll || contentFilters.stream()
81+
.anyMatch(item -> item.getIdentifier() == YoutubeFilters.ID_CF_MAIN_VIDEOS);
82+
extractChannelResults = nullOrAll || contentFilters.stream()
83+
.anyMatch(item -> item.getIdentifier() == YoutubeFilters.ID_CF_MAIN_CHANNELS);
84+
extractPlaylistResults = nullOrAll || contentFilters.stream()
85+
.anyMatch(item -> item.getIdentifier() == YoutubeFilters.ID_CF_MAIN_PLAYLISTS);
8886
}
8987

9088
@Override
9189
public void onFetchPage(@Nonnull final Downloader downloader) throws IOException,
9290
ExtractionException {
9391
final String query = super.getSearchString();
9492
final Localization localization = getExtractorLocalization();
95-
final String params = getSearchParameter(searchType);
93+
94+
final YoutubeFilters.YoutubeContentFilterItem contentFilterItem =
95+
Utils.getFirstContentFilterItem(getLinkHandler());
96+
// Get the search parameter for the request. If getParams() be null
97+
// (which should never happen - only in test cases), JsonWriter.string() can handle it
98+
final String params = (contentFilterItem != null) ? contentFilterItem.getParams() : null;
9699

97100
final JsonBuilder<JsonObject> jsonBody = prepareDesktopJsonBuilder(localization,
98101
getExtractorContentCountry())

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@
2020

2121
package org.schabi.newpipe.extractor.services.youtube.linkHandler;
2222

23+
import org.schabi.newpipe.extractor.search.filter.FilterItem;
24+
2325
import org.schabi.newpipe.extractor.exceptions.ParsingException;
2426
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
2527
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
@@ -53,12 +55,14 @@ public static YoutubeChannelLinkHandlerFactory getInstance() {
5355
* Returns the URL to a channel from an ID.
5456
*
5557
* @param id the channel ID including e.g. 'channel/'
58+
* @param contentFilters
59+
* @param searchFilter
5660
* @return the URL to the channel
5761
*/
5862
@Override
5963
public String getUrl(final String id,
60-
final List<String> contentFilters,
61-
final String searchFilter)
64+
final List<FilterItem> contentFilters,
65+
final List<FilterItem> searchFilter)
6266
throws ParsingException, UnsupportedOperationException {
6367
return "https://www.youtube.com/" + id;
6468
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeCommentsLinkHandlerFactory.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
package org.schabi.newpipe.extractor.services.youtube.linkHandler;
22

3+
import org.schabi.newpipe.extractor.search.filter.FilterItem;
4+
35
import org.schabi.newpipe.extractor.exceptions.FoundAdException;
46
import org.schabi.newpipe.extractor.exceptions.ParsingException;
57
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
@@ -44,8 +46,8 @@ public boolean onAcceptUrl(final String url) throws FoundAdException {
4446

4547
@Override
4648
public String getUrl(final String id,
47-
final List<String> contentFilter,
48-
final String sortFilter)
49+
final List<FilterItem> contentFilter,
50+
final List<FilterItem> sortFilter)
4951
throws ParsingException, UnsupportedOperationException {
5052
return getUrl(id);
5153
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@
88
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
99
import org.schabi.newpipe.extractor.utils.Utils;
1010

11+
import org.schabi.newpipe.extractor.search.filter.FilterItem;
12+
1113
import java.net.MalformedURLException;
1214
import java.net.URL;
1315
import java.util.List;
@@ -25,8 +27,9 @@ public static YoutubePlaylistLinkHandlerFactory getInstance() {
2527
}
2628

2729
@Override
28-
public String getUrl(final String id, final List<String> contentFilters,
29-
final String sortFilter)
30+
public String getUrl(final String id,
31+
final List<FilterItem> contentFilters,
32+
final List<FilterItem> sortFilter)
3033
throws ParsingException, UnsupportedOperationException {
3134
return "https://www.youtube.com/playlist?list=" + id;
3235
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java‎

Lines changed: 13 additions & 77 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,16 @@
11
package org.schabi.newpipe.extractor.services.youtube.linkHandler;
22

3-
import static org.schabi.newpipe.extractor.utils.Utils.encodeUrlUtf8;
4-
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
5-
63
import org.schabi.newpipe.extractor.exceptions.ParsingException;
74
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
5+
import org.schabi.newpipe.extractor.search.filter.FilterItem;
6+
import org.schabi.newpipe.extractor.services.youtube.search.filter.YoutubeFilters;
87

9-
import java.io.UnsupportedEncodingException;
108
import java.util.List;
119

1210
import javax.annotation.Nonnull;
1311

1412
public final class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
1513

16-
private static final YoutubeSearchQueryHandlerFactory INSTANCE =
17-
new YoutubeSearchQueryHandlerFactory();
18-
1914
public static final String ALL = "all";
2015
public static final String VIDEOS = "videos";
2116
public static final String CHANNELS = "channels";
@@ -27,8 +22,12 @@ public final class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFa
2722
public static final String MUSIC_PLAYLISTS = "music_playlists";
2823
public static final String MUSIC_ARTISTS = "music_artists";
2924

30-
private static final String SEARCH_URL = "https://www.youtube.com/results?search_query=";
31-
private static final String MUSIC_SEARCH_URL = "https://music.youtube.com/search?q=";
25+
private static final YoutubeSearchQueryHandlerFactory INSTANCE =
26+
new YoutubeSearchQueryHandlerFactory();
27+
28+
private YoutubeSearchQueryHandlerFactory() {
29+
super(new YoutubeFilters());
30+
}
3231

3332
@Nonnull
3433
public static YoutubeSearchQueryHandlerFactory getInstance() {
@@ -37,73 +36,10 @@ public static YoutubeSearchQueryHandlerFactory getInstance() {
3736

3837
@Override
3938
public String getUrl(final String searchString,
40-
@Nonnull final List<String> contentFilters,
41-
final String sortFilter)
42-
throws ParsingException, UnsupportedOperationException {
43-
try {
44-
if (!contentFilters.isEmpty()) {
45-
final String contentFilter = contentFilters.get(0);
46-
switch (contentFilter) {
47-
case VIDEOS:
48-
return SEARCH_URL + encodeUrlUtf8(searchString)
49-
+ "&sp=EgIQAfABAQ%253D%253D";
50-
case CHANNELS:
51-
return SEARCH_URL + encodeUrlUtf8(searchString)
52-
+ "&sp=EgIQAvABAQ%253D%253D";
53-
case PLAYLISTS:
54-
return SEARCH_URL + encodeUrlUtf8(searchString)
55-
+ "&sp=EgIQA_ABAQ%253D%253D";
56-
case MUSIC_SONGS:
57-
case MUSIC_VIDEOS:
58-
case MUSIC_ALBUMS:
59-
case MUSIC_PLAYLISTS:
60-
case MUSIC_ARTISTS:
61-
return MUSIC_SEARCH_URL + encodeUrlUtf8(searchString);
62-
}
63-
}
64-
65-
return SEARCH_URL + encodeUrlUtf8(searchString) + "&sp=8AEB";
66-
} catch (final UnsupportedEncodingException e) {
67-
throw new ParsingException("Could not encode query", e);
68-
}
69-
}
70-
71-
@Override
72-
public String[] getAvailableContentFilter() {
73-
return new String[]{
74-
ALL,
75-
VIDEOS,
76-
CHANNELS,
77-
PLAYLISTS,
78-
MUSIC_SONGS,
79-
MUSIC_VIDEOS,
80-
MUSIC_ALBUMS,
81-
MUSIC_PLAYLISTS
82-
// MUSIC_ARTISTS
83-
};
84-
}
85-
86-
@Nonnull
87-
public static String getSearchParameter(final String contentFilter) {
88-
if (isNullOrEmpty(contentFilter)) {
89-
return "8AEB";
90-
}
91-
92-
switch (contentFilter) {
93-
case VIDEOS:
94-
return "EgIQAfABAQ%3D%3D";
95-
case CHANNELS:
96-
return "EgIQAvABAQ%3D%3D";
97-
case PLAYLISTS:
98-
return "EgIQA_ABAQ%3D%3D";
99-
case MUSIC_SONGS:
100-
case MUSIC_VIDEOS:
101-
case MUSIC_ALBUMS:
102-
case MUSIC_PLAYLISTS:
103-
case MUSIC_ARTISTS:
104-
return "";
105-
default:
106-
return "8AEB";
107-
}
39+
@Nonnull final List<FilterItem> selectedContentFilter,
40+
final List<FilterItem> selectedSortFilter) throws ParsingException {
41+
searchFilters.setSelectedContentFilter(selectedContentFilter);
42+
searchFilters.setSelectedSortFilter(selectedSortFilter);
43+
return searchFilters.evaluateSelectedFilters(searchString);
10844
}
10945
}

0 commit comments

Comments
 (0)