Skip to content

Commit 05e90c0

Browse files
ChanGu3ChanGu3
committed
refactor: AtomicInteger + ListLinkHandler + additons to youtubeparserhelper
1 parent c7e7404 commit 05e90c0

6 files changed

Lines changed: 123 additions & 55 deletions

File tree

‎extractor/src/main/java/org/schabi/newpipe/extractor/channel/tabs/rendererlist/RendererListInfoItem.java‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
11
package org.schabi.newpipe.extractor.channel.tabs.rendererlist;
22

33
import org.schabi.newpipe.extractor.InfoItem;
4+
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
45

56
public class RendererListInfoItem extends InfoItem {
67

78
private String title;
8-
private int parentListIndex;
9-
9+
private ListLinkHandler listLinkHandler;
1010
private String rendererListItemType;
1111

1212
public RendererListInfoItem(final int serviceId, final String url, final String name) {
@@ -21,12 +21,12 @@ public void setTitle(final String title) {
2121
this.title = title;
2222
}
2323

24-
public int getParentListIndex() {
25-
return parentListIndex;
24+
public ListLinkHandler getListLinkHandler() {
25+
return this.listLinkHandler;
2626
}
2727

28-
public void setParentListIndex(final int parentListIndex) {
29-
this.parentListIndex = parentListIndex;
28+
public void setListLinkHandler(final ListLinkHandler listLinkHandler) {
29+
this.listLinkHandler = listLinkHandler;
3030
}
3131

3232
public String getRendererListItemType() {

‎extractor/src/main/java/org/schabi/newpipe/extractor/channel/tabs/rendererlist/RendererListInfoItemExtractor.java‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,15 @@
22

33
import org.schabi.newpipe.extractor.InfoItemExtractor;
44
import org.schabi.newpipe.extractor.exceptions.ParsingException;
5+
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
56

67
public interface RendererListInfoItemExtractor extends InfoItemExtractor {
78

89
/**
9-
* Get the index of the parent list
10-
* @return the index of the parent list
10+
* Get the list link handler of the rendererlist
11+
* @return the ListLinkHandler of the list
1112
*/
12-
int getParentListIndex() throws ParsingException;
13+
ListLinkHandler getListLinkHandler() throws ParsingException;
1314

1415
/**
1516
* Get the item type that exist in the renderer list
@@ -40,4 +41,9 @@ public interface RendererListInfoItemExtractor extends InfoItemExtractor {
4041
* @return whether the uploader is verified
4142
*/
4243
boolean isUploaderVerified() throws ParsingException;
44+
45+
46+
static String getRendererListIndexContentFilter(final int index) {
47+
return "rendererlist_index=" + index;
48+
}
4349
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/channel/tabs/rendererlist/RendererListInfoItemsCollector.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ public RendererListInfoItem extract(final RendererListInfoItemExtractor extracto
2323
addError(e);
2424
}
2525
try {
26-
resultItem.setParentListIndex(extractor.getParentListIndex());
26+
resultItem.setListLinkHandler(extractor.getListLinkHandler());
2727
} catch (final Exception e) {
2828
addError(e);
2929
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@
4747
import org.jsoup.nodes.Entities;
4848
import org.schabi.newpipe.extractor.Image;
4949
import org.schabi.newpipe.extractor.Image.ResolutionLevel;
50+
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
5051
import org.schabi.newpipe.extractor.downloader.Response;
5152
import org.schabi.newpipe.extractor.exceptions.AccountTerminatedException;
5253
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
@@ -56,6 +57,7 @@
5657
import org.schabi.newpipe.extractor.localization.ContentCountry;
5758
import org.schabi.newpipe.extractor.localization.Localization;
5859
import org.schabi.newpipe.extractor.playlist.PlaylistInfo;
60+
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelTabLinkHandlerFactory;
5961
import org.schabi.newpipe.extractor.stream.AudioTrackType;
6062
import org.schabi.newpipe.extractor.utils.JsonUtils;
6163
import org.schabi.newpipe.extractor.utils.Parser;
@@ -73,6 +75,7 @@
7375
import java.util.Optional;
7476
import java.util.Random;
7577
import java.util.Set;
78+
import java.util.regex.Matcher;
7679
import java.util.regex.Pattern;
7780
import java.util.stream.Collectors;
7881
import java.util.stream.Stream;
@@ -252,6 +255,82 @@ public static int parseDurationString(@Nonnull final String input)
252255
return duration;
253256
}
254257

258+
/**
259+
* Parses a param string expecting the string to match exactly rendererlist_index={index}
260+
*
261+
* @return the index of the rendererlist
262+
* @throws ParsingException when the url does not match the expected format
263+
*/
264+
public static int parseRendererListIndexParam(@Nonnull final String input)
265+
throws ParsingException, NumberFormatException {
266+
final Pattern pattern = Pattern.compile("^rendererlist_index=(\\d+)$");
267+
final Matcher matcher = pattern.matcher(input);
268+
269+
if (!matcher.matches()) {
270+
throw new ParsingException(
271+
"Error url string does not match the expected format: "
272+
+ input);
273+
}
274+
275+
final String indexMatch = matcher.group(1);
276+
return Integer.parseInt(indexMatch);
277+
}
278+
279+
/**
280+
* Parses a json response of a channel using the tab name and
281+
* gets the tab data and returns tab data if content existence is verified
282+
*
283+
* @return the data of the channel tab
284+
*/
285+
public static Optional<JsonObject> getTabData(@Nonnull final JsonObject jsonResponse,
286+
@Nonnull final String channelTab)
287+
throws NumberFormatException {
288+
final String urlSuffix = YoutubeChannelTabLinkHandlerFactory.getUrlSuffix(channelTab);
289+
290+
return jsonResponse.getObject("contents")
291+
.getObject("twoColumnBrowseResultsRenderer")
292+
.getArray("tabs")
293+
.stream()
294+
.filter(JsonObject.class::isInstance)
295+
.map(JsonObject.class::cast)
296+
.filter(tab -> tab.has("tabRenderer"))
297+
.map(tab -> tab.getObject("tabRenderer"))
298+
.filter(tabRenderer -> tabRenderer.getObject("endpoint")
299+
.getObject("commandMetadata").getObject("webCommandMetadata")
300+
.getString("url", "").endsWith(urlSuffix))
301+
.findFirst()
302+
// Check if feature tab has no content
303+
.filter(tabRenderer -> {
304+
final JsonArray tabContents = tabRenderer.getObject("content")
305+
.getObject("sectionListRenderer")
306+
.getArray("contents")
307+
.getObject(0)
308+
.getObject("itemSectionRenderer")
309+
.getArray("contents");
310+
return tabContents.size() != 1
311+
|| !tabContents.getObject(0).has("messageRenderer");
312+
});
313+
}
314+
315+
/**
316+
* Parses a json response of the channel tab featured gets the renderlist data
317+
*
318+
* @return the data of the renderer list
319+
*/
320+
public static JsonObject getRendererListData(@Nonnull final JsonObject jsonResponse,
321+
final int rendererListIndex) {
322+
final Optional<JsonObject> tab = getTabData(jsonResponse, ChannelTabs.FEATURED);
323+
324+
return tab.map(jsonObject -> jsonObject.getObject("content")
325+
.getObject("sectionListRenderer")
326+
.getArray("contents")
327+
.getObject(rendererListIndex)
328+
.getObject("itemSectionRenderer")
329+
.getArray("contents")
330+
.getObject(0)
331+
).orElse(null);
332+
}
333+
255334
/**
256335
* Tries to convert a duration string to an integer without throwing an exception.
257336
* <br/>

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java‎

Lines changed: 6 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
1616
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
1717
import org.schabi.newpipe.extractor.services.youtube.YoutubeChannelHelper;
18+
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
1819
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelTabLinkHandlerFactory;
1920

2021
import javax.annotation.Nonnull;
@@ -23,6 +24,7 @@
2324
import java.nio.charset.StandardCharsets;
2425
import java.util.List;
2526
import java.util.Optional;
27+
import java.util.concurrent.atomic.AtomicInteger;
2628

2729
import static org.schabi.newpipe.extractor.services.youtube.YoutubeChannelHelper.getChannelResponse;
2830
import static org.schabi.newpipe.extractor.services.youtube.YoutubeChannelHelper.resolveChannelId;
@@ -48,8 +50,6 @@ public class YoutubeChannelTabExtractor extends ChannelTabExtractor {
4850
private JsonObject jsonResponse;
4951
private String channelId;
5052

51-
private final String itemIndexKey = "itemIndex";
52-
5353
public YoutubeChannelTabExtractor(final StreamingService service,
5454
final ListLinkHandler linkHandler) {
5555
super(service, linkHandler);
@@ -202,31 +202,7 @@ public InfoItemsPage<InfoItem> getPage(final Page page)
202202
}
203203

204204
Optional<JsonObject> getTabData() {
205-
final String urlSuffix = YoutubeChannelTabLinkHandlerFactory.getUrlSuffix(getName());
206-
207-
return jsonResponse.getObject("contents")
208-
.getObject("twoColumnBrowseResultsRenderer")
209-
.getArray("tabs")
210-
.stream()
211-
.filter(JsonObject.class::isInstance)
212-
.map(JsonObject.class::cast)
213-
.filter(tab -> tab.has("tabRenderer"))
214-
.map(tab -> tab.getObject("tabRenderer"))
215-
.filter(tabRenderer -> tabRenderer.getObject("endpoint")
216-
.getObject("commandMetadata").getObject("webCommandMetadata")
217-
.getString("url", "").endsWith(urlSuffix))
218-
.findFirst()
219-
// Check if tab has no content
220-
.filter(tabRenderer -> {
221-
final JsonArray tabContents = tabRenderer.getObject("content")
222-
.getObject("sectionListRenderer")
223-
.getArray("contents")
224-
.getObject(0)
225-
.getObject("itemSectionRenderer")
226-
.getArray("contents");
227-
return tabContents.size() != 1
228-
|| !tabContents.getObject(0).has("messageRenderer");
229-
});
205+
return YoutubeParsingHelper.getTabData(jsonResponse, getName());
230206
}
231207

232208
private Optional<JsonObject> collectItemsFrom(@Nonnull final MultiInfoItemsCollector collector,
@@ -263,21 +239,14 @@ private Optional<JsonObject> collectItemsFrom(@Nonnull final MultiInfoItemsColle
263239
@Nullable final String channelName,
264240
@Nullable final String channelUrl) {
265241

266-
// creating ItemIndex of the collectItemsFrom first call
267-
if (rootItemIndex == -1) {
268-
for (int i = 0; i < items.size(); i++) {
269-
if (items.get(i) instanceof JsonObject) {
270-
((JsonObject) items.get(i)).put(itemIndexKey, i);
271-
}
272-
}
273-
}
242+
final AtomicInteger itemIndexKey = new AtomicInteger(0);
274243

275244
return items.stream()
276245
.filter(JsonObject.class::isInstance)
277246
.map(JsonObject.class::cast)
278247
.map(item ->
279248
collectItem(collector,
280-
(rootItemIndex == -1 ? item.getInt(itemIndexKey) : rootItemIndex),
249+
(rootItemIndex == -1) ? itemIndexKey.getAndIncrement() : rootItemIndex,
281250
item, verifiedStatus, channelName, channelUrl)
282251
)
283252
.reduce(Optional.empty(), (c1, c2) -> c1.or(() -> c2));
@@ -553,7 +522,7 @@ private void commitRendererList(@Nonnull final MultiInfoItemsCollector collector
553522

554523
if (listItemsType != null) {
555524
collector.commit(new YoutubeShelfRendererListInfoItemExtractor(jsonObject,
556-
itemIndex, listItemsType) {
525+
listItemsType, this.channelId, ChannelTabs.FEATURED, itemIndex) {
557526
@Override
558527
public String getUploaderName() throws ParsingException {
559528
return isNullOrEmpty(channelName) ? super.getUploaderName() : channelName;

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeShelfRendererListInfoItemExtractor.java‎

Lines changed: 22 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@
99
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs;
1010
import org.schabi.newpipe.extractor.channel.tabs.rendererlist.RendererListInfoItemExtractor;
1111
import org.schabi.newpipe.extractor.exceptions.ParsingException;
12+
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
13+
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
1214
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelTabLinkHandlerFactory;
1315

1416
import java.util.List;
@@ -19,15 +21,23 @@ public class YoutubeShelfRendererListInfoItemExtractor implements RendererListIn
1921

2022
public static final String FEATURED_CHANNEL_LIST = "FEATURED_CHANNELS_LIST";
2123
private final JsonObject rendererListInfoItem;
22-
private final int parentListIndex;
23-
2424
private final String rendererListItemType;
2525

26+
private final String id;
27+
private final List<String> contentFilter;
28+
2629
public YoutubeShelfRendererListInfoItemExtractor(final JsonObject rendererListInfoItem,
27-
final int parentListIndex, final String rendererListItemType) {
30+
final String rendererListItemType,
31+
final String id,
32+
final String tab,
33+
final int index) {
2834
this.rendererListInfoItem = rendererListInfoItem;
29-
this.parentListIndex = parentListIndex;
3035
this.rendererListItemType = rendererListItemType;
36+
this.contentFilter = List.of(
37+
tab,
38+
RendererListInfoItemExtractor
39+
.getRendererListIndexContentFilter(index));
40+
this.id = id;
3141
}
3242

3343

@@ -55,8 +65,11 @@ public String getUrl() throws ParsingException {
5565
if (url == null) {
5666
final String uploaderTabURL = getUploaderTabUrl();
5767

58-
if (uploaderTabURL != null) {
59-
return uploaderTabURL + "/rendererlist/" + parentListIndex; // virtual url
68+
if (uploaderTabURL != null && contentFilter.get(1) != null) {
69+
final int index = YoutubeParsingHelper
70+
.parseRendererListIndexParam(contentFilter.get(1));
71+
// virtual url since they are a list
72+
return uploaderTabURL + "/rendererlist/" + index;
6073
}
6174
}
6275

@@ -73,8 +86,9 @@ public List<Image> getThumbnails() throws ParsingException {
7386
}
7487

7588
@Override
76-
public int getParentListIndex() throws ParsingException {
77-
return parentListIndex;
89+
public ListLinkHandler getListLinkHandler() throws ParsingException {
90+
final String tabUrl = getUploaderTabUrl();
91+
return new ListLinkHandler(tabUrl, tabUrl, this.id, this.contentFilter, "");
7892
}
7993

8094
@Override

0 commit comments

Comments
 (0)