Skip to content

Commit 2aa5f98

Browse files
litetexlitetex
authored
Merge pull request #796 from FireMasterK/streaminfo-subscriber-count
Add support for extracting channel subscriber count in StreamInfo
2 parents a267093 + e6d3347 commit 2aa5f98

12 files changed

Lines changed: 92 additions & 30 deletions

‎extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@
2828

2929
public abstract class ChannelExtractor extends ListExtractor<StreamInfoItem> {
3030

31+
public static final long UNKNOWN_SUBSCRIBER_COUNT = -1;
32+
3133
public ChannelExtractor(StreamingService service, ListLinkHandler linkHandler) {
3234
super(service, linkHandler);
3335
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java‎

Lines changed: 20 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -89,9 +89,9 @@ public void onFetchPage(@Nonnull final Downloader downloader) throws IOException
8989
// we couldn't get information about the channel associated with this URL, if there is one.
9090
if (!channelId[0].equals("channel")) {
9191
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
92-
getExtractorLocalization(), getExtractorContentCountry())
93-
.value("url", "https://www.youtube.com/" + channelPath)
94-
.done())
92+
getExtractorLocalization(), getExtractorContentCountry())
93+
.value("url", "https://www.youtube.com/" + channelPath)
94+
.done())
9595
.getBytes(UTF_8);
9696

9797
final JsonObject jsonResponse = getJsonPostResponse("navigation/resolve_url",
@@ -104,8 +104,8 @@ public void onFetchPage(@Nonnull final Downloader downloader) throws IOException
104104
throw new ContentNotAvailableException("This channel doesn't exist.");
105105
} else {
106106
throw new ContentNotAvailableException("Got error:\""
107-
+ errorJsonObject.getString("status") + "\": "
108-
+ errorJsonObject.getString("message"));
107+
+ errorJsonObject.getString("status") + "\": "
108+
+ errorJsonObject.getString("message"));
109109
}
110110
}
111111

@@ -136,10 +136,10 @@ public void onFetchPage(@Nonnull final Downloader downloader) throws IOException
136136
int level = 0;
137137
while (level < 3) {
138138
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
139-
getExtractorLocalization(), getExtractorContentCountry())
140-
.value("browseId", id)
141-
.value("params", "EgZ2aWRlb3M%3D") // Equal to videos
142-
.done())
139+
getExtractorLocalization(), getExtractorContentCountry())
140+
.value("browseId", id)
141+
.value("params", "EgZ2aWRlb3M%3D") // Equal to videos
142+
.done())
143143
.getBytes(UTF_8);
144144

145145
final JsonObject jsonResponse = getJsonPostResponse("browse", body,
@@ -273,15 +273,14 @@ public String getFeedUrl() throws ParsingException {
273273
public long getSubscriberCount() throws ParsingException {
274274
final JsonObject c4TabbedHeaderRenderer = initialData.getObject("header")
275275
.getObject("c4TabbedHeaderRenderer");
276-
if (c4TabbedHeaderRenderer.has("subscriberCountText")) {
277-
try {
278-
return Utils.mixedNumberWordToLong(getTextFromObject(c4TabbedHeaderRenderer
279-
.getObject("subscriberCountText")));
280-
} catch (final NumberFormatException e) {
281-
throw new ParsingException("Could not get subscriber count", e);
282-
}
283-
} else {
284-
return ITEM_COUNT_UNKNOWN;
276+
if (!c4TabbedHeaderRenderer.has("subscriberCountText")) {
277+
return UNKNOWN_SUBSCRIBER_COUNT;
278+
}
279+
try {
280+
return Utils.mixedNumberWordToLong(getTextFromObject(c4TabbedHeaderRenderer
281+
.getObject("subscriberCountText")));
282+
} catch (final NumberFormatException e) {
283+
throw new ParsingException("Could not get subscriber count", e);
285284
}
286285
}
287286

@@ -385,9 +384,9 @@ private Page getNextPageFrom(final JsonObject continuations,
385384
.getString("token");
386385

387386
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(getExtractorLocalization(),
388-
getExtractorContentCountry())
389-
.value("continuation", continuation)
390-
.done())
387+
getExtractorContentCountry())
388+
.value("continuation", continuation)
389+
.done())
391390
.getBytes(UTF_8);
392391

393392
return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey(), null, channelIds, null, body);

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java‎

Lines changed: 22 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -180,7 +180,7 @@ public String getTextualUploadDate() throws ParsingException {
180180
// TODO: this parses English formatted dates only, we need a better approach to parse
181181
// the textual date
182182
final LocalDate localDate = LocalDate.parse(getTextFromObject(
183-
getVideoPrimaryInfoRenderer().getObject("dateText")),
183+
getVideoPrimaryInfoRenderer().getObject("dateText")),
184184
DateTimeFormatter.ofPattern("dd MMM yyyy", Locale.ENGLISH));
185185
return DateTimeFormatter.ISO_LOCAL_DATE.format(localDate);
186186
} catch (final Exception ignored) {
@@ -437,6 +437,19 @@ public String getUploaderAvatarUrl() throws ParsingException {
437437
return fixThumbnailUrl(url);
438438
}
439439

440+
@Override
441+
public long getUploaderSubscriberCount() throws ParsingException {
442+
final JsonObject videoOwnerRenderer = JsonUtils.getObject(videoSecondaryInfoRenderer, "owner.videoOwnerRenderer");
443+
if (!videoOwnerRenderer.has("subscriberCountText")) {
444+
return UNKNOWN_SUBSCRIBER_COUNT;
445+
}
446+
try {
447+
return Utils.mixedNumberWordToLong(getTextFromObject(videoOwnerRenderer.getObject("subscriberCountText")));
448+
} catch (final NumberFormatException e) {
449+
throw new ParsingException("Could not get uploader subscriber count", e);
450+
}
451+
}
452+
440453
@Nonnull
441454
@Override
442455
public String getDashMpdUrl() throws ParsingException {
@@ -666,9 +679,9 @@ public void onFetchPage(@Nonnull final Downloader downloader)
666679
final Localization localization = getExtractorLocalization();
667680
final ContentCountry contentCountry = getExtractorContentCountry();
668681
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
669-
localization, contentCountry)
670-
.value("videoId", videoId)
671-
.done())
682+
localization, contentCountry)
683+
.value("videoId", videoId)
684+
.done())
672685
.getBytes(UTF_8);
673686

674687
// Put the sts string if we already know it so we don't have to fetch again the player
@@ -717,8 +730,8 @@ public void onFetchPage(@Nonnull final Downloader downloader)
717730

718731
if (ageRestricted) {
719732
final byte[] ageRestrictedBody = JsonWriter.string(prepareDesktopEmbedVideoJsonBuilder(
720-
localization, contentCountry, videoId)
721-
.done())
733+
localization, contentCountry, videoId)
734+
.done())
722735
.getBytes(UTF_8);
723736
nextResponse = getJsonPostResponse("next", ageRestrictedBody, localization);
724737
} else {
@@ -800,7 +813,7 @@ private void fetchAndroidMobileJsonPlayer(final ContentCountry contentCountry,
800813
final String videoId)
801814
throws IOException, ExtractionException {
802815
final byte[] mobileBody = JsonWriter.string(prepareAndroidMobileJsonBuilder(
803-
localization, contentCountry)
816+
localization, contentCountry)
804817
.value("videoId", videoId)
805818
.done())
806819
.getBytes(UTF_8);
@@ -873,8 +886,8 @@ private void fetchAndroidEmbedJsonPlayer(final ContentCountry contentCountry,
873886
final String videoId)
874887
throws IOException, ExtractionException {
875888
final byte[] androidMobileEmbedBody = JsonWriter.string(
876-
prepareAndroidMobileEmbedVideoJsonBuilder(localization, contentCountry, videoId)
877-
.done())
889+
prepareAndroidMobileEmbedVideoJsonBuilder(localization, contentCountry, videoId)
890+
.done())
878891
.getBytes(UTF_8);
879892
final JsonObject androidMobileEmbedPlayerResponse = getJsonMobilePostResponse("player",
880893
androidMobileEmbedBody, contentCountry, localization);

‎extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@
4747
public abstract class StreamExtractor extends Extractor {
4848

4949
public static final int NO_AGE_LIMIT = 0;
50+
public static final long UNKNOWN_SUBSCRIBER_COUNT = -1;
5051

5152
public StreamExtractor(StreamingService service, LinkHandler linkHandler) {
5253
super(service, linkHandler);
@@ -202,6 +203,17 @@ public boolean isUploaderVerified() throws ParsingException {
202203
return false;
203204
}
204205

206+
/**
207+
* The subscriber count of the uploader.
208+
* If the subscriber count is not implemented, or is unavailable, return <code>-1</code>.
209+
*
210+
* @return the subscriber count of the uploader or {@value UNKNOWN_SUBSCRIBER_COUNT} if not available
211+
* @throws ParsingException
212+
*/
213+
public long getUploaderSubscriberCount() throws ParsingException {
214+
return UNKNOWN_SUBSCRIBER_COUNT;
215+
}
216+
205217
/**
206218
* The url to the image file/profile picture/avatar of the creator/uploader of the stream.
207219
* If the url is not available you can return an empty String.

‎extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -235,6 +235,11 @@ private static StreamInfo extractOptionalData(StreamInfo streamInfo, StreamExtra
235235
} catch (Exception e) {
236236
streamInfo.addError(e);
237237
}
238+
try {
239+
streamInfo.setUploaderSubscriberCount(extractor.getUploaderSubscriberCount());
240+
} catch (Exception e) {
241+
streamInfo.addError(e);
242+
}
238243

239244
try {
240245
streamInfo.setSubChannelName(extractor.getSubChannelName());
@@ -367,6 +372,7 @@ private static StreamInfo extractOptionalData(StreamInfo streamInfo, StreamExtra
367372
private String uploaderUrl = "";
368373
private String uploaderAvatarUrl = "";
369374
private boolean uploaderVerified = false;
375+
private long uploaderSubscriberCount = -1;
370376

371377
private String subChannelName = "";
372378
private String subChannelUrl = "";
@@ -540,6 +546,14 @@ public void setUploaderVerified(final boolean uploaderVerified) {
540546
this.uploaderVerified = uploaderVerified;
541547
}
542548

549+
public long getUploaderSubscriberCount() {
550+
return uploaderSubscriberCount;
551+
}
552+
553+
public void setUploaderSubscriberCount(long uploaderSubscriberCount) {
554+
this.uploaderSubscriberCount = uploaderSubscriberCount;
555+
}
556+
543557
public String getSubChannelName() {
544558
return subChannelName;
545559
}

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseStreamExtractorTest.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ public interface BaseStreamExtractorTest extends BaseExtractorTest {
55
void testUploaderName() throws Exception;
66
void testUploaderUrl() throws Exception;
77
void testUploaderAvatarUrl() throws Exception;
8+
void testSubscriberCount() throws Exception;
89
void testSubChannelName() throws Exception;
910
void testSubChannelUrl() throws Exception;
1011
void testSubChannelAvatarUrl() throws Exception;

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
3535
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsValidUrl;
3636
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestListOfItems;
37+
import static org.schabi.newpipe.extractor.stream.StreamExtractor.UNKNOWN_SUBSCRIBER_COUNT;
3738

3839
/**
3940
* Test for {@link StreamExtractor}
@@ -45,6 +46,7 @@ public abstract class DefaultStreamExtractorTest extends DefaultExtractorTest<St
4546
public abstract String expectedUploaderName();
4647
public abstract String expectedUploaderUrl();
4748
public boolean expectedUploaderVerified() { return false; }
49+
public long expectedUploaderSubscriberCountAtLeast() { return UNKNOWN_SUBSCRIBER_COUNT; }
4850
public String expectedSubChannelName() { return ""; } // default: there is no subchannel
4951
public String expectedSubChannelUrl() { return ""; } // default: there is no subchannel
5052
public boolean expectedDescriptionIsEmpty() { return false; } // default: description is not empty
@@ -105,6 +107,16 @@ public void testUploaderVerified() throws Exception {
105107
assertEquals(expectedUploaderVerified(), extractor().isUploaderVerified());
106108
}
107109

110+
@Test
111+
@Override
112+
public void testSubscriberCount() throws Exception {
113+
if (expectedUploaderSubscriberCountAtLeast() == UNKNOWN_SUBSCRIBER_COUNT) {
114+
assertEquals(UNKNOWN_SUBSCRIBER_COUNT, extractor().getUploaderSubscriberCount());
115+
} else {
116+
assertGreaterOrEqual(expectedUploaderSubscriberCountAtLeast(), extractor().getUploaderSubscriberCount());
117+
}
118+
}
119+
108120
@Test
109121
@Override
110122
public void testSubChannelName() throws Exception {

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ public static void setUp() throws Exception {
4545
@Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
4646
@Override public String expectedUploaderName() { return "DAN TV"; }
4747
@Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCcQHIVL83g5BEQe2IJFb-6w"; }
48+
@Override public long expectedUploaderSubscriberCountAtLeast() { return 50; }
4849
@Override public boolean expectedUploaderVerified() { return false; }
4950
@Override public boolean expectedDescriptionIsEmpty() { return true; }
5051
@Override public List<String> expectedDescriptionContains() { return Collections.emptyList(); }

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ public static void setUp() throws Exception {
4949
@Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
5050
@Override public String expectedUploaderName() { return "Amazing Atheist"; }
5151
@Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCjNxszyFPasDdRoD9J6X-sw"; }
52+
@Override public long expectedUploaderSubscriberCountAtLeast() { return 977_000; }
5253
@Override public List<String> expectedDescriptionContains() {
5354
return Arrays.asList("http://www.huffingtonpost.com/2010/09/09/obama-gma-interview-quran_n_710282.html",
5455
"freedom");

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,7 @@ public static void setUp() throws Exception {
138138
@Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
139139
@Override public String expectedUploaderName() { return "PewDiePie"; }
140140
@Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UC-lHJZR3Gqxm24_Vd_AJ5Yw"; }
141+
@Override public long expectedUploaderSubscriberCountAtLeast() { return 110_000_000; }
141142
@Override public List<String> expectedDescriptionContains() {
142143
return Arrays.asList("https://www.youtube.com/channel/UC7l23W7gFi4Uho6WSzckZRA",
143144
"https://www.handcraftpictures.com/");
@@ -182,6 +183,7 @@ public static void setUp() throws Exception {
182183
@Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
183184
@Override public String expectedUploaderName() { return "Unbox Therapy"; }
184185
@Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCsTcErHg8oDvUnTzoqsYeNw"; }
186+
@Override public long expectedUploaderSubscriberCountAtLeast() { return 18_000_000; }
185187
@Override public List<String> expectedDescriptionContains() {
186188
return Arrays.asList("https://www.youtube.com/watch?v=X7FLCHVXpsA&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34",
187189
"https://www.youtube.com/watch?v=Lqv6G0pDNnw&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34",
@@ -274,6 +276,7 @@ public static void setUp() throws Exception {
274276
@Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
275277
@Override public String expectedUploaderName() { return "tagesschau"; }
276278
@Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UC5NOEUbkLheQcaaRldYW5GA"; }
279+
@Override public long expectedUploaderSubscriberCountAtLeast() { return 1_000_000; }
277280
@Override public boolean expectedUploaderVerified() { return true; }
278281
@Override public List<String> expectedDescriptionContains() {
279282
return Arrays.asList("Themen der Sendung", "07:15", "Wetter", "Sendung nachträglich bearbeitet");
@@ -336,6 +339,7 @@ public static void setUp() throws Exception {
336339
@Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
337340
@Override public String expectedUploaderName() { return "maiLab"; }
338341
@Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCyHDQ5C6z1NDmJ4g6SerW8g"; }
342+
@Override public long expectedUploaderSubscriberCountAtLeast() { return 1_400_000; }
339343
@Override public List<String> expectedDescriptionContains() {return Arrays.asList("Vitamin", "2:44", "Was ist Vitamin D?");}
340344
@Override public boolean expectedUploaderVerified() { return true; }
341345
@Override public long expectedLength() { return 1010; }
@@ -405,6 +409,7 @@ public static void setUp() throws Exception {
405409
@Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
406410
@Override public String expectedUploaderName() { return "Dinge Erklärt – Kurzgesagt"; }
407411
@Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCwRH985XgMYXQ6NxXDo8npw"; }
412+
@Override public long expectedUploaderSubscriberCountAtLeast() { return 1_500_000; }
408413
@Override public List<String> expectedDescriptionContains() { return Arrays.asList("Lasst uns abtauchen!", "Angebot von funk", "Dinge"); }
409414
@Override public long expectedLength() { return 631; }
410415
@Override public long expectedTimestamp() { return TIMESTAMP; }

0 commit comments

Comments
 (0)