Skip to content

Commit 558a2f3

Browse files
TobiGrTobiGr
committed
Add SongMetadata for YouTube Music and Bandcamp
1 parent ba70ea7 commit 558a2f3

6 files changed

Lines changed: 294 additions & 0 deletions

File tree

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemsCollector;
2727
import org.schabi.newpipe.extractor.stream.AudioStream;
2828
import org.schabi.newpipe.extractor.stream.Description;
29+
import org.schabi.newpipe.extractor.stream.SongMetadata;
2930
import org.schabi.newpipe.extractor.stream.StreamExtractor;
3031
import org.schabi.newpipe.extractor.stream.StreamType;
3132
import org.schabi.newpipe.extractor.stream.VideoStream;
@@ -244,4 +245,22 @@ public List<String> getTags() {
244245
.map(Element::text)
245246
.collect(Collectors.toList());
246247
}
248+
249+
@Nullable
250+
@Override
251+
public SongMetadata getSongMetadata() throws ParsingException {
252+
final SongMetadata.Builder builder = new SongMetadata.Builder(
253+
getName(), getUploaderName())
254+
.setAlbum(current.getString("album_title"))
255+
.setTrack(current.getInt("track_number"))
256+
.setReleaseDate(getUploadDate());
257+
if (!current.getArray("packages").isEmpty()) {
258+
final JsonObject packageInfo = current.getArray("packages").getObject(0);
259+
if (!packageInfo.getString("label").isEmpty()) {
260+
builder.setLabel(packageInfo.getString("label"));
261+
}
262+
}
263+
return builder.build();
264+
}
265+
247266
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@
4444
import com.grack.nanojson.JsonParserException;
4545
import com.grack.nanojson.JsonWriter;
4646

47+
import org.jsoup.Jsoup;
4748
import org.jsoup.nodes.Entities;
4849
import org.schabi.newpipe.extractor.Image;
4950
import org.schabi.newpipe.extractor.Image.ResolutionLevel;
@@ -54,9 +55,12 @@
5455
import org.schabi.newpipe.extractor.exceptions.ParsingException;
5556
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
5657
import org.schabi.newpipe.extractor.localization.ContentCountry;
58+
import org.schabi.newpipe.extractor.localization.DateWrapper;
5759
import org.schabi.newpipe.extractor.localization.Localization;
5860
import org.schabi.newpipe.extractor.playlist.PlaylistInfo;
5961
import org.schabi.newpipe.extractor.stream.AudioTrackType;
62+
import org.schabi.newpipe.extractor.stream.Description;
63+
import org.schabi.newpipe.extractor.stream.SongMetadata;
6064
import org.schabi.newpipe.extractor.utils.JsonUtils;
6165
import org.schabi.newpipe.extractor.utils.Parser;
6266
import org.schabi.newpipe.extractor.utils.RandomStringFromAlphabetGenerator;
@@ -66,13 +70,16 @@
6670
import java.net.MalformedURLException;
6771
import java.net.URL;
6872
import java.nio.charset.StandardCharsets;
73+
import java.time.Instant;
74+
import java.time.format.DateTimeParseException;
6975
import java.util.HashMap;
7076
import java.util.List;
7177
import java.util.Locale;
7278
import java.util.Map;
7379
import java.util.Optional;
7480
import java.util.Random;
7581
import java.util.Set;
82+
import java.util.regex.Matcher;
7683
import java.util.regex.Pattern;
7784
import java.util.stream.Collectors;
7885
import java.util.stream.Stream;
@@ -1583,4 +1590,50 @@ public static JsonObject getFirstCollaborator(final JsonObject navigationEndpoin
15831590
return null;
15841591
}
15851592
}
1593+
1594+
private static final Pattern SONG_METADATA_PATTERN = Pattern.compile(
1595+
"Provided to YouTube by (.+)\\n\\n" // label
1596+
+ "(.+)\\s\\u00b7\\s([\\w\\s]+)\\n\\n" // title and artist
1597+
+ "(.+)\\n\\n" // album
1598+
+ "(.+)\\n\\n" // publisher
1599+
+ "Released on:\\s([\\d-]+)\\n" // release date
1600+
+ "([\\s\\S]+)\\n" // performers, composers, etc.
1601+
+ "Auto-generated by YouTube."
1602+
);
1603+
@Nullable
1604+
public static SongMetadata getSongMetadata(@Nonnull final Description description)
1605+
throws ParsingException {
1606+
final String descriptionText;
1607+
if (description.getType() == Description.PLAIN_TEXT) {
1608+
descriptionText = description.getContent();
1609+
} else {
1610+
descriptionText = Jsoup.parse(description.getContent()).text();
1611+
}
1612+
final Matcher matcher = SONG_METADATA_PATTERN.matcher(descriptionText);
1613+
if (matcher.find()) {
1614+
final var builder = new SongMetadata.Builder(matcher.group(2), matcher.group(3))
1615+
.setAlbum(matcher.group(4))
1616+
.setCopyright(matcher.group(5).replace("℗", "").trim());
1617+
final String releaseDateString = matcher.group(6);
1618+
try {
1619+
builder.setReleaseDate(new DateWrapper(
1620+
Instant.parse(releaseDateString + "T00:00:00Z"), false));
1621+
} catch (final DateTimeParseException e) {
1622+
// Ignore parsing errors for the release date, as it's not critical information.
1623+
}
1624+
try {
1625+
final String collaborators = matcher.group(7);
1626+
for (final String line : collaborators.split("\\n")) {
1627+
final String[] parts = line.split(":");
1628+
if (parts.length == 2) {
1629+
builder.addPerformer(parts[1].trim());
1630+
}
1631+
}
1632+
} catch (final Exception ignored) {
1633+
// Ignore parsing errors for collaborators, as it's not critical information.
1634+
}
1635+
return builder.build();
1636+
}
1637+
return null;
1638+
}
15861639
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@
7373
import org.schabi.newpipe.extractor.stream.DeliveryMethod;
7474
import org.schabi.newpipe.extractor.stream.Description;
7575
import org.schabi.newpipe.extractor.stream.Frameset;
76+
import org.schabi.newpipe.extractor.stream.SongMetadata;
7677
import org.schabi.newpipe.extractor.stream.Stream;
7778
import org.schabi.newpipe.extractor.stream.StreamExtractor;
7879
import org.schabi.newpipe.extractor.stream.StreamSegment;
@@ -1621,6 +1622,21 @@ public List<MetaInfo> getMetaInfo() throws ParsingException {
16211622
.getArray("contents"));
16221623
}
16231624

1625+
@Nullable
1626+
@Override
1627+
public SongMetadata getSongMetadata() throws ParsingException {
1628+
assertPageFetched();
1629+
// The song info is only available for music videos
1630+
final String attributedDescription = getVideoSecondaryInfoRenderer()
1631+
.getObject("attributedDescription")
1632+
.getString("content");
1633+
if (isNullOrEmpty(attributedDescription)) {
1634+
return null;
1635+
}
1636+
return YoutubeParsingHelper.getSongMetadata(
1637+
new Description(attributedDescription, Description.PLAIN_TEXT));
1638+
}
1639+
16241640
/**
16251641
* Set the {@link PoTokenProvider} instance to be used for fetching {@code poToken}s.
16261642
*

‎extractor/src/main/java/org/schabi/newpipe/extractor/stream/SongMetadata.java‎

Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
2+
package org.schabi.newpipe.extractor.stream;
3+
4+
import org.schabi.newpipe.extractor.localization.DateWrapper;
5+
6+
import javax.annotation.Nonnull;
7+
import javax.annotation.Nullable;
8+
9+
import java.io.Serializable;
10+
import java.time.Duration;
11+
import java.util.ArrayList;
12+
import java.util.Collections;
13+
import java.util.List;
14+
15+
/**
16+
* Class representing the information on a song or single.
17+
*/
18+
public final class SongMetadata implements Serializable {
19+
/**
20+
* Constant representing that the track number of a {@link SongMetadata} is unknown.
21+
*/
22+
public static final int TRACK_UNKNOWN = -1;
23+
24+
@Nonnull
25+
public final String title;
26+
@Nonnull
27+
public final String artist;
28+
@Nonnull
29+
public final List<String> performer;
30+
@Nullable
31+
public final String composer;
32+
@Nullable
33+
public final String genre;
34+
@Nullable
35+
public final String album;
36+
public final int track;
37+
@Nullable
38+
public final Duration duration;
39+
@Nullable
40+
public final DateWrapper releaseDate;
41+
@Nullable
42+
public final String label;
43+
@Nullable
44+
public final String copyright;
45+
@Nullable
46+
public final String location;
47+
48+
public SongMetadata(@Nonnull final String title, @Nonnull final String artist,
49+
@Nonnull final List<String> performer, @Nullable final String composer,
50+
@Nullable final String genre, @Nullable final String album,
51+
final int track, @Nullable final Duration duration,
52+
@Nullable final DateWrapper releaseDate, @Nullable final String label,
53+
@Nullable final String copyright, @Nullable final String location) {
54+
this.title = title;
55+
this.artist = artist;
56+
this.performer = performer;
57+
this.composer = composer;
58+
this.genre = genre;
59+
this.album = album;
60+
this.track = track;
61+
this.duration = duration;
62+
this.releaseDate = releaseDate;
63+
this.label = label;
64+
this.copyright = copyright;
65+
this.location = location;
66+
}
67+
68+
public static final class Builder {
69+
@Nonnull
70+
private final String mTitle;
71+
@Nonnull
72+
private final String mArtist;
73+
@Nonnull
74+
private List<String> mPerformer = new ArrayList<>();
75+
@Nullable
76+
private String mComposer;
77+
@Nullable
78+
private String mGenre;
79+
@Nullable
80+
private String mAlbum;
81+
private int mTrack = TRACK_UNKNOWN;
82+
@Nullable
83+
private Duration mDuration;
84+
@Nullable
85+
private DateWrapper mReleaseDate;
86+
@Nullable
87+
private String mLabel;
88+
@Nullable
89+
private String mCopyright;
90+
@Nullable
91+
private String mLocation;
92+
93+
public Builder(@Nonnull final String title, @Nonnull final String artist) {
94+
this.mTitle = title;
95+
this.mArtist = artist;
96+
}
97+
98+
public Builder setPerformer(@Nonnull final List<String> performer) {
99+
this.mPerformer = performer;
100+
return this;
101+
}
102+
103+
public Builder addPerformer(@Nonnull final String performer) {
104+
this.mPerformer.add(performer);
105+
return this;
106+
}
107+
108+
109+
public Builder setComposer(@Nullable final String composer) {
110+
this.mComposer = composer;
111+
return this;
112+
}
113+
114+
public Builder setGenre(@Nullable final String genre) {
115+
this.mGenre = genre;
116+
return this;
117+
}
118+
119+
public Builder setAlbum(@Nullable final String album) {
120+
this.mAlbum = album;
121+
return this;
122+
}
123+
124+
public Builder setTrack(final int track) {
125+
this.mTrack = track;
126+
return this;
127+
}
128+
129+
public Builder setDuration(@Nullable final Duration duration) {
130+
this.mDuration = duration;
131+
return this;
132+
}
133+
134+
public Builder setReleaseDate(@Nullable final DateWrapper releaseDate) {
135+
this.mReleaseDate = releaseDate;
136+
return this;
137+
}
138+
139+
public Builder setLabel(@Nullable final String label) {
140+
this.mLabel = label;
141+
return this;
142+
}
143+
144+
public Builder setCopyright(@Nullable final String copyright) {
145+
this.mCopyright = copyright;
146+
return this;
147+
}
148+
149+
public Builder setLocation(@Nullable final String location) {
150+
this.mLocation = location;
151+
return this;
152+
}
153+
154+
public SongMetadata build() {
155+
return new SongMetadata(
156+
mTitle, mArtist, Collections.unmodifiableList(mPerformer), mComposer, mGenre,
157+
mAlbum, mTrack, mDuration, mReleaseDate, mLabel, mCopyright, mLocation);
158+
}
159+
}
160+
161+
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -593,6 +593,23 @@ public ContentAvailability getContentAvailability() throws ParsingException {
593593
return ContentAvailability.UNKNOWN;
594594
}
595595

596+
/**
597+
* Get the song info if this stream is a music stream and the service provides this information.
598+
* <p>
599+
* A SongInfo is only provided if the information on the song does not match the StreamInfo's
600+
* title and uploader or if the service explicitly provides this information.
601+
* Otherwise, the StreamInfo's title and uploader are assumed to be the song's title and artist.
602+
* </p>
603+
*
604+
* @return the song info or null if this stream is not a music stream
605+
* or if the service does not provide this information
606+
* @throws ParsingException if there is an error in the extraction
607+
*/
608+
@Nullable
609+
public SongMetadata getSongMetadata() throws ParsingException {
610+
return null;
611+
}
612+
596613
public enum Privacy {
597614
PUBLIC,
598615
UNLISTED,

‎extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
import java.util.Locale;
3838

3939
import javax.annotation.Nonnull;
40+
import javax.annotation.Nullable;
4041

4142
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
4243

@@ -335,6 +336,11 @@ private static void extractOptionalData(final StreamInfo streamInfo,
335336
} catch (final Exception e) {
336337
streamInfo.addError(e);
337338
}
339+
try {
340+
streamInfo.setSongMetadata(extractor.getSongMetadata());
341+
} catch (final Exception e) {
342+
streamInfo.addError(e);
343+
}
338344

339345
streamInfo.setRelatedItems(ExtractorHelper.getRelatedItemsOrLogError(streamInfo,
340346
extractor));
@@ -388,6 +394,7 @@ private static void extractOptionalData(final StreamInfo streamInfo,
388394
private boolean shortFormContent = false;
389395
@Nonnull
390396
private ContentAvailability contentAvailability = ContentAvailability.AVAILABLE;
397+
private SongMetadata songMetadata = null;
391398

392399
/**
393400
* Preview frames, e.g. for the storyboard / seekbar thumbnail preview
@@ -743,4 +750,25 @@ public ContentAvailability getContentAvailability() {
743750
public void setContentAvailability(@Nonnull final ContentAvailability availability) {
744751
this.contentAvailability = availability;
745752
}
753+
754+
/**
755+
* Get the song metadata if this stream is a music stream
756+
* and the service provides this information.
757+
* <p>
758+
* A {@link SongMetadata} is only provided if the information on the song does not match
759+
* the {@link StreamInfo}'s title and uploader or if the service explicitly provides
760+
* this information. Otherwise, the {@link StreamInfo}'s title and uploader are assumed
761+
* to be the song's title and artist.
762+
* </p>
763+
*
764+
* @return the song metadata or {@code null}
765+
*/
766+
@Nullable
767+
public SongMetadata getSongMetadata() {
768+
return songMetadata;
769+
}
770+
771+
public void setSongMetadata(final SongMetadata songMetadata) {
772+
this.songMetadata = songMetadata;
773+
}
746774
}

0 commit comments

Comments
 (0)