added metadata, fix descriptions, fix thumbnail, update tests

thumbnail: quality before: https://peertube.cpy.re/static/thumbnails/d2a5ec78-5f85-4090-8ec5-dc1102e022ea.jpg
quality after: https://peertube.cpy.re/static/previews/d2a5ec78-5f85-4090-8ec5-dc1102e022ea.jpg

description: we were getting about the first 260 characters, we now get full description (with fallback to first 260 chars if the get request for full description fails)
test: updated tests to match description, also changed some test: it was assertEquals(extracted, expected), but the proper way to do it is assertEquals(expected, extracted)

metadata: got host, privacy (public, private, unlisted), licence, language, tags

Author: B0pol

README.md

@@ -21,7 +21,7 @@ To test changes quickly you can build the library locally. Using the local Maven
 2. It's _recommended_ that you change the `version` of this library (e.g. `LOCAL_SNAPSHOT`).
 3. Run gradle's `ìnstall` task to deploy this library to your local repository (using the wrapper, present in the root of this project: `./gradlew install`)
 4. Change the dependency version used in your project to match the one you chose in step 2 (`implementation 'com.github.TeamNewPipe:NewPipeExtractor:LOCAL_SNAPSHOT'`)
-     
+
 > Tip for Android Studio users: After you make changes and run the `install` task, use the menu option `File → "Sync with File System"` to refresh the library in your project.
 
 ## Supported sites

extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java

@@ -12,6 +12,7 @@
 import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
 import org.schabi.newpipe.extractor.localization.DateWrapper;
 import org.schabi.newpipe.extractor.stream.*;
+import org.schabi.newpipe.extractor.utils.JsonUtils;
 
 import javax.annotation.Nonnull;
 import java.io.IOException;
@@ -225,4 +226,35 @@ public String getName() throws ParsingException {
     public String getOriginalUrl() throws ParsingException {
         return data.getString("frontend_link");
     }
+
+    @Override
+    public String getHost() throws ParsingException {
+        return "";
+    }
+
+    @Override
+    public String getPrivacy() throws ParsingException {
+        return "";
+    }
+
+    @Override
+    public String getCategory() throws ParsingException {
+        return "";
+    }
+
+    @Override
+    public String getLicence() throws ParsingException {
+        return "";
+    }
+
+    @Override
+    public String getStreamInfoLanguage() throws ParsingException {
+        return "";
+    }
+
+    @Nonnull
+    @Override
+    public List<String> getTags() throws ParsingException {
+        return new ArrayList<>();
+    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java

@@ -9,7 +9,7 @@
 
 import org.jsoup.helper.StringUtil;
 import org.schabi.newpipe.extractor.MediaFormat;
-import org.schabi.newpipe.extractor.ServiceList;
+import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.downloader.Downloader;
 import org.schabi.newpipe.extractor.downloader.Response;
@@ -29,7 +29,6 @@
 import org.schabi.newpipe.extractor.stream.SubtitlesStream;
 import org.schabi.newpipe.extractor.stream.VideoStream;
 import org.schabi.newpipe.extractor.utils.JsonUtils;
-import org.schabi.newpipe.extractor.utils.Utils;
 
 import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonObject;
@@ -66,16 +65,25 @@ public DateWrapper getUploadDate() throws ParsingException {
 
     @Override
     public String getThumbnailUrl() throws ParsingException {
-        return baseUrl + JsonUtils.getString(json, "thumbnailPath");
+        return baseUrl + JsonUtils.getString(json, "previewPath");
     }
 
     @Override
     public String getDescription() throws ParsingException {
+        String description = "";
+        Downloader dl = NewPipe.getDownloader();
         try {
-            return JsonUtils.getString(json, "description");
-        }catch(ParsingException e) {
-            return "No description";
+            Response response = dl.get(getUrl() + "/description");
+            JsonObject jsonObject = JsonParser.object().from(response.responseBody());
+            description = JsonUtils.getString(jsonObject, "description");
+        } catch (ReCaptchaException | IOException | JsonParserException e) {
+            e.printStackTrace();
         }
+        if (description.equals("")) {
+            //if the request above failed
+            description = JsonUtils.getString(json, "description");
+        }
+        return description;
     }
 
     @Override
@@ -224,8 +232,9 @@ public StreamInfoItemsCollector getRelatedStreams() throws IOException, Extracti
         if(!StringUtil.isBlank(apiUrl)) getStreamsFromApi(collector, apiUrl);
         return collector;
     }
-    
-    private List<String> getTags(){
+
+    @Override
+    public List<String> getTags(){
         try {
             return (List) JsonUtils.getArray(json, "tags");
         } catch (Exception e) {
@@ -339,4 +348,29 @@ public String getOriginalUrl() throws ParsingException {
         return baseUrl + "/videos/watch/" + getId();
     }
 
+    //TODO: change privacy, category, licence by getting ID, therefore we will be able to translate it
+    @Override
+    public String getHost() throws ParsingException {
+        return JsonUtils.getString(json, "account.host");
+    }
+
+    @Override
+    public String getPrivacy() throws ParsingException {
+        return JsonUtils.getString(json, "privacy.label");
+    }
+
+    @Override
+    public String getCategory() throws ParsingException {
+        return JsonUtils.getString(json, "category.label");
+    }
+
+    @Override
+    public String getLicence() throws ParsingException {
+        return JsonUtils.getString(json, "licence.label");
+    }
+
+    @Override
+    public String getStreamInfoLanguage() throws ParsingException {
+        return JsonUtils.getString(json, "language.label");
+    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java

@@ -254,4 +254,35 @@ public StreamInfoItemsCollector getRelatedStreams() throws IOException, Extracti
     public String getErrorMessage() {
         return null;
     }
+
+    @Override
+    public String getHost() throws ParsingException {
+        return "";
+    }
+
+    @Override
+    public String getPrivacy() throws ParsingException {
+        return "";
+    }
+
+    @Override
+    public String getCategory() throws ParsingException {
+        return "";
+    }
+
+    @Override
+    public String getLicence() throws ParsingException {
+        return "";
+    }
+
+    @Override
+    public String getStreamInfoLanguage() throws ParsingException {
+        return "";
+    }
+
+    @Nonnull
+    @Override
+    public List<String> getTags() throws ParsingException {
+        return new ArrayList<>();
+    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java

@@ -1113,4 +1113,35 @@ public List<Frameset> getFrames() throws ExtractionException {
 			throw new ExtractionException(e);
 		}
 	}
+
+    @Override
+    public String getHost() throws ParsingException {
+        return "";
+    }
+
+    @Override
+    public String getPrivacy() throws ParsingException {
+        return "";
+    }
+
+    @Override
+    public String getCategory() throws ParsingException {
+        return "";
+    }
+
+    @Override
+    public String getLicence() throws ParsingException {
+        return "";
+    }
+
+    @Override
+    public String getStreamInfoLanguage() throws ParsingException {
+        return "";
+    }
+
+    @Nonnull
+    @Override
+    public List<String> getTags() throws ParsingException {
+        return new ArrayList<>();
+    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java

@@ -27,6 +27,7 @@
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
 import org.schabi.newpipe.extractor.localization.DateWrapper;
+import org.schabi.newpipe.extractor.utils.JsonUtils;
 import org.schabi.newpipe.extractor.utils.Parser;
 
 import javax.annotation.Nonnull;
@@ -349,4 +350,60 @@ protected long getTimestampSeconds(String regexPattern) throws ParsingException
             return 0;
         }
     }
+
+    /**
+     * The host of the stream (Eg. peertube.cpy.re).
+     * If the privacy is not available, or if the service doesn't use
+     * a federated system, but a centralised system,
+     * you can simply return an empty string.
+     * @return the host of the stream or an empty String.
+     * @throws ParsingException
+     */
+    @Nonnull
+    public abstract String getHost() throws ParsingException;
+
+    /**
+     * The privacy of the stream (Eg. Public, Private, Unlisted…).
+     * If the privacy is not available you can simply return an empty string.
+     * @return the privacy of the stream or an empty String.
+     * @throws ParsingException
+     */
+    @Nonnull
+    public abstract String getPrivacy() throws ParsingException;
+
+    /**
+     * The name of the category of the stream.
+     * If the category is not available you can simply return an empty string.
+     * @return the category of the stream or an empty String.
+     * @throws ParsingException
+     */
+    @Nonnull
+    public abstract String getCategory() throws ParsingException;
+
+    /**
+     * The name of the licence of the stream.
+     * If the licence is not available you can simply return an empty string.
+     * @return the licence of the stream or an empty String.
+     * @throws ParsingException
+     */
+    @Nonnull
+    public abstract String getLicence() throws ParsingException;
+
+    /**
+     * The language of the stream.
+     * If the language is not available you can simply return an empty string.
+     * @return the licence of the stream or an empty String.
+     * @throws ParsingException
+     */
+    @Nonnull
+    public abstract String getStreamInfoLanguage() throws ParsingException;
+
+    /**
+     * The list of tags of the stream.
+     * If the tag list is not available you can simply return an empty list.
+     * @return the list of tags of the stream or an empty list.
+     * @throws ParsingException
+     */
+    @Nonnull
+    public abstract List<String> getTags() throws ParsingException;
 }

extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java

@@ -270,6 +270,38 @@ private static StreamInfo extractOptionalData(StreamInfo streamInfo, StreamExtra
             streamInfo.addError(e);
         }
 
+        //additional info
+        try {
+            streamInfo.setHost(extractor.getHost());
+        } catch (Exception e) {
+            streamInfo.addError(e);
+        }
+        try {
+            streamInfo.setPrivacy(extractor.getPrivacy());
+        } catch (Exception e) {
+            streamInfo.addError(e);
+        }
+        try {
+            streamInfo.setCategory(extractor.getCategory());
+        } catch (Exception e) {
+            streamInfo.addError(e);
+        }
+        try {
+            streamInfo.setLicence(extractor.getLicence());
+        } catch (Exception e) {
+            streamInfo.addError(e);
+        }
+        try {
+            streamInfo.setLanguage(extractor.getStreamInfoLanguage());
+        } catch (Exception e) {
+            streamInfo.addError(e);
+        }
+        try {
+            streamInfo.setTags(extractor.getTags());
+        } catch (Exception e) {
+            streamInfo.addError(e);
+        }
+
         streamInfo.setRelatedStreams(ExtractorHelper.getRelatedVideosOrLogError(streamInfo, extractor));
 
         return streamInfo;
@@ -308,6 +340,13 @@ private static StreamInfo extractOptionalData(StreamInfo streamInfo, StreamExtra
     private long startPosition = 0;
     private List<SubtitlesStream> subtitles = new ArrayList<>();
 
+    private String host = "";
+    private String privacy = "";
+    private String category = "";
+    private String licence = "";
+    private String language = "";
+    private List<String> tags = new ArrayList<>();
+
     /**
      * Get the stream type
      *
@@ -533,4 +572,53 @@ public void setSubtitles(List<SubtitlesStream> subtitles) {
         this.subtitles = subtitles;
     }
 
+    public String getHost() {
+        return this.host;
+    }
+
+    public void setHost(String str) {
+        this.host = str;
+    }
+
+    public String getPrivacy() {
+        return this.privacy;
+    }
+
+    public void setPrivacy(String str) {
+        this.privacy = str;
+    }
+
+    public String getCategory() {
+        return this.category;
+    }
+
+    public void setCategory(String cat) {
+        this.category = cat;
+    }
+
+    public String getLicence() {
+        return this.licence;
+    }
+
+    public void setLicence(String str) {
+        this.licence = str;
+    }
+
+    public String getLanguage() {
+        return this.language;
+    }
+
+    public void setLanguage(String lang) {
+        this.language = lang;
+    }
+
+    public List<String> getTags() {
+        return this.tags;
+    }
+
+    public void setTags(List<String> tags) {
+        this.tags = tags;
+    }
+
+
 }

extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java

@@ -185,5 +185,4 @@ public static String getBaseUrl(String url) throws ParsingException {
         }
         return uri.getProtocol() + "://" + uri.getAuthority();
     }
-    
 }