[PeerTube] Add support for stream frames/storyboards extraction

Implement PeerTubeStreamExtractor.getFrames()

Author: TobiGr

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java

@@ -26,9 +26,11 @@
 import org.schabi.newpipe.extractor.stream.AudioStream;
 import org.schabi.newpipe.extractor.stream.DeliveryMethod;
 import org.schabi.newpipe.extractor.stream.Description;
+import org.schabi.newpipe.extractor.stream.Frameset;
 import org.schabi.newpipe.extractor.stream.Stream;
 import org.schabi.newpipe.extractor.stream.StreamExtractor;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
+import org.schabi.newpipe.extractor.stream.StreamSegment;
 import org.schabi.newpipe.extractor.stream.StreamType;
 import org.schabi.newpipe.extractor.stream.SubtitlesStream;
 import org.schabi.newpipe.extractor.stream.VideoStream;
@@ -316,6 +318,66 @@ public String getSupportInfo() {
         }
     }
 
+    @Nonnull
+    @Override
+    public List<StreamSegment> getStreamSegments() throws ParsingException {
+        final List<StreamSegment> segments = new ArrayList<>();
+        final JsonObject segmentsJson;
+        try {
+            segmentsJson = fetchSubApiContent("chapters");
+        } catch (final IOException | ReCaptchaException e) {
+            throw new ParsingException("Could not get stream segments", e);
+        }
+        if (segmentsJson != null && segmentsJson.has("chapters")) {
+            final JsonArray segmentsArray = segmentsJson.getArray("chapters");
+            for (int i = 0; i < segmentsArray.size(); i++) {
+                final JsonObject segmentObject = segmentsArray.getObject(i);
+                segments.add(new StreamSegment(
+                        segmentObject.getString("title"),
+                        segmentObject.getInt("timecode")));
+            }
+        }
+
+        return segments;
+    }
+
+    @Nonnull
+    @Override
+    public List<Frameset> getFrames() throws ExtractionException {
+        final List<Frameset> framesets = new ArrayList<>();
+        final JsonObject storyboards;
+        try {
+            storyboards = fetchSubApiContent("storyboards");
+        } catch (final IOException | ReCaptchaException e) {
+            throw new ExtractionException("Could not get frames", e);
+        }
+        if (storyboards != null && storyboards.has("storyboards")) {
+            final JsonArray storyboardsArray = storyboards.getArray("storyboards");
+            for (final Object storyboard : storyboardsArray) {
+                if (storyboard instanceof JsonObject) {
+                    final JsonObject storyboardObject = (JsonObject) storyboard;
+                    final String url = storyboardObject.getString("storyboardPath");
+                    final int width = storyboardObject.getInt("spriteWidth");
+                    final int height = storyboardObject.getInt("spriteHeight");
+                    final int totalWidth = storyboardObject.getInt("totalWidth");
+                    final int totalHeight = storyboardObject.getInt("totalHeight");
+                    final int framesPerPageX = totalWidth / width;
+                    final int framesPerPageY = totalHeight / height;
+                    final int count = framesPerPageX * framesPerPageY;
+                    final int durationPerFrame = storyboardObject.getInt("spriteDuration") * 1000;
+
+                    framesets.add(new Frameset(
+                            // there is only one composite image per video containing all frames
+                            List.of(baseUrl + url),
+                            width, height, count,
+                            durationPerFrame, framesPerPageX, framesPerPageY));
+                }
+            }
+        }
+
+        return framesets;
+    }
+
     @Nonnull
     private String getRelatedItemsUrl(@Nonnull final List<String> tags)
             throws UnsupportedEncodingException {
@@ -636,6 +698,41 @@ private void addNewVideoStream(@Nonnull final JsonObject streamJsonObject,
         }
     }
 
+    /**
+     * Fetch content from a sub-API of the video.
+     * @param subPath the API subpath after the video id,
+     *                e.g. "storyboards" for "/api/v1/videos/{id}/storyboards"
+     * @return the {@link JsonObject} of the sub-API or null if the API does not exist
+     * which is the case if the instance has an outdated PeerTube version.
+     * @throws ParsingException if the API response could not be parsed to a {@link JsonObject}
+     * @throws IOException if the API response could not be fetched
+     * @throws ReCaptchaException if the API response is a reCaptcha
+     */
+    @Nullable
+    private JsonObject fetchSubApiContent(@Nonnull final String subPath)
+            throws ParsingException, IOException, ReCaptchaException {
+        final String apiUrl = baseUrl + PeertubeStreamLinkHandlerFactory.VIDEO_API_ENDPOINT
+                + getId() + "/" + subPath;
+        final Response response = getDownloader().get(apiUrl);
+        if (response == null) {
+            throw new ParsingException("Could not get segments from API.");
+        }
+        if (response.responseCode() == 400) {
+            // Chapter or segments support was added with PeerTube v6.0.0
+            // This instance does not support it yet.
+            return null;
+        }
+        if (response.responseCode() != 200) {
+            throw new ParsingException("Could not get segments from API. Response code: "
+                    + response.responseCode());
+        }
+        try {
+            return JsonParser.object().from(response.responseBody());
+        } catch (final JsonParserException e) {
+            throw new ParsingException("Could not parse json data for segments", e);
+        }
+    }
+
     @Nonnull
     @Override
     public String getName() throws ParsingException {

extractor/src/main/java/org/schabi/newpipe/extractor/stream/Frameset.java

@@ -3,6 +3,9 @@
 import java.io.Serializable;
 import java.util.List;
 
+/**
+ * Class to handle framesets / storyboards which summarize the stream content.
+ */
 public final class Frameset implements Serializable {
 
     private final List<String> urls;
@@ -13,6 +16,17 @@ public final class Frameset implements Serializable {
     private final int framesPerPageX;
     private final int framesPerPageY;
 
+    /**
+     * Creates a new Frameset or set of storyboards.
+     * @param urls the URLs to the images with frames / storyboards
+     * @param frameWidth the width of a single frame, in pixels
+     * @param frameHeight the height of a single frame, in pixels
+     * @param totalCount the total count of frames
+     * @param durationPerFrame the duration per frame in milliseconds
+     * @param framesPerPageX the maximum count of frames per page by x / over the width of the image
+     * @param framesPerPageY the maximum count of frames per page by y / over the height
+     *                       of the image
+     */
     public Frameset(
             final List<String> urls,
             final int frameWidth,
@@ -32,7 +46,7 @@ public Frameset(
     }
 
     /**
-     * @return list of urls to images with frames
+     * @return list of URLs to images with frames
      */
     public List<String> getUrls() {
         return urls;