Refactor frames extraction

Author: Koitharu

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java

@@ -1036,28 +1036,60 @@ public String getThumbnailUrl() throws ParsingException {
         };
     }
 
-    @Nullable
-    public StreamFrames getFrames() {
-        try {
-            final String script = doc.select("#player-api").first().siblingElements().select("script").html();
-            int p = script.indexOf("ytplayer.config");
-            if (p == -1) {
-                return null;
-            }
-            p = script.indexOf('{', p);
-            int e = script.indexOf("ytplayer.load", p);
-            if (e == -1) {
-                return null;
-            }
-            JsonObject jo = JsonParser.object().from(script.substring(p, e - 1));
-            final String resp = jo.getObject("args").getString("player_response");
-            jo = JsonParser.object().from(resp);
-            final String[] spec = jo.getObject("storyboards").getObject("playerStoryboardSpecRenderer").getString("spec").split("\\|");
-            final String url = spec[0];
-            final List<String> opts = Arrays.asList(spec).subList(1, spec.length);
-            return new StreamFrames(url, opts);
-        } catch (Exception e) {
-            return null;
-        }
-    }
+	@Nonnull
+	@Override
+	public List<Frameset> getFrames() throws ExtractionException {
+		try {
+			final String script = doc.select("#player-api").first().siblingElements().select("script").html();
+			int p = script.indexOf("ytplayer.config");
+			if (p == -1) {
+				return Collections.emptyList();
+			}
+			p = script.indexOf('{', p);
+			int e = script.indexOf("ytplayer.load", p);
+			if (e == -1) {
+				return Collections.emptyList();
+			}
+			JsonObject jo = JsonParser.object().from(script.substring(p, e - 1));
+			final String resp = jo.getObject("args").getString("player_response");
+			jo = JsonParser.object().from(resp);
+			final String[] spec = jo.getObject("storyboards").getObject("playerStoryboardSpecRenderer").getString("spec").split("\\|");
+			final String url = spec[0];
+			final ArrayList<Frameset> result = new ArrayList<>(spec.length - 1);
+			for (int i = 1; i < spec.length; ++i) {
+				final String[] parts = spec[i].split("#");
+				if (parts.length != 8) {
+					continue;
+				}
+				final int frameWidth = Integer.parseInt(parts[0]);
+				final int frameHeight = Integer.parseInt(parts[1]);
+				final int totalCount = Integer.parseInt(parts[2]);
+				final int framesPerPageX = Integer.parseInt(parts[3]);
+				final int framesPerPageY = Integer.parseInt(parts[4]);
+				final String baseUrl = url.replace("$L", String.valueOf(i - 1)).replace("$N", parts[6]) + "&sigh=" + parts[7];
+				final List<String> urls;
+				if (baseUrl.contains("$M")) {
+					final int totalPages = (int) Math.ceil(totalCount / (double) (framesPerPageX * framesPerPageY));
+					urls = new ArrayList<>(totalPages);
+					for (int j = 0; j < totalPages; j++) {
+						urls.add(baseUrl.replace("$M", String.valueOf(j)));
+					}
+				} else {
+					urls = Collections.singletonList(baseUrl);
+				}
+				result.add(new Frameset(
+						urls,
+						frameWidth,
+						frameHeight,
+						totalCount,
+						framesPerPageX,
+						framesPerPageY
+				));
+			}
+			result.trimToSize();
+			return result;
+		} catch (Exception e) {
+			throw new ExtractionException(e);
+		}
+	}
 }

extractor/src/main/java/org/schabi/newpipe/extractor/stream/Frameset.java

@@ -0,0 +1,63 @@
+package org.schabi.newpipe.extractor.stream;
+
+import javax.annotation.Nullable;
+import java.util.Collection;
+import java.util.List;
+
+public final class Frameset {
+
+	private List<String> urls;
+	private int frameWidth;
+	private int frameHeight;
+	private int totalCount;
+	private int framesPerPageX;
+	private int framesPerPageY;
+
+	public Frameset(List<String> urls, int frameWidth, int frameHeight, int totalCount, int framesPerPageX, int framesPerPageY) {
+		this.urls = urls;
+		this.totalCount = totalCount;
+		this.frameWidth = frameWidth;
+		this.frameHeight = frameHeight;
+		this.framesPerPageX = framesPerPageX;
+		this.framesPerPageY = framesPerPageY;
+	}
+
+	public List<String> getUrls() {
+		return urls;
+	}
+
+	/**
+	 * @return total count of frames
+	 */
+	public int getTotalCount() {
+		return totalCount;
+	}
+
+	/**
+	 * @return maximum frames count by x
+	 */
+	public int getFramesPerPageX() {
+		return framesPerPageX;
+	}
+
+	/**
+	 * @return maximum frames count by y
+	 */
+	public int getFramesPerPageY() {
+		return framesPerPageY;
+	}
+
+	/**
+	 * @return width of a one frame, in pixels
+	 */
+	public int getFrameWidth() {
+		return frameWidth;
+	}
+
+	/**
+	 * @return height of a one frame, in pixels
+	 */
+	public int getFrameHeight() {
+		return frameHeight;
+	}
+}

extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java

@@ -30,7 +30,10 @@
 import org.schabi.newpipe.extractor.utils.Parser;
 
 import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 
 /**
@@ -255,6 +258,15 @@ public StreamExtractor(StreamingService service, LinkHandler linkHandler, Locali
      */
     public abstract StreamInfoItemsCollector getRelatedStreams() throws IOException, ExtractionException;
 
+    /**
+     * Should return a list of frames
+     * @return
+     */
+    @Nonnull
+    public List<Frameset> getFrames() throws IOException, ExtractionException {
+        return Collections.emptyList();
+    }
+
     /**
      * Should analyse the webpage's document and extracts any error message there might be. (e.g. GEMA block)
      *

extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamFrames.java

@@ -3,6 +3,7 @@
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.schabi.newpipe.Downloader;
+import org.schabi.newpipe.extractor.ExtractorAsserts;
 import org.schabi.newpipe.extractor.MediaFormat;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
@@ -13,6 +14,7 @@
 import org.schabi.newpipe.extractor.utils.Utils;
 
 import java.io.IOException;
+import java.util.List;
 
 import static org.junit.Assert.*;
 import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
@@ -244,15 +246,14 @@ public static void setUp() throws Exception {
         }
 
         @Test
-        public void testGetFrames() {
-            final StreamFrames frames = extractor.getFrames();
+        public void testGetFrames() throws ExtractionException {
+            final List<Frameset> frames = extractor.getFrames();
             assertNotNull(frames);
-            assertNotNull(frames.getDefaultVariant());
-            for (int i=0;i<frames.getVariantsCount();i++) {
-                final StreamFrames.Frameset frameset = frames.getVariant(i);
-                final int pages = frameset.getTotalPages();
-                final String url = pages == 0 ? frameset.getUrl() : frameset.getUrl(pages - 1);
-                assertNotNull(url);
+            assertFalse(frames.isEmpty());
+            for (final Frameset f : frames) {
+                for (final String url : f.getUrls()) {
+                    ExtractorAsserts.assertIsValidUrl(url);
+                }
             }
         }
     }