Merge branch 'master' of github.com:TeamNewPipe/NewPipeExtractor

Author: theScrabi

.travis.yml

@@ -1,7 +1,7 @@
 language: java
 
 script: ./gradlew check
-after_success: ./gradlew javadoc
+after_success: ./gradlew aggregatedJavadocs
 
 deploy:
   provider: pages

build.gradle

@@ -29,3 +29,19 @@ subprojects {
     }
 }
 
+// https://discuss.gradle.org/t/best-approach-gradle-multi-module-project-generate-just-one-global-javadoc/18657/21
+task aggregatedJavadocs(type: Javadoc, group: 'Documentation') {
+    destinationDir = file("$buildDir/docs/javadoc")
+    title = "$project.name $version"
+    // options.memberLevel = JavadocMemberLevel.PRIVATE
+    options.links 'https://docs.oracle.com/javase/7/docs/api/'
+    
+    subprojects.each { project ->
+        project.tasks.withType(Javadoc).each { javadocTask ->
+            source += javadocTask.source
+            classpath += javadocTask.classpath
+            excludes += javadocTask.excludes
+            includes += javadocTask.includes
+        }
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java

@@ -9,7 +9,7 @@
 
 public abstract class Extractor {
     /**
-     * {@link StreamingService} currently related to this extractor.<br/>
+     * {@link StreamingService} currently related to this extractor.<br>
      * Useful for getting other things from a service (like the url handlers for cleaning/accepting/get id from urls).
      */
     private final StreamingService service;
@@ -20,7 +20,7 @@ public abstract class Extractor {
      * What makes a url "dirty" or not is, for example, the additional parameters
      * (not important as—in this case—the id):
      * <pre>
-     *     https://www.youtube.com/watch?v=a9Zf_258aTI<i>&t=4s</i>  →  <i><b>&t=4s</b></i>
+     *     https://www.youtube.com/watch?v=a9Zf_258aTI<i>&amp;t=4s</i>  →  <i><b>&amp;t=4s</b></i>
      * </pre>
      * But as you can imagine, the time parameter is very important when calling {@link org.schabi.newpipe.extractor.stream.StreamExtractor#getTimeStamp()}.
      */

extractor/src/main/java/org/schabi/newpipe/extractor/ListExtractor.java

@@ -26,7 +26,7 @@ public ListExtractor(StreamingService service, String url) {
     public abstract InfoItemsPage<R> getInitialPage() throws IOException, ExtractionException;
 
     /**
-     * Returns an url that can be used to get the next page relative to the initial one.<br/>
+     * Returns an url that can be used to get the next page relative to the initial one.
      * <p>Usually, these links will only work in the implementation itself.</p>
      *
      * @return an url pointing to the next page relative to the initial page

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamUrlIdHandler.java

@@ -119,7 +119,7 @@ public String getId(String url) throws ParsingException, IllegalArgumentExceptio
      * Shared URI's look like this:
      * <pre>
      *     * https://www.youtube.com/shared?ci=PJICrTByb3E
-     *     * vnd.youtube://www.youtube.com/shared?ci=PJICrTByb3E&feature=twitter-deep-link
+     *     * vnd.youtube://www.youtube.com/shared?ci=PJICrTByb3E&amp;feature=twitter-deep-link
      * </pre>
      *
      * @param url The shared url

timeago-parser/raw/README.md

@@ -0,0 +1,13 @@
+# Time ago parser raw resources
+
+This directory contains some JSON-formatted time ago strings gather directly from YouTube.
+
+#### Java directory
+
+Some useful classes that can generate an overview and check if we have all the time units for all the languages.
+
+It also contains the resource bundle generator.
+
+#### Times directory
+
+All the units organized by their unit value and name (e.g. 1s = "1 second", 2y = "2 años", 4w = "4 semanas").

timeago-parser/raw/java/CheckAll.java

@@ -0,0 +1,75 @@
+import com.grack.nanojson.JsonObject;
+import com.grack.nanojson.JsonParser;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.util.Arrays;
+import java.util.Map;
+
+public class CheckAll {
+
+    public static void main(String[] args) throws Exception {
+        int SECONDS = 59, currentSeconds = 0;
+        int MINUTES = 59, currentMinutes = 0;
+        int HOURS = 23, currentHours = 0;
+        int DAYS = 6, currentDays = 0;
+        int WEEKS = 4, currentWeeks = 0;
+        int MONTHS = 11, currentMonths = 0;
+        int YEARS = 12, currentYears = 0;
+
+        for (String name : Arrays.asList("seconds", "minutes", "hours", "days", "weeks", "months", "years")) {
+            JsonObject object = JsonParser.object().from(new FileInputStream(new File("timeago-parser/raw/times/" + name + ".json")));
+
+            for (Map.Entry<String, Object> entry : object.entrySet()) {
+                JsonObject value = (JsonObject) entry.getValue();
+
+                final int size = value.keySet().size();
+                if (size >= 80) {
+                    if (name.equals("seconds")) currentSeconds++;
+                    if (name.equals("minutes")) currentMinutes++;
+                    if (name.equals("hours")) currentHours++;
+                    if (name.equals("days")) currentDays++;
+                    if (name.equals("weeks")) currentWeeks++;
+                    if (name.equals("months")) currentMonths++;
+                    if (name.equals("years")) currentYears++;
+                } else {
+                    System.err.println("Missing some units in: " + name + " → " + entry.getKey() + " (current size = " + size + ")");
+                }
+
+                String number = entry.getKey().replaceAll("\\D", "");
+                for (Map.Entry<String, Object> langsKeys : value.entrySet()) {
+                    String lang = langsKeys.getKey();
+                    String langValue = String.valueOf(langsKeys.getValue());
+
+                    String langValueNumber = langValue.replaceAll("\\D", "");
+                    if (!langValueNumber.equals(number)) {
+                        final String msg = langValueNumber.isEmpty() ? "doesn't contain number" : "different number";
+                        System.out.printf("%-20s[!]   %22s: %10s   = %s \n", entry.getKey(), msg, lang, langValue);
+                    }
+                }
+            }
+        }
+        System.out.println("\n\nHow many:\n");
+
+        if (currentSeconds == SECONDS) System.out.println("seconds: " + currentSeconds);
+        else System.out.println("[!] missing seconds: " + currentSeconds);
+
+        if (currentMinutes == MINUTES) System.out.println("minutes: " + currentMinutes);
+        else System.out.println("[!] missing minutes: " + currentMinutes);
+
+        if (currentHours == HOURS) System.out.println("hours: " + currentHours);
+        else System.out.println("[!] missing hours: " + currentHours);
+
+        if (currentDays == DAYS) System.out.println("days: " + currentDays);
+        else System.out.println("[!] missing days: " + currentDays);
+
+        if (currentWeeks == WEEKS) System.out.println("weeks: " + currentWeeks);
+        else System.out.println("[!] missing weeks: " + currentWeeks);
+
+        if (currentMonths == MONTHS) System.out.println("months: " + currentMonths);
+        else System.out.println("[!] missing months: " + currentMonths);
+
+        if (currentYears == YEARS) System.out.println("years: " + currentYears);
+        else System.out.println("[!] missing years: " + currentYears);
+    }
+}

timeago-parser/raw/java/GenerateOverview.java

@@ -0,0 +1,70 @@
+import com.grack.nanojson.JsonAppendableWriter;
+import com.grack.nanojson.JsonObject;
+import com.grack.nanojson.JsonParser;
+import com.grack.nanojson.JsonWriter;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.util.*;
+
+public class GenerateOverview {
+
+    public static void main(String[] args) throws Exception {
+        Map<String, Map<String, Collection<String>>> outMap = new TreeMap<>(new Comparator<String>() {
+            @Override
+            public int compare(String o1, String o2) {
+                return o1.compareToIgnoreCase(o2);
+            }
+        });
+        for (String unitName : Arrays.asList("seconds", "minutes", "hours", "days", "weeks", "months", "years")) {
+            JsonObject object = JsonParser.object().from(new FileInputStream(new File("timeago-parser/raw/times/" + unitName + ".json")));
+
+            for (Map.Entry<String, Object> timeKeyValue : object.entrySet()) {
+                JsonObject timeObject = (JsonObject) timeKeyValue.getValue();
+                for (Map.Entry<String, Object> langsKeyValue : timeObject.entrySet()) {
+                    String langKey = langsKeyValue.getKey();
+                    String langValue = langsKeyValue.getValue().toString();
+
+                    Map<String, Collection<String>> langUnitsMap;
+                    if (outMap.containsKey(langKey)) {
+                        langUnitsMap = outMap.get(langKey);
+                    } else {
+                        langUnitsMap = new TreeMap<>(Utils.compareByUnitName());
+                        outMap.put(langKey, langUnitsMap);
+                    }
+
+                    Collection<String> langUnitListValues;
+                    if (langUnitsMap.containsKey(unitName)) {
+                        langUnitListValues = langUnitsMap.get(unitName);
+                    } else {
+                        langUnitListValues = new TreeSet<>(Utils.compareByNumber());
+                        langUnitsMap.put(unitName, langUnitListValues);
+                    }
+
+
+                    langUnitListValues.add(langValue);
+                }
+            }
+        }
+
+        writeMapTo(outMap, JsonWriter.indent("  ").on(new FileOutputStream(new File("timeago-parser/raw/overview.json"))));
+    }
+
+    public static void writeMapTo(Map<String, Map<String, Collection<String>>> outMap, JsonAppendableWriter out) {
+        out.object();
+        for (Map.Entry<String, Map<String, Collection<String>>> langMapEntry : outMap.entrySet()) {
+            final String langName = langMapEntry.getKey();
+            out.object(langName);
+            for (Map.Entry<String, Collection<String>> langValuesEntry : langMapEntry.getValue().entrySet()) {
+                final String unitName = langValuesEntry.getKey();
+                out.array(unitName);
+                for (String timeValue : langValuesEntry.getValue()) out.value(timeValue);
+                out.end();
+            }
+            out.end();
+        }
+        out.end().done();
+    }
+
+}

timeago-parser/raw/java/GenerateResourceBundles.java

@@ -0,0 +1,59 @@
+import com.grack.nanojson.JsonArray;
+import com.grack.nanojson.JsonObject;
+import com.grack.nanojson.JsonParser;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.OutputStream;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
+import static org.schabi.newpipe.extractor.timeago.TimeAgoPatternsManager.RESOURCE_BUNDLE_ARRAY_SEPARATOR;
+
+public class GenerateResourceBundles {
+
+    public static void main(String[] args) throws Exception {
+        File outDir = new File("timeago-parser/outBundle");
+        if (!outDir.isDirectory()) outDir.mkdir();
+
+        JsonObject object = JsonParser.object().from(new FileInputStream(new File("timeago-parser/raw/unique_patterns.json")));
+
+        for (Map.Entry<String, Object> langTimeEntry : new TreeMap<>(object).entrySet()) {
+            final String langName = langTimeEntry.getKey();
+            StringBuilder outString = new StringBuilder();
+
+            final TreeMap<String, Object> sortedMap = new TreeMap<>(Utils.compareByUnitName());
+            sortedMap.putAll((JsonObject) langTimeEntry.getValue());
+
+            final Iterator<Map.Entry<String, Object>> unitEntriesIterator = sortedMap.entrySet().iterator();
+            while (unitEntriesIterator.hasNext()) {
+                final Map.Entry<String, Object> unitEntry = unitEntriesIterator.next();
+                final String unitName = unitEntry.getKey();
+                final List<Object> unitList = (JsonArray) unitEntry.getValue();
+
+                outString.append(unitName).append("=\\\n");
+
+                for (int i = 0; i < unitList.size(); i++) {
+                    final String s = unitList.get(i).toString();
+                    outString.append("  ").append(s);
+
+                    if (i < unitList.size() - 1) {
+                        outString.append(RESOURCE_BUNDLE_ARRAY_SEPARATOR).append("\\").append("\n");
+                    }
+                }
+
+                if (unitEntriesIterator.hasNext()) outString.append("\n\n");
+            }
+
+            String fileName = "time_units_" + langName.replaceAll("-", "_") + ".properties";
+            System.out.println("Writing " + fileName + "...");
+            try (OutputStream out = new FileOutputStream(new File(outDir, fileName))) {
+                out.write(outString.toString().getBytes("UTF-8"));
+            }
+        }
+
+    }
+}

timeago-parser/raw/java/Utils.java

@@ -0,0 +1,29 @@
+import java.util.*;
+
+public class Utils {
+
+    static Comparator<String> compareByNumber() {
+        return new Comparator<String>() {
+            @Override
+            public int compare(String o1, String o2) {
+                return extractInt(o1) - extractInt(o2);
+            }
+
+            private int extractInt(String s) {
+                String num = s.replaceAll("\\D", "");
+                return num.isEmpty() ? 0 : Integer.parseInt(num);
+            }
+        };
+    }
+
+    static Comparator<Object> compareByUnitName() {
+        return new Comparator<Object>() {
+            private final List<String> ORDER = Arrays.asList("seconds", "minutes", "hours", "days", "weeks", "months", "years");
+
+            @Override
+            public int compare(Object o1, Object o2) {
+                return Integer.compare(ORDER.indexOf(o1.toString()), ORDER.indexOf(o2.toString()));
+            }
+        };
+    }
+}