[YouTube] Switch to new search suggestion domain and improve error handling

- Switch to the new domain used by YouTube for search suggestions,
suggestqueries-clients6.youtube.com, and add the xhr query parameter with the
t value, to allow getting responses without requiring trim;
- Use the Java 8 Stream API to collect search suggestions and improve invalid
response detection by checking whether the content type of the response
returned is JSON;
- Move the licence header at the top of the file.

Author: AudricV

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java

@@ -1,42 +1,46 @@
+/*
+ * Created by Christian Schabesberger on 28.09.16.
+ *
+ * Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
+ * YoutubeSuggestionExtractor.java is part of NewPipe Extractor.
+ *
+ * NewPipe Extractor is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe Extractor is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe Extractor.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package org.schabi.newpipe.extractor.services.youtube.extractors;
 
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getCookieHeader;
+import static org.schabi.newpipe.extractor.utils.Utils.isBlank;
+import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
 
 import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonParser;
 import com.grack.nanojson.JsonParserException;
 
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.StreamingService;
-import org.schabi.newpipe.extractor.downloader.Downloader;
+import org.schabi.newpipe.extractor.downloader.Response;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
 import org.schabi.newpipe.extractor.utils.Utils;
 
 import java.io.IOException;
-import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
-
-/*
- * Created by Christian Schabesberger on 28.09.16.
- *
- * Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
- * YoutubeSuggestionExtractor.java is part of NewPipe.
- *
- * NewPipe is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * NewPipe is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
- */
+import java.util.Map;
+import java.util.stream.Collectors;
 
 public class YoutubeSuggestionExtractor extends SuggestionExtractor {
 
@@ -46,35 +50,45 @@ public YoutubeSuggestionExtractor(final StreamingService service) {
 
     @Override
     public List<String> suggestionList(final String query) throws IOException, ExtractionException {
-        final Downloader dl = NewPipe.getDownloader();
-        final List<String> suggestions = new ArrayList<>();
-
-        final String url = "https://suggestqueries.google.com/complete/search"
-                + "?client=" + "youtube" //"firefox" for JSON, 'toolbar' for xml
-                + "&jsonp=" + "JP"
+        final String url = "https://suggestqueries-clients6.youtube.com/complete/search"
+                + "?client=" + "youtube"
                 + "&ds=" + "yt"
                 + "&gl=" + Utils.encodeUrlUtf8(getExtractorContentCountry().getCountryCode())
-                + "&q=" + Utils.encodeUrlUtf8(query);
+                + "&q=" + Utils.encodeUrlUtf8(query)
+                + "&xhr=t";
 
-        String response = dl.get(url, getCookieHeader(), getExtractorLocalization()).responseBody();
-        // trim JSONP part "JP(...)"
-        response = response.substring(3, response.length() - 1);
-        try {
-            final JsonArray collection = JsonParser.array().from(response).getArray(1);
-            for (final Object suggestion : collection) {
-                if (!(suggestion instanceof JsonArray)) {
-                    continue;
-                }
-                final String suggestionStr = ((JsonArray) suggestion).getString(0);
-                if (suggestionStr == null) {
-                    continue;
-                }
-                suggestions.add(suggestionStr);
-            }
+        final Map<String, List<String>> headers = new HashMap<>();
+        headers.put("Origin", Collections.singletonList("https://www.youtube.com"));
+        headers.put("Referer", Collections.singletonList("https://www.youtube.com"));
+
+        final Response response = NewPipe.getDownloader()
+                .get(url, headers, getExtractorLocalization());
+
+        final String contentTypeHeader = response.getHeader("Content-Type");
+        if (isNullOrEmpty(contentTypeHeader) || !contentTypeHeader.contains("application/json")) {
+            throw new ExtractionException("Invalid response type (got \"" + contentTypeHeader
+                    + "\", excepted a JSON response) (response code "
+                    + response.responseCode() + ")");
+        }
+
+        final String responseBody = response.responseBody();
 
-            return suggestions;
+        if (responseBody.isEmpty()) {
+            throw new ExtractionException("Empty response received");
+        }
+
+        try {
+            final JsonArray suggestions = JsonParser.array()
+                    .from(responseBody)
+                    .getArray(1); // 0: search query, 1: search suggestions, 2: tracking data?
+            return suggestions.stream()
+                    .filter(JsonArray.class::isInstance)
+                    .map(JsonArray.class::cast)
+                    .map(suggestion -> suggestion.getString(0)) // 0 is the search suggestion
+                    .filter(suggestion -> !isBlank(suggestion)) // Filter blank suggestions
+                    .collect(Collectors.toUnmodifiableList());
         } catch (final JsonParserException e) {
-            throw new ParsingException("Could not parse json response", e);
+            throw new ParsingException("Could not parse JSON response", e);
         }
     }
 }