Implemented the "remove duplicates" feature.

Author: Jared234

app/src/main/java/org/schabi/newpipe/database/playlist/dao/PlaylistStreamDAO.java

@@ -84,14 +84,22 @@ default Flowable<List<PlaylistStreamEntity>> listByService(final int serviceId)
             + " ORDER BY " + PLAYLIST_NAME + " COLLATE NOCASE ASC")
     Flowable<List<PlaylistMetadataEntry>> getPlaylistMetadata();
 
+    @RewriteQueriesToDropUnusedColumns
     @Transaction
-    @Query("DELETE FROM " + PLAYLIST_STREAM_JOIN_TABLE
-            + " WHERE " + JOIN_PLAYLIST_ID + "=:playlistId"
-            + " AND " + JOIN_STREAM_ID + " IN ("
-            + " SELECT " + JOIN_STREAM_ID
+    @Query("SELECT *, MIN(" + JOIN_INDEX + ") FROM " + STREAM_TABLE + " INNER JOIN "
+            + "(SELECT " + JOIN_STREAM_ID + "," + JOIN_INDEX
             + " FROM " + PLAYLIST_STREAM_JOIN_TABLE
-            + " WHERE " + JOIN_PLAYLIST_ID + "=:playlistId"
-            + " GROUP BY " + JOIN_STREAM_ID
-            + " HAVING COUNT(*) > 1 )" )
-    Flowable<List<PlaylistMetadataEntry>> removeDuplicates(long playlistId);
+            + " WHERE " + JOIN_PLAYLIST_ID + " = :playlistId)"
+            + " ON " + STREAM_ID + " = " + JOIN_STREAM_ID
+            + " LEFT JOIN "
+            + "(SELECT " + JOIN_STREAM_ID + " AS " + JOIN_STREAM_ID_ALIAS + ", "
+            + STREAM_PROGRESS_MILLIS
+            + " FROM " + STREAM_STATE_TABLE + " )"
+            + " ON " + STREAM_ID + " = " + JOIN_STREAM_ID_ALIAS
+            + " GROUP BY " + STREAM_ID
+            + " ORDER BY " + JOIN_INDEX + " ASC")
+    Flowable<List<PlaylistStreamEntry>> getStreamsWithoutDuplicates(long playlistId);
+
+
+
 }

app/src/main/java/org/schabi/newpipe/local/playlist/LocalPlaylistFragment.java

@@ -627,8 +627,8 @@ private void updateThumbnailUrl() {
     private void openRemoveDuplicatesDialog() {
         final AlertDialog.Builder builder = new AlertDialog.Builder(this.getActivity());
 
-        builder.setTitle("R.string.duplicate_stream_in_playlist_title")
-                .setMessage("test")
+        builder.setTitle(R.string.remove_duplicates_title)
+                .setMessage(R.string.remove_duplicates_message)
                 .setPositiveButton(android.R.string.yes, (dialog, i) -> {
                     removeDuplicatesInPlaylist();
                 })
@@ -638,7 +638,20 @@ private void openRemoveDuplicatesDialog() {
     }
 
     private void removeDuplicatesInPlaylist() {
+        final List<PlaylistStreamEntry> itemsToKeep = playlistManager
+                .getDistinctPlaylistStreams(playlistId).blockingFirst();
 
+        itemListAdapter.clearStreamItemList();
+        itemListAdapter.addItems(itemsToKeep);
+        saveChanges();
+
+        final long videoCount = itemListAdapter.getItemsList().size();
+        setVideoCount(videoCount);
+        if (videoCount == 0) {
+            showEmptyState();
+        }
+        //TODO: Do we have to show loading?
+        //hideLoading();
     }
 
     private void deleteItem(final PlaylistStreamEntry item) {

app/src/main/java/org/schabi/newpipe/local/playlist/LocalPlaylistManager.java

@@ -86,10 +86,9 @@ public Flowable<List<PlaylistMetadataEntry>> getPlaylists() {
         return playlistStreamTable.getPlaylistMetadata().subscribeOn(Schedulers.io());
     }
 
-    public Flowable<List<PlaylistMetadataEntry>> removeDuplicateStreams() {
-        // TODO: Delete Duplicates and rebuild the index
-        // TODO: Rebuild the index
-        return playlistStreamTable.getPlaylistMetadata().subscribeOn(Schedulers.io());
+    public Flowable<List<PlaylistStreamEntry>> getDistinctPlaylistStreams(final long playlistId) {
+        return playlistStreamTable
+                .getStreamsWithoutDuplicates(playlistId).subscribeOn(Schedulers.io());
     }
 
     public Flowable<List<PlaylistStreamEntry>> getPlaylistStreams(final long playlistId) {

app/src/main/res/values/strings.xml

@@ -627,6 +627,8 @@
     <string name="remove_watched">Remove watched</string>
     <string name="remove_watched_popup_title">Remove watched videos?</string>
     <string name="remove_duplicates">Remove duplicates</string>
+    <string name="remove_duplicates_title">Remove duplicates?</string>
+    <string name="remove_duplicates_message">Do you want to remove all duplicate streams in this playlist?</string>
     <string name="remove_watched_popup_warning">Videos that have been watched before and after being added to the playlist will be removed.
 \nAre you sure\? This cannot be undone!</string>
     <string name="remove_watched_popup_yes_and_partially_watched_videos">Yes, and partially watched videos</string>