Include a Other Software section in Help file
[geeqie.git] / src / dupe.c
index e52df5e..f9c624e 100644 (file)
@@ -1,15 +1,25 @@
 /*
- * Geeqie
- * (C) 2005 John Ellis
- * Copyright (C) 2008 - 2009 The Geeqie Team
+ * Copyright (C) 2005 John Ellis
+ * Copyright (C) 2008 - 2016 The Geeqie Team
  *
  * Author: John Ellis
  *
- * This software is released under the GNU General Public License (GNU GPL).
- * Please read the included file COPYING for more information.
- * This software comes with no warranty of any kind, use at your own risk!
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  */
 
+#include <inttypes.h>
 
 #include "main.h"
 #include "dupe.h"
 #include "dnd.h"
 #include "editors.h"
 #include "filedata.h"
+#include "history_list.h"
 #include "image-load.h"
 #include "img-view.h"
-#include "info.h"
 #include "layout.h"
 #include "layout_image.h"
+#include "layout_util.h"
 #include "md5-util.h"
 #include "menu.h"
 #include "misc.h"
+#include "pixbuf_util.h"
 #include "print.h"
 #include "thumb.h"
 #include "ui_fileops.h"
 
 #define DUPE_DEF_WIDTH 800
 #define DUPE_DEF_HEIGHT 400
+#define DUPE_PROGRESS_PULSE_STEP 0.0001
 
-/* column assignment order (simply change them here) */
+/** column assignment order (simply change them here)
+ */
 enum {
        DUPE_COLUMN_POINTER = 0,
        DUPE_COLUMN_RANK,
@@ -58,11 +72,41 @@ enum {
        DUPE_COLUMN_DIMENSIONS,
        DUPE_COLUMN_PATH,
        DUPE_COLUMN_COLOR,
-       DUPE_COLUMN_COUNT       /* total columns */
+       DUPE_COLUMN_SET,
+       DUPE_COLUMN_COUNT       /**< total columns */
+};
+
+typedef enum {
+       DUPE_MATCH = 0,
+       DUPE_NO_MATCH,
+       DUPE_NAME_MATCH
+} DUPE_CHECK_RESULT;
+
+typedef struct _DupeQueueItem DupeQueueItem;
+/** Used for similarity checks. One for each item pushed
+ * onto the thread pool.
+ */
+struct _DupeQueueItem
+{
+       DupeItem *needle;
+       DupeWindow *dw;
+       GList *work; /**< pointer into \a dw->list or \a dw->second_list (#DupeItem) */
+       gint index; /**< The order items pushed onto thread pool. Used to sort returned matches */
 };
 
+typedef struct _DupeSearchMatch DupeSearchMatch;
+/** Used for similarity checks thread. One for each pair match found.
+ */
+struct _DupeSearchMatch
+{
+       DupeItem *a; /**< \a a / \a b matched pair found */
+       DupeItem *b; /**< \a a / \a b matched pair found */
+       gdouble rank;
+       gint index; /**< The order items pushed onto thread pool. Used to sort returned matches */
+};
 
-static GList *dupe_window_list = NULL; /* list of open DupeWindow *s */
+static DupeMatchType param_match_mask;
+static GList *dupe_window_list = NULL; /**< list of open DupeWindow *s */
 
 /*
  * Well, after adding the 'compare two sets' option things got a little sloppy in here
@@ -84,6 +128,81 @@ static GtkWidget *dupe_menu_popup_second(DupeWindow *dw, DupeItem *di);
 static void dupe_dnd_init(DupeWindow *dw);
 
 static void dupe_notify_cb(FileData *fd, NotifyType type, gpointer data);
+static void delete_finished_cb(gboolean success, const gchar *dest_path, gpointer data);
+
+static GtkWidget *submenu_add_export(GtkWidget *menu, GtkWidget **menu_item, GCallback func, gpointer data);
+static void dupe_pop_menu_export_cb(GtkWidget *widget, gpointer data);
+
+static void dupe_init_list_cache(DupeWindow *dw);
+static void dupe_destroy_list_cache(DupeWindow *dw);
+static gboolean dupe_insert_in_list_cache(DupeWindow *dw, FileData *fd);
+
+static void dupe_match_link(DupeItem *a, DupeItem *b, gdouble rank);
+static gint dupe_match_link_exists(DupeItem *child, DupeItem *parent);
+
+/**
+ * @brief The function run in threads for similarity checks
+ * @param d1 #DupeQueueItem
+ * @param d2 #DupeWindow
+ * 
+ * Used only for similarity checks.\n
+ * Search \a dqi->list for \a dqi->needle and if a match is
+ * found, create a #DupeSearchMatch and add to \a dw->search_matches list\n
+ * If \a dw->abort is set, just increment \a dw->thread_count
+ */
+static void dupe_comparison_func(gpointer d1, gpointer d2)
+{
+       DupeQueueItem *dqi = d1;
+       DupeWindow *dw = d2;
+       DupeSearchMatch *dsm;
+       DupeItem *di;
+       GList *matches = NULL;
+       gdouble rank = 0;
+
+       if (!dw->abort)
+               {
+               GList *work = dqi->work;
+               while (work)
+                       {
+                       di = work->data;
+
+                       /* forward for second set, back for simple compare */
+                       if (dw->second_set)
+                               {
+                               work = work->next;
+                               }
+                       else
+                               {
+                               work = work->prev;
+                               }
+
+                       if (dupe_match(di, dqi->needle, dqi->dw->match_mask, &rank, TRUE))
+                               {
+                               dsm = g_new0(DupeSearchMatch, 1);
+                               dsm->a = di;
+                               dsm->b = dqi->needle;
+                               dsm->rank = rank;
+                               matches = g_list_prepend(matches, dsm);
+                               dsm->index = dqi->index;
+                               }
+
+                       if (dw->abort)
+                               {
+                               break;
+                               }
+                       }
+
+               matches = g_list_reverse(matches);
+               g_mutex_lock(&dw->search_matches_mutex);
+               dw->search_matches = g_list_concat(dw->search_matches, matches);
+               g_mutex_unlock(&dw->search_matches_mutex);
+               }
+
+       g_mutex_lock(&dw->thread_count_mutex);
+       dw->thread_count++;
+       g_mutex_unlock(&dw->thread_count_mutex);
+       g_free(dqi);
+}
 
 /*
  * ------------------------------------------------------------------
@@ -91,8 +210,14 @@ static void dupe_notify_cb(FileData *fd, NotifyType type, gpointer data);
  * ------------------------------------------------------------------
  */
 
-
-static void dupe_window_update_count(DupeWindow *dw, gint count_only)
+/**
+ * @brief Update display of status label
+ * @param dw 
+ * @param count_only 
+ * 
+ * 
+ */
+static void dupe_window_update_count(DupeWindow *dw, gboolean count_only)
 {
        gchar *text;
 
@@ -120,6 +245,12 @@ static void dupe_window_update_count(DupeWindow *dw, gint count_only)
        g_free(text);
 }
 
+/**
+ * @brief Returns time in µsec since Epoch
+ * @returns 
+ * 
+ * 
+ */
 static guint64 msec_time(void)
 {
        struct timeval tv;
@@ -134,7 +265,17 @@ static gint dupe_iterations(gint n)
        return (n * ((n + 1) / 2));
 }
 
-static void dupe_window_update_progress(DupeWindow *dw, const gchar *status, gdouble value, gint force)
+/**
+ * @brief 
+ * @param dw 
+ * @param status 
+ * @param value 
+ * @param force 
+ * 
+ * If \a status is blank, clear status bar text and set progress to zero. \n
+ * If \a force is not set, after 2 secs has elapsed, update time-to-go every 250 ms. 
+ */
+static void dupe_window_update_progress(DupeWindow *dw, const gchar *status, gdouble value, gboolean force)
 {
        const gchar *status_text;
 
@@ -218,7 +359,7 @@ static void widget_set_cursor(GtkWidget *widget, gint icon)
 {
        GdkCursor *cursor;
 
-       if (!widget->window) return;
+       if (!gtk_widget_get_window(widget)) return;
 
        if (icon == -1)
                {
@@ -229,7 +370,7 @@ static void widget_set_cursor(GtkWidget *widget, gint icon)
                cursor = gdk_cursor_new(icon);
                }
 
-       gdk_window_set_cursor(widget->window, cursor);
+       gdk_window_set_cursor(gtk_widget_get_window(widget), cursor);
 
        if (cursor) gdk_cursor_unref(cursor);
 }
@@ -246,7 +387,7 @@ static void dupe_listview_realign_colors(DupeWindow *dw)
        GtkTreeIter iter;
        gboolean color_set = TRUE;
        DupeItem *parent = NULL;
-       gint valid;
+       gboolean valid;
 
        store = gtk_tree_view_get_model(GTK_TREE_VIEW(dw->listview));
        valid = gtk_tree_model_get_iter_first(store, &iter);
@@ -289,18 +430,8 @@ static DupeItem *dupe_item_new(FileData *fd)
        di = g_new0(DupeItem, 1);
 
        di->fd = file_data_ref(fd);
-
-       di->group = NULL;
        di->group_rank = 0.0;
 
-       di->simd = NULL;
-       di->checksum = 0;
-       di->md5sum = NULL;
-       di->width = 0;
-       di->height = 0;
-
-       di->second = FALSE;
-
        return di;
 }
 
@@ -414,10 +545,7 @@ static void dupe_item_read_cache(DupeItem *di)
                        {
                        di->width = cd->width;
                        di->height = cd->height;
-                       }
-               if (di->checksum == 0 && cd->have_checksum)
-                       {
-                       di->checksum = cd->checksum;
+                       di->dimensions = (di->width << 16) + di->height;
                        }
                if (!di->md5sum && cd->have_md5sum)
                        {
@@ -443,7 +571,6 @@ static void dupe_item_write_cache(DupeItem *di)
                cd->path = cache_get_location(CACHE_TYPE_SIM, di->fd->path, TRUE, NULL);
 
                if (di->width != 0) cache_sim_data_set_dimensions(cd, di->width, di->height);
-               if (di->checksum != 0) cache_sim_data_set_checksum(cd, di->checksum);
                if (di->md5sum)
                        {
                        guchar digest[16];
@@ -468,7 +595,7 @@ static void dupe_item_write_cache(DupeItem *di)
 
 static gint dupe_listview_find_item(GtkListStore *store, DupeItem *item, GtkTreeIter *iter)
 {
-       gint valid;
+       gboolean valid;
        gint row = 0;
 
        valid = gtk_tree_model_get_iter_first(GTK_TREE_MODEL(store), iter);
@@ -525,6 +652,7 @@ static void dupe_listview_add(DupeWindow *dw, DupeItem *parent, DupeItem *child)
                        {
                        gtk_tree_model_get(GTK_TREE_MODEL(store), &iter, DUPE_COLUMN_COLOR, &color_set, -1);
                        color_set = !color_set;
+                       dw->set_count++;
                        }
                else
                        {
@@ -575,6 +703,7 @@ static void dupe_listview_add(DupeWindow *dw, DupeItem *parent, DupeItem *child)
                                DUPE_COLUMN_DIMENSIONS, text[DUPE_COLUMN_DIMENSIONS],
                                DUPE_COLUMN_PATH, text[DUPE_COLUMN_PATH],
                                DUPE_COLUMN_COLOR, color_set,
+                               DUPE_COLUMN_SET, dw->set_count,
                                -1);
 
        g_free(text[DUPE_COLUMN_RANK]);
@@ -582,6 +711,8 @@ static void dupe_listview_add(DupeWindow *dw, DupeItem *parent, DupeItem *child)
        g_free(text[DUPE_COLUMN_DIMENSIONS]);
 }
 
+static void dupe_listview_select_dupes(DupeWindow *dw, DupeSelectType parents);
+
 static void dupe_listview_populate(DupeWindow *dw)
 {
        GtkListStore *store;
@@ -615,6 +746,16 @@ static void dupe_listview_populate(DupeWindow *dw)
                }
 
        gtk_tree_view_columns_autosize(GTK_TREE_VIEW(dw->listview));
+
+       if (options->duplicates_select_type == DUPE_SELECT_GROUP1)
+               {
+               dupe_listview_select_dupes(dw, DUPE_SELECT_GROUP1);
+               }
+       else if (options->duplicates_select_type == DUPE_SELECT_GROUP2)
+               {
+               dupe_listview_select_dupes(dw, DUPE_SELECT_GROUP2);
+               }
+
 }
 
 static void dupe_listview_remove(DupeWindow *dw, DupeItem *di)
@@ -643,7 +784,7 @@ static GList *dupe_listview_get_filelist(DupeWindow *dw, GtkWidget *listview)
 {
        GtkTreeModel *store;
        GtkTreeIter iter;
-       gint valid;
+       gboolean valid;
        GList *list = NULL;
 
        store = gtk_tree_view_get_model(GTK_TREE_VIEW(listview));
@@ -686,19 +827,19 @@ static GList *dupe_listview_get_selection(DupeWindow *dw, GtkWidget *listview)
                        }
                work = work->next;
                }
-       g_list_foreach(slist, (GFunc)gtk_tree_path_free, NULL);
+       g_list_foreach(slist, (GFunc)tree_path_free_wrapper, NULL);
        g_list_free(slist);
 
        return g_list_reverse(list);
 }
 
-static gint dupe_listview_item_is_selected(DupeWindow *dw, DupeItem *di, GtkWidget *listview)
+static gboolean dupe_listview_item_is_selected(DupeWindow *dw, DupeItem *di, GtkWidget *listview)
 {
        GtkTreeModel *store;
        GtkTreeSelection *selection;
        GList *slist;
        GList *work;
-       gint found = FALSE;
+       gboolean found = FALSE;
 
        selection = gtk_tree_view_get_selection(GTK_TREE_VIEW(listview));
        slist = gtk_tree_selection_get_selected_rows(selection, &store);
@@ -714,18 +855,20 @@ static gint dupe_listview_item_is_selected(DupeWindow *dw, DupeItem *di, GtkWidg
                if (di_n == di) found = TRUE;
                work = work->next;
                }
-       g_list_foreach(slist, (GFunc)gtk_tree_path_free, NULL);
+       g_list_foreach(slist, (GFunc)tree_path_free_wrapper, NULL);
        g_list_free(slist);
 
        return found;
 }
 
-static void dupe_listview_select_dupes(DupeWindow *dw, gint parents)
+static void dupe_listview_select_dupes(DupeWindow *dw, DupeSelectType parents)
 {
        GtkTreeModel *store;
        GtkTreeSelection *selection;
        GtkTreeIter iter;
-       gint valid;
+       gboolean valid;
+       gint set_count = 0;
+       gint set_count_last = -1;
 
        selection = gtk_tree_view_get_selection(GTK_TREE_VIEW(dw->listview));
        gtk_tree_selection_unselect_all(selection);
@@ -736,10 +879,21 @@ static void dupe_listview_select_dupes(DupeWindow *dw, gint parents)
                {
                DupeItem *di;
 
-               gtk_tree_model_get(store, &iter, DUPE_COLUMN_POINTER, &di, -1);
-               if ( (dupe_match_find_parent(dw, di) == di) == (parents) )
+               gtk_tree_model_get(store, &iter, DUPE_COLUMN_POINTER, &di, DUPE_COLUMN_SET, &set_count, -1);
+               if (set_count != set_count_last)
                        {
-                       gtk_tree_selection_select_iter(selection, &iter);
+                       set_count_last = set_count;
+                       if (parents == DUPE_SELECT_GROUP1)
+                               {
+                               gtk_tree_selection_select_iter(selection, &iter);
+                               }
+                       }
+               else
+                       {
+                       if (parents == DUPE_SELECT_GROUP2)
+                               {
+                               gtk_tree_selection_select_iter(selection, &iter);
+                               }
                        }
                valid = gtk_tree_model_iter_next(store, &iter);
                }
@@ -751,6 +905,13 @@ static void dupe_listview_select_dupes(DupeWindow *dw, gint parents)
  * ------------------------------------------------------------------
  */
 
+/**
+ * @brief Search \a parent->group for \a child (#DupeItem)
+ * @param child 
+ * @param parent 
+ * @returns 
+ * 
+ */
 static DupeMatch *dupe_match_find_match(DupeItem *child, DupeItem *parent)
 {
        GList *work;
@@ -765,6 +926,13 @@ static DupeMatch *dupe_match_find_match(DupeItem *child, DupeItem *parent)
        return NULL;
 }
 
+/**
+ * @brief Create #DupeMatch structure for \a child, and insert into \a parent->group list.
+ * @param child 
+ * @param parent 
+ * @param rank 
+ * 
+ */
 static void dupe_match_link_child(DupeItem *child, DupeItem *parent, gdouble rank)
 {
        DupeMatch *dm;
@@ -775,12 +943,26 @@ static void dupe_match_link_child(DupeItem *child, DupeItem *parent, gdouble ran
        parent->group = g_list_append(parent->group, dm);
 }
 
+/**
+ * @brief Link \a a & \a b as both parent and child
+ * @param a 
+ * @param b 
+ * @param rank 
+ * 
+ * Link \a a as child of \a b, and \a b as child of \a a
+ */
 static void dupe_match_link(DupeItem *a, DupeItem *b, gdouble rank)
 {
        dupe_match_link_child(a, b, rank);
        dupe_match_link_child(b, a, rank);
 }
 
+/**
+ * @brief Remove \a child #DupeMatch from \a parent->group list.
+ * @param child 
+ * @param parent 
+ * 
+ */
 static void dupe_match_unlink_child(DupeItem *child, DupeItem *parent)
 {
        DupeMatch *dm;
@@ -793,13 +975,28 @@ static void dupe_match_unlink_child(DupeItem *child, DupeItem *parent)
                }
 }
 
+/**
+ * @brief  Unlink \a a from \a b, and \a b from \a a
+ * @param a 
+ * @param b 
+ *
+ * Free the relevant #DupeMatch items from the #DupeItem group lists
+ */
 static void dupe_match_unlink(DupeItem *a, DupeItem *b)
 {
        dupe_match_unlink_child(a, b);
        dupe_match_unlink_child(b, a);
 }
 
-static void dupe_match_link_clear(DupeItem *parent, gint unlink_children)
+/**
+ * @brief 
+ * @param parent 
+ * @param unlink_children 
+ * 
+ * If \a unlink_children is set, unlink all entries in \a parent->group list. \n
+ * Free the \a parent->group list and set group_rank to zero;
+ */
+static void dupe_match_link_clear(DupeItem *parent, gboolean unlink_children)
 {
        GList *work;
 
@@ -819,11 +1016,25 @@ static void dupe_match_link_clear(DupeItem *parent, gint unlink_children)
        parent->group_rank = 0.0;
 }
 
+/**
+ * @brief Search \a parent->group list for \a child
+ * @param child 
+ * @param parent 
+ * @returns boolean TRUE/FALSE found/not found
+ * 
+ */
 static gint dupe_match_link_exists(DupeItem *child, DupeItem *parent)
 {
        return (dupe_match_find_match(child, parent) != NULL);
 }
 
+/**
+ * @brief  Search \a parent->group for \a child, and return \a child->rank
+ * @param child 
+ * @param parent 
+ * @returns \a dm->di->rank
+ *
+ */
 static gdouble dupe_match_link_rank(DupeItem *child, DupeItem *parent)
 {
        DupeMatch *dm;
@@ -834,6 +1045,15 @@ static gdouble dupe_match_link_rank(DupeItem *child, DupeItem *parent)
        return 0.0;
 }
 
+/**
+ * @brief Find highest rank in \a child->group
+ * @param child 
+ * @returns 
+ * 
+ * Search the #DupeMatch entries in the \a child->group list.
+ * Return the #DupeItem with the highest rank. If more than one have
+ * the same rank, the first encountered is used.
+ */
 static DupeItem *dupe_match_highest_rank(DupeItem *child)
 {
        DupeMatch *dr;
@@ -844,13 +1064,22 @@ static DupeItem *dupe_match_highest_rank(DupeItem *child)
        while (work)
                {
                DupeMatch *dm = work->data;
-               if (!dr || dm->rank > dr->rank) dr = dm;
+               if (!dr || dm->rank > dr->rank)
+                       {
+                       dr = dm;
+                       }
                work = work->next;
                }
 
        return (dr) ? dr->di : NULL;
 }
 
+/** 
+ * @brief Compute and store \a parent->group_rank
+ * @param parent 
+ * 
+ * Group_rank = (sum of all child ranks) / n
+ */
 static void dupe_match_rank_update(DupeItem *parent)
 {
        GList *work;
@@ -893,6 +1122,13 @@ static DupeItem *dupe_match_find_parent(DupeWindow *dw, DupeItem *child)
        return NULL;
 }
 
+/**
+ * @brief 
+ * @param work (#DupeItem) dw->list or dw->second_list
+ * 
+ * Unlink all #DupeItem-s in \a work.
+ * Do not unlink children.
+ */
 static void dupe_match_reset_list(GList *work)
 {
        while (work)
@@ -959,11 +1195,25 @@ static void dupe_match_print_list(GList *list)
 }
 
 /* level 3, unlinking and orphan handling */
+/**
+ * @brief 
+ * @param child 
+ * @param parent \a di from \a child->group
+ * @param[inout] list \a dw->list sorted by rank (#DupeItem)
+ * @param dw 
+ * @returns modified \a list
+ *
+ * Called for each entry in \a child->group (#DupeMatch) with \a parent set to \a dm->di. \n
+ * Find the highest rank #DupeItem of the \a parent's children. \n
+ * If that is == \a child OR
+ * highest rank #DupeItem of \a child == \a parent then FIXME:
+ * 
+ */
 static GList *dupe_match_unlink_by_rank(DupeItem *child, DupeItem *parent, GList *list, DupeWindow *dw)
 {
-       DupeItem *best;
+       DupeItem *best = NULL;
 
-       best = dupe_match_highest_rank(parent);
+       best = dupe_match_highest_rank(parent); // highest rank in parent->group
        if (best == child || dupe_match_highest_rank(child) == parent)
                {
                GList *work;
@@ -991,7 +1241,7 @@ static GList *dupe_match_unlink_by_rank(DupeItem *child, DupeItem *parent, GList
                                }
                        }
 
-               rank = dupe_match_link_rank(child, parent);
+               rank = dupe_match_link_rank(child, parent); // child->rank
                dupe_match_link_clear(parent, TRUE);
                dupe_match_link(child, parent, rank);
                list = g_list_remove(list, parent);
@@ -1007,6 +1257,16 @@ static GList *dupe_match_unlink_by_rank(DupeItem *child, DupeItem *parent, GList
 }
 
 /* level 2 */
+/**
+ * @brief 
+ * @param[inout] list \a dw->list sorted by rank (#DupeItem)
+ * @param di 
+ * @param dw 
+ * @returns modified \a list
+ * 
+ * Called for each entry in \a list.
+ * Call unlink for each child in \a di->group
+ */
 static GList *dupe_match_group_filter(GList *list, DupeItem *di, DupeWindow *dw)
 {
        GList *work;
@@ -1023,6 +1283,15 @@ static GList *dupe_match_group_filter(GList *list, DupeItem *di, DupeWindow *dw)
 }
 
 /* level 1 (top) */
+/**
+ * @brief 
+ * @param[inout] list \a dw->list sorted by rank (#DupeItem)
+ * @param dw 
+ * @returns Filtered \a list
+ * 
+ * Called once.
+ * Call group filter for each \a di in \a list
+ */
 static GList *dupe_match_group_trim(GList *list, DupeWindow *dw)
 {
        GList *work;
@@ -1049,6 +1318,12 @@ static gint dupe_match_sort_groups_cb(gconstpointer a, gconstpointer b)
        return 0;
 }
 
+/**
+ * @brief Sorts the children of each #DupeItem in \a list
+ * @param list #DupeItem
+ * 
+ * Sorts the #DupeItem->group children on rank
+ */
 static void dupe_match_sort_groups(GList *list)
 {
        GList *work;
@@ -1062,6 +1337,28 @@ static void dupe_match_sort_groups(GList *list)
                }
 }
 
+static gint dupe_match_totals_sort_cb(gconstpointer a, gconstpointer b)
+{
+       DupeItem *da = (DupeItem *)a;
+       DupeItem *db = (DupeItem *)b;
+
+       if (g_list_length(da->group) > g_list_length(db->group)) return -1;
+       if (g_list_length(da->group) < g_list_length(db->group)) return 1;
+
+       if (da->group_rank < db->group_rank) return -1;
+       if (da->group_rank > db->group_rank) return 1;
+
+       return 0;
+}
+
+/**
+ * @brief Callback for group_rank sort
+ * @param a 
+ * @param b 
+ * @returns 
+ * 
+ * 
+ */
 static gint dupe_match_rank_sort_cb(gconstpointer a, gconstpointer b)
 {
        DupeItem *da = (DupeItem *)a;
@@ -1072,7 +1369,15 @@ static gint dupe_match_rank_sort_cb(gconstpointer a, gconstpointer b)
        return 0;
 }
 
-/* returns allocated GList of dupes sorted by rank */
+/**
+ * @brief Sorts \a source_list by group-rank
+ * @param source_list #DupeItem
+ * @returns 
+ *
+ * Computes group_rank for each #DupeItem. \n
+ * Items with no group list are ignored.
+ * Returns allocated GList of #DupeItem-s sorted by group_rank
+ */
 static GList *dupe_match_rank_sort(GList *source_list)
 {
        GList *list = NULL;
@@ -1085,7 +1390,7 @@ static GList *dupe_match_rank_sort(GList *source_list)
 
                if (di->group)
                        {
-                       dupe_match_rank_update(di);
+                       dupe_match_rank_update(di); // Compute and store group_rank for di
                        list = g_list_prepend(list, di);
                        }
 
@@ -1095,11 +1400,32 @@ static GList *dupe_match_rank_sort(GList *source_list)
        return g_list_sort(list, dupe_match_rank_sort_cb);
 }
 
+/**
+ * @brief Returns allocated GList of dupes sorted by totals
+ * @param source_list 
+ * @returns 
+ * 
+ * 
+ */
+static GList *dupe_match_totals_sort(GList *source_list)
+{
+       source_list = g_list_sort(source_list, dupe_match_totals_sort_cb);
+
+       source_list = g_list_first(source_list);
+       return g_list_reverse(source_list);
+}
+
+/**
+ * @brief 
+ * @param dw 
+ * 
+ * Called once.
+ */
 static void dupe_match_rank(DupeWindow *dw)
 {
        GList *list;
 
-       list = dupe_match_rank_sort(dw->list);
+       list = dupe_match_rank_sort(dw->list); // sorted by group_rank, no-matches filtered out
 
        if (required_debug_level(2)) dupe_match_print_list(list);
 
@@ -1112,6 +1438,11 @@ static void dupe_match_rank(DupeWindow *dw)
        if (required_debug_level(2)) dupe_match_print_list(list);
 
        list = dupe_match_rank_sort(list);
+       if (options->sort_totals)
+               {
+               list = dupe_match_totals_sort(list);
+               }
+       if (required_debug_level(2)) dupe_match_print_list(list);
 
        g_list_free(dw->dupes);
        dw->dupes = list;
@@ -1123,12 +1454,28 @@ static void dupe_match_rank(DupeWindow *dw)
  * ------------------------------------------------------------------
  */
 
-static gint dupe_match(DupeItem *a, DupeItem *b, DupeMatchType mask, gdouble *rank, gint fast)
+/**
+ * @brief 
+ * @param[in] a 
+ * @param[in] b 
+ * @param[in] mask 
+ * @param[out] rank 
+ * @param[in] fast 
+ * @returns 
+ * 
+ * For similarity checks, compute rank - (similarity factor between a and b). \n
+ * If rank < user-set sim value, returns FALSE.
+ */
+static gboolean dupe_match(DupeItem *a, DupeItem *b, DupeMatchType mask, gdouble *rank, gint fast)
 {
        *rank = 0.0;
 
-       if (a == b) return FALSE;
+       if (a->fd->path == b->fd->path) return FALSE;
 
+       if (mask & DUPE_MATCH_ALL)
+               {
+               return TRUE;
+               }
        if (mask & DUPE_MATCH_PATH)
                {
                if (utf8_compare(a->fd->path, b->fd->path, TRUE) != 0) return FALSE;
@@ -1141,6 +1488,50 @@ static gint dupe_match(DupeItem *a, DupeItem *b, DupeMatchType mask, gdouble *ra
                {
                if (strcmp(a->fd->collate_key_name_nocase, b->fd->collate_key_name_nocase) != 0) return FALSE;
                }
+       if (mask & DUPE_MATCH_NAME_CONTENT)
+               {
+               if (strcmp(a->fd->collate_key_name, b->fd->collate_key_name) == 0)
+                       {
+                       if (!a->md5sum) a->md5sum = md5_text_from_file_utf8(a->fd->path, "");
+                       if (!b->md5sum) b->md5sum = md5_text_from_file_utf8(b->fd->path, "");
+                       if (a->md5sum[0] == '\0' ||
+                           b->md5sum[0] == '\0' ||
+                           strcmp(a->md5sum, b->md5sum) != 0)
+                               {
+                               return TRUE;
+                               }
+                       else
+                               {
+                               return FALSE;
+                               }
+                       }
+               else
+                       {
+                       return FALSE;
+                       }
+               }
+       if (mask & DUPE_MATCH_NAME_CI_CONTENT)
+               {
+               if (strcmp(a->fd->collate_key_name_nocase, b->fd->collate_key_name_nocase) == 0)
+                       {
+                       if (!a->md5sum) a->md5sum = md5_text_from_file_utf8(a->fd->path, "");
+                       if (!b->md5sum) b->md5sum = md5_text_from_file_utf8(b->fd->path, "");
+                       if (a->md5sum[0] == '\0' ||
+                           b->md5sum[0] == '\0' ||
+                           strcmp(a->md5sum, b->md5sum) != 0)
+                               {
+                               return TRUE;
+                               }
+                       else
+                               {
+                               return FALSE;
+                               }
+                       }
+               else
+                       {
+                       return FALSE;
+                       }
+               }
        if (mask & DUPE_MATCH_SIZE)
                {
                if (a->fd->size != b->fd->size) return FALSE;
@@ -1195,94 +1586,514 @@ static gint dupe_match(DupeItem *a, DupeItem *b, DupeMatchType mask, gdouble *ra
        return TRUE;
 }
 
-static void dupe_list_check_match(DupeWindow *dw, DupeItem *needle, GList *start)
+/**
+ * @brief  Determine if there is a match
+ * @param di1 
+ * @param di2 
+ * @param data 
+ * @returns DUPE_MATCH/DUPE_NO_MATCH/DUPE_NAME_MATCH
+ *                     DUPE_NAME_MATCH is used for name != contents searches:
+ *                                                     the name and content match i.e.
+ *                                                     no match, but keep searching
+ * 
+ * Called when stepping down the array looking for adjacent matches,
+ * and from the 2nd set search.
+ * 
+ * Is not used for similarity checks.
+ */
+static DUPE_CHECK_RESULT dupe_match_check(DupeItem *di1, DupeItem *di2, gpointer data)
 {
-       GList *work;
+       DupeWindow *dw = data;
+       DupeMatchType mask = dw->match_mask;
 
-       if (dw->second_set)
+       if (mask & DUPE_MATCH_ALL)
                {
-               work = dw->second_list;
+               return DUPE_MATCH;
                }
-       else if (start)
+       if (mask & DUPE_MATCH_PATH)
                {
-               work = start;
+               if (utf8_compare(di1->fd->path, di2->fd->path, TRUE) != 0)
+                       {
+                       return DUPE_NO_MATCH;
+                       }
                }
-       else
+       if (mask & DUPE_MATCH_NAME)
                {
-               work = g_list_last(dw->list);
+               if (g_strcmp0(di1->fd->collate_key_name, di2->fd->collate_key_name) != 0)
+                       {
+                       return DUPE_NO_MATCH;
+                       }
                }
-
-       while (work)
+       if (mask & DUPE_MATCH_NAME_CI)
                {
-               DupeItem *di = work->data;
-
-               /* speed opt: forward for second set, back for simple compare */
-               if (dw->second_set)
-                       work = work->next;
+               if (g_strcmp0(di1->fd->collate_key_name_nocase, di2->fd->collate_key_name_nocase) != 0 )
+                       {
+                       return DUPE_NO_MATCH;
+                       }
+               }
+       if (mask & DUPE_MATCH_NAME_CONTENT)
+               {
+               if (g_strcmp0(di1->fd->collate_key_name, di2->fd->collate_key_name) == 0)
+                       {
+                       if (g_strcmp0(di1->md5sum, di2->md5sum) == 0)
+                               {
+                               return DUPE_NAME_MATCH;
+                               }
+                       }
                else
-                       work = work->prev;
-
-               if (!dupe_match_link_exists(needle, di))
                        {
-                       gdouble rank;
-
-                       if (dupe_match(di, needle, dw->match_mask, &rank, TRUE))
+                       return DUPE_NO_MATCH;
+                       }
+               }
+       if (mask & DUPE_MATCH_NAME_CI_CONTENT)
+               {
+               if (strcmp(di1->fd->collate_key_name_nocase, di2->fd->collate_key_name_nocase) == 0)
+                       {
+                       if (g_strcmp0(di1->md5sum, di2->md5sum) == 0)
                                {
-                               dupe_match_link(di, needle, rank);
+                               return DUPE_NAME_MATCH;
                                }
                        }
+               else
+                       {
+                       return DUPE_NO_MATCH;
+                       }
                }
-}
-
-/*
- * ------------------------------------------------------------------
- * Thumbnail handling
- * ------------------------------------------------------------------
- */
-
-static void dupe_listview_set_thumb(DupeWindow *dw, DupeItem *di, GtkTreeIter *iter)
-{
-       GtkListStore *store;
-       GtkTreeIter iter_n;
-
-       store = GTK_LIST_STORE(gtk_tree_view_get_model(GTK_TREE_VIEW(dw->listview)));
-       if (!iter)
+       if (mask & DUPE_MATCH_SIZE)
                {
-               if (dupe_listview_find_item(store, di, &iter_n) >= 0)
+               if (di1->fd->size != di2->fd->size)
                        {
-                       iter = &iter_n;
+                       return DUPE_NO_MATCH;
+                       }
+               }
+       if (mask & DUPE_MATCH_DATE)
+               {
+               if (di1->fd->date != di2->fd->date)
+                       {
+                       return DUPE_NO_MATCH;
+                       }
+               }
+       if (mask & DUPE_MATCH_SUM)
+               {
+               if (g_strcmp0(di1->md5sum, di2->md5sum) != 0)
+                       {
+                       return DUPE_NO_MATCH;
+                       }
+               }
+       if (mask & DUPE_MATCH_DIM)
+               {
+               if (di1->dimensions != di2->dimensions)
+                       {
+                       return DUPE_NO_MATCH;
                        }
                }
 
-       if (iter) gtk_list_store_set(store, iter, DUPE_COLUMN_THUMB, di->pixbuf, -1);
+       return DUPE_MATCH;
 }
 
-static void dupe_thumb_do(DupeWindow *dw)
+/**
+ * @brief The callback for the binary search
+ * @param a 
+ * @param b 
+ * @param param_match_mask
+ * @returns negative/0/positive
+ * 
+ * Is not used for similarity checks.
+ *
+ * Used only when two file sets are used.
+ * Requires use of a global for param_match_mask because there is no
+ * g_array_binary_search_with_data() function in glib.
+ */
+static gint dupe_match_binary_search_cb(gconstpointer a, gconstpointer b)
 {
-       DupeItem *di;
-
-       if (!dw->thumb_loader || !dw->thumb_item) return;
-       di = dw->thumb_item;
-
-       if (di->pixbuf) g_object_unref(di->pixbuf);
-       di->pixbuf = thumb_loader_get_pixbuf(dw->thumb_loader);
-
-       dupe_listview_set_thumb(dw, di, NULL);
-}
+       const DupeItem *di1 = *((DupeItem **) a);
+       const DupeItem *di2 = b;
+       DupeMatchType mask = param_match_mask;
 
-static void dupe_thumb_error_cb(ThumbLoader *tl, gpointer data)
-{
-       DupeWindow *dw = data;
+       if (mask & DUPE_MATCH_ALL)
+               {
+               return 0;
+               }
+       if (mask & DUPE_MATCH_PATH)
+               {
+               return utf8_compare(di1->fd->path, di2->fd->path, TRUE);
+               }
+       if (mask & DUPE_MATCH_NAME)
+               {
+               return g_strcmp0(di1->fd->collate_key_name, di2->fd->collate_key_name);
+               }
+       if (mask & DUPE_MATCH_NAME_CI)
+               {
+               return strcmp(di1->fd->collate_key_name_nocase, di2->fd->collate_key_name_nocase);
+               }
+       if (mask & DUPE_MATCH_NAME_CONTENT)
+               {
+               return g_strcmp0(di1->fd->collate_key_name, di2->fd->collate_key_name);
+               }
+       if (mask & DUPE_MATCH_NAME_CI_CONTENT)
+               {
+               return strcmp(di1->fd->collate_key_name_nocase, di2->fd->collate_key_name_nocase);
+               }
+       if (mask & DUPE_MATCH_SIZE)
+               {
+               return (di1->fd->size - di2->fd->size);
+               }
+       if (mask & DUPE_MATCH_DATE)
+               {
+               return (di1->fd->date - di2->fd->date);
+               }
+       if (mask & DUPE_MATCH_SUM)
+               {
+               return g_strcmp0(di1->md5sum, di2->md5sum);
+               }
+       if (mask & DUPE_MATCH_DIM)
+               {
+               return (di1->dimensions - di2->dimensions);
+               }
 
-       dupe_thumb_do(dw);
-       dupe_thumb_step(dw);
+       return 0;
 }
 
-static void dupe_thumb_done_cb(ThumbLoader *tl, gpointer data)
+/**
+ * @brief The callback for the array sort
+ * @param a 
+ * @param b 
+ * @param data 
+ * @returns negative/0/positive
+ * 
+ * Is not used for similarity checks.
+*/
+static gint dupe_match_sort_cb(gconstpointer a, gconstpointer b, gpointer data)
 {
+       const DupeItem *di1 = *((DupeItem **) a);
+       const DupeItem *di2 = *((DupeItem **) b);
        DupeWindow *dw = data;
+       DupeMatchType mask = dw->match_mask;
 
-       dupe_thumb_do(dw);
+       if (mask & DUPE_MATCH_ALL)
+               {
+               return 0;
+               }
+       if (mask & DUPE_MATCH_PATH)
+               {
+               return utf8_compare(di1->fd->path, di2->fd->path, TRUE);
+               }
+       if (mask & DUPE_MATCH_NAME)
+               {
+               return g_strcmp0(di1->fd->collate_key_name, di2->fd->collate_key_name);
+               }
+       if (mask & DUPE_MATCH_NAME_CI)
+               {
+               return strcmp(di1->fd->collate_key_name_nocase, di2->fd->collate_key_name_nocase);
+               }
+       if (mask & DUPE_MATCH_NAME_CONTENT)
+               {
+               return g_strcmp0(di1->fd->collate_key_name, di2->fd->collate_key_name);
+               }
+       if (mask & DUPE_MATCH_NAME_CI_CONTENT)
+               {
+               return strcmp(di1->fd->collate_key_name_nocase, di2->fd->collate_key_name_nocase);
+               }
+       if (mask & DUPE_MATCH_SIZE)
+               {
+               return (di1->fd->size - di2->fd->size);
+               }
+       if (mask & DUPE_MATCH_DATE)
+               {
+               return (di1->fd->date - di2->fd->date);
+               }
+       if (mask & DUPE_MATCH_SUM)
+               {
+               if (di1->md5sum[0] == '\0' || di2->md5sum[0] == '\0')
+                   {
+                       return -1;
+                       }
+               else
+                       {
+                       return strcmp(di1->md5sum, di2->md5sum);
+                       }
+               }
+       if (mask & DUPE_MATCH_DIM)
+               {
+               if (!di1 || !di2 || !di1->width || !di1->height || !di2->width || !di2->height)
+                       {
+                       return -1;
+                       }
+               return (di1->dimensions - di2->dimensions);
+               }
+
+       return 0; // should not execute
+}
+
+/**
+ * @brief Check for duplicate matches
+ * @param dw 
+ *
+ * Is not used for similarity checks.
+ *
+ * Loads the file sets into an array and sorts on the searched
+ * for parameter.
+ * 
+ * If one file set, steps down the array looking for adjacent equal values.
+ * 
+ * If two file sets, steps down the first set and for each value
+ * does a binary search for matches in the second set.
+ */ 
+static void dupe_array_check(DupeWindow *dw )
+{
+       GArray *array_set1;
+       GArray *array_set2;
+       GList *work;
+       gint i_set1;
+       gint i_set2;
+       DUPE_CHECK_RESULT check_result;
+       DupeMatchType mask = dw->match_mask;
+       param_match_mask = dw->match_mask;
+       guint out_match_index;
+       gboolean match_found = FALSE;;
+
+       if (!dw->list) return;
+
+       array_set1 = g_array_new(TRUE, TRUE, sizeof(gpointer));
+       array_set2 = g_array_new(TRUE, TRUE, sizeof(gpointer));
+       dupe_match_reset_list(dw->list);
+
+       work = dw->list;
+       while (work)
+               {
+               DupeItem *di = work->data;
+               g_array_append_val(array_set1, di);
+               work = work->next;
+               }
+
+       g_array_sort_with_data(array_set1, dupe_match_sort_cb, dw);
+
+       if (dw->second_set)
+               {
+               /* Two sets - nothing can be done until a second set is loaded */
+               if (dw->second_list)
+                       {
+                       work = dw->second_list;
+                       while (work)
+                               {
+                               DupeItem *di = work->data;
+                               g_array_append_val(array_set2, (work->data));
+                               work = work->next;
+                               }
+                       g_array_sort_with_data(array_set2, dupe_match_sort_cb, dw);
+
+                       for (i_set1 = 0; i_set1 <= (gint)(array_set1->len) - 1; i_set1++)
+                               {
+                               DupeItem *di1 = g_array_index(array_set1, gpointer, i_set1);
+                               DupeItem *di2 = NULL;
+                               /* If multiple identical entries in set 1, use the last one */
+                               if (i_set1 < (gint)(array_set1->len) - 2)
+                                       {
+                                       di2 = g_array_index(array_set1, gpointer, i_set1 + 1);
+                                       check_result = dupe_match_check(di1, di2, dw);
+                                       if (check_result == DUPE_MATCH || check_result == DUPE_NAME_MATCH)
+                                               {
+                                               continue;
+                                               }
+                                       }
+
+#if ((GLIB_MAJOR_VERSION == 2) && (GLIB_MINOR_VERSION >= 62))
+                               match_found = g_array_binary_search(array_set2, di1, dupe_match_binary_search_cb, &out_match_index);
+#else
+                               gint i;
+
+                               match_found = FALSE;
+                               for(i=0; i < array_set2->len; i++)
+                                       {
+                                       di2 = g_array_index(array_set2,  gpointer, i);
+                                       check_result = dupe_match_check(di1, di2, dw);
+                                       if (check_result == DUPE_MATCH)
+                                               {
+                                               match_found = TRUE;
+                                               out_match_index = i;
+                                               break;
+                                               }
+                                       }
+#endif
+
+                               if (match_found)
+                                       {
+                                       di2 = g_array_index(array_set2, gpointer, out_match_index);
+
+                                       check_result = dupe_match_check(di1, di2, dw);
+                                       if (check_result == DUPE_MATCH || check_result == DUPE_NAME_MATCH)
+                                               {
+                                               if (check_result == DUPE_MATCH)
+                                                       {
+                                                       dupe_match_link(di2, di1, 0.0);
+                                                       }
+                                               i_set2 = out_match_index + 1;
+
+                                               if (i_set2 > (gint)(array_set2->len) - 1)
+                                                       {
+                                                       break;
+                                                       }
+                                               /* Look for multiple matches in set 2 for item di1 */
+                                               di2 = g_array_index(array_set2, gpointer, i_set2);
+                                               check_result = dupe_match_check(di1, di2, dw);
+                                               while (check_result == DUPE_MATCH || check_result == DUPE_NAME_MATCH)
+                                                       {
+                                                       if (check_result == DUPE_MATCH)
+                                                               {
+                                                               dupe_match_link(di2, di1, 0.0);
+                                                               }
+                                                       i_set2++;
+                                                       if (i_set2 > (gint)(array_set2->len) - 1)
+                                                               {
+                                                               break;
+                                                               }
+                                                       di2 = g_array_index(array_set2, gpointer, i_set2);
+                                                       check_result = dupe_match_check(di1, di2, dw);
+                                                       }
+                                               }
+                                       }
+                               }
+                       }
+               }
+       else
+               {
+               /* File set 1 only */
+               g_list_free(dw->dupes);
+               dw->dupes = NULL;
+
+               if ((gint)(array_set1->len) > 1)
+                       {
+                       for (i_set1 = 0; i_set1 <= (gint)(array_set1->len) - 2; i_set1++)
+                               {
+                               DupeItem *di1 = g_array_index(array_set1, gpointer, i_set1);
+                               DupeItem *di2 = g_array_index(array_set1, gpointer, i_set1 + 1);
+
+                               check_result = dupe_match_check(di1, di2, dw);
+                               if (check_result == DUPE_MATCH || check_result == DUPE_NAME_MATCH)
+                                       {
+                                       if (check_result == DUPE_MATCH)
+                                               {
+                                               dupe_match_link(di2, di1, 0.0);
+                                               }
+                                       i_set1++;
+
+                                       if ( i_set1 + 1 > (gint)(array_set1->len) - 1)
+                                               {
+                                               break;
+                                               }
+                                       /* Look for multiple matches for item di1 */
+                                       di2 = g_array_index(array_set1, gpointer, i_set1 + 1);
+                                       check_result = dupe_match_check(di1, di2, dw);
+                                       while (check_result == DUPE_MATCH || check_result == DUPE_NAME_MATCH)
+                                               {
+                                               if (check_result == DUPE_MATCH)
+                                                       {
+                                                       dupe_match_link(di2, di1, 0.0);
+                                                       }
+                                               i_set1++;
+
+                                               if (i_set1 + 1 > (gint)(array_set1->len) - 1)
+                                                       {
+                                                       break;
+                                                       }
+                                               di2 = g_array_index(array_set1, gpointer, i_set1 + 1);
+                                               check_result = dupe_match_check(di1, di2, dw);
+                                               }
+                                       }
+                               }
+                       }
+               }
+       g_array_free(array_set1, TRUE);
+       g_array_free(array_set2, TRUE);
+}
+
+/**
+ * @brief Look for similarity match
+ * @param dw 
+ * @param needle 
+ * @param start 
+ * 
+ * Only used for similarity checks.\n
+ * Called from dupe_check_cb.
+ * Called for each entry in the list.
+ * Steps through the list looking for matches against needle.
+ * Pushes a #DupeQueueItem onto thread pool queue.
+ */
+static void dupe_list_check_match(DupeWindow *dw, DupeItem *needle, GList *start)
+{
+       GList *work;
+       DupeQueueItem *dqi;
+
+       if (dw->second_set)
+               {
+               work = dw->second_list;
+               }
+       else if (start)
+               {
+               work = start;
+               }
+       else
+               {
+               work = g_list_last(dw->list);
+               }
+
+       dqi = g_new0(DupeQueueItem, 1);
+       dqi->needle = needle;
+       dqi->dw = dw;
+       dqi->work = work;
+       dqi->index = dw->queue_count;
+       g_thread_pool_push(dw->dupe_comparison_thread_pool, dqi, NULL);
+}
+
+/*
+ * ------------------------------------------------------------------
+ * Thumbnail handling
+ * ------------------------------------------------------------------
+ */
+
+static void dupe_listview_set_thumb(DupeWindow *dw, DupeItem *di, GtkTreeIter *iter)
+{
+       GtkListStore *store;
+       GtkTreeIter iter_n;
+
+       store = GTK_LIST_STORE(gtk_tree_view_get_model(GTK_TREE_VIEW(dw->listview)));
+       if (!iter)
+               {
+               if (dupe_listview_find_item(store, di, &iter_n) >= 0)
+                       {
+                       iter = &iter_n;
+                       }
+               }
+
+       if (iter) gtk_list_store_set(store, iter, DUPE_COLUMN_THUMB, di->pixbuf, -1);
+}
+
+static void dupe_thumb_do(DupeWindow *dw)
+{
+       DupeItem *di;
+
+       if (!dw->thumb_loader || !dw->thumb_item) return;
+       di = dw->thumb_item;
+
+       if (di->pixbuf) g_object_unref(di->pixbuf);
+       di->pixbuf = thumb_loader_get_pixbuf(dw->thumb_loader);
+
+       dupe_listview_set_thumb(dw, di, NULL);
+}
+
+static void dupe_thumb_error_cb(ThumbLoader *tl, gpointer data)
+{
+       DupeWindow *dw = data;
+
+       dupe_thumb_do(dw);
+       dupe_thumb_step(dw);
+}
+
+static void dupe_thumb_done_cb(ThumbLoader *tl, gpointer data)
+{
+       DupeWindow *dw = data;
+
+       dupe_thumb_do(dw);
        dupe_thumb_step(dw);
 }
 
@@ -1291,7 +2102,7 @@ static void dupe_thumb_step(DupeWindow *dw)
        GtkTreeModel *store;
        GtkTreeIter iter;
        DupeItem *di = NULL;
-       gint valid;
+       gboolean valid;
        gint row = 0;
        gint length = 0;
 
@@ -1358,10 +2169,45 @@ static void dupe_thumb_step(DupeWindow *dw)
 
 static void dupe_check_stop(DupeWindow *dw)
 {
-       if (dw->idle_id != -1 || dw->img_loader || dw->thumb_loader)
+       if (dw->idle_id > 0)
                {
                g_source_remove(dw->idle_id);
-               dw->idle_id = -1;
+               dw->idle_id = 0;
+               }
+
+       dw->abort = TRUE;
+
+       while (dw->thread_count < dw->queue_count) // Wait for the queue to empty
+               {
+               dupe_window_update_progress(dw, NULL, 0.0, FALSE);
+               widget_set_cursor(dw->listview, -1);
+               }
+
+       g_list_free(dw->search_matches);
+       dw->search_matches = NULL;
+
+       if (dw->idle_id || dw->img_loader || dw->thumb_loader)
+               {
+               if (dw->idle_id > 0)
+                       {
+                       g_source_remove(dw->idle_id);
+                       dw->idle_id = 0;
+                       }
+               dupe_window_update_progress(dw, NULL, 0.0, FALSE);
+               widget_set_cursor(dw->listview, -1);
+               }
+
+       if (dw->add_files_queue_id)
+               {
+               g_source_remove(dw->add_files_queue_id);
+               dw->add_files_queue_id = 0;
+               dupe_destroy_list_cache(dw);
+               gtk_widget_set_sensitive(dw->controls_box, TRUE);
+               if (g_list_length(dw->add_files_queue) > 0)
+                       {
+                       filelist_free(dw->add_files_queue);
+                       }
+               dw->add_files_queue = NULL;
                dupe_window_update_progress(dw, NULL, 0.0, FALSE);
                widget_set_cursor(dw->listview, -1);
                }
@@ -1373,6 +2219,13 @@ static void dupe_check_stop(DupeWindow *dw)
        dw->img_loader = NULL;
 }
 
+static void dupe_check_stop_cb(GtkWidget *widget, gpointer data)
+{
+       DupeWindow *dw = data;
+
+       dupe_check_stop(dw);
+}
+
 static void dupe_loader_done_cb(ImageLoader *il, gpointer data)
 {
        DupeWindow *dw = data;
@@ -1431,18 +2284,22 @@ static GList *dupe_setup_point_step(DupeWindow *dw, GList *p)
        return NULL;
 }
 
-static gint dupe_check_cb(gpointer data)
+/**
+ * @brief Generates the sumcheck or dimensions
+ * @param list Set1 or set2
+ * @returns TRUE/FALSE = not completed/completed
+ * 
+ * Ensures that the DIs contain the MD5SUM or dimensions for all items in
+ * the list. One item at a time. Re-enters if not completed.
+ */
+static gboolean create_checksums_dimensions(DupeWindow *dw, GList *list)
 {
-       DupeWindow *dw = data;
-
-       if (dw->idle_id == -1) return FALSE;
-
-       if (!dw->setup_done)
-               {
-               if ((dw->match_mask & DUPE_MATCH_SUM) &&
-                   !(dw->setup_mask & DUPE_MATCH_SUM) )
+               if ((dw->match_mask & DUPE_MATCH_SUM) ||
+                       (dw->match_mask & DUPE_MATCH_NAME_CONTENT) ||
+                       (dw->match_mask & DUPE_MATCH_NAME_CI_CONTENT))
                        {
-                       if (!dw->setup_point) dw->setup_point = dw->list;
+                       /* MD5SUM only */
+                       if (!dw->setup_point) dw->setup_point = list; // setup_point clear on 1st entry
 
                        while (dw->setup_point)
                                {
@@ -1459,7 +2316,10 @@ static gint dupe_check_cb(gpointer data)
                                        if (options->thumbnails.enable_caching)
                                                {
                                                dupe_item_read_cache(di);
-                                               if (di->md5sum) return TRUE;
+                                               if (di->md5sum)
+                                                       {
+                                                       return TRUE;
+                                                       }
                                                }
 
                                        di->md5sum = md5_text_from_file_utf8(di->fd->path, "");
@@ -1470,13 +2330,13 @@ static gint dupe_check_cb(gpointer data)
                                        return TRUE;
                                        }
                                }
-                       dw->setup_mask |= DUPE_MATCH_SUM;
                        dupe_setup_reset(dw);
                        }
-               if ((dw->match_mask & DUPE_MATCH_DIM) &&
-                   !(dw->setup_mask & DUPE_MATCH_DIM) )
+
+               if ((dw->match_mask & DUPE_MATCH_DIM)  )
                        {
-                       if (!dw->setup_point) dw->setup_point = dw->list;
+                       /* Dimensions only */
+                       if (!dw->setup_point) dw->setup_point = list;
 
                        while (dw->setup_point)
                                {
@@ -1492,10 +2352,14 @@ static gint dupe_check_cb(gpointer data)
                                        if (options->thumbnails.enable_caching)
                                                {
                                                dupe_item_read_cache(di);
-                                               if (di->width != 0 || di->height != 0) return TRUE;
+                                               if (di->width != 0 || di->height != 0)
+                                                       {
+                                                       return TRUE;
+                                                       }
                                                }
 
                                        image_load_dimensions(di->fd, &di->width, &di->height);
+                                       di->dimensions = (di->width << 16) + di->height;
                                        if (options->thumbnails.enable_caching)
                                                {
                                                dupe_item_write_cache(di);
@@ -1503,15 +2367,69 @@ static gint dupe_check_cb(gpointer data)
                                        return TRUE;
                                        }
                                }
-                       dw->setup_mask |= DUPE_MATCH_DIM;
                        dupe_setup_reset(dw);
                        }
+
+       return FALSE;
+}
+
+/**
+ * @brief Compare func. for sorting search matches
+ * @param a #DupeSearchMatch
+ * @param b #DupeSearchMatch
+ * @returns 
+ * 
+ * Used only for similarity checks\n
+ * Sorts search matches on order they were inserted into the pool queue
+ */
+static gint sort_func(gconstpointer a, gconstpointer b)
+{
+       return (((DupeSearchMatch *)a)->index - ((DupeSearchMatch *)b)->index);
+}
+
+/**
+ * @brief Check set 1 (and set 2) for matches
+ * @param data DupeWindow
+ * @returns TRUE/FALSE = not completed/completed
+ * 
+ * Initiated from start, loader done and item remove
+ *
+ * On first entry generates di->MD5SUM, di->dimensions and sim data,
+ * and updates the cache.
+ */
+static gboolean dupe_check_cb(gpointer data)
+{
+       DupeWindow *dw = data;
+       DupeSearchMatch *search_match_list_item;
+
+       if (!dw->idle_id)
+               {
+               return FALSE;
+               }
+
+       if (!dw->setup_done) /* Clear on 1st entry */
+               {
+               if (dw->list)
+                       {
+                       if (create_checksums_dimensions(dw, dw->list))
+                               {
+                               return TRUE;
+                               }
+                       }
+               if (dw->second_list)
+                       {
+                       if (create_checksums_dimensions(dw, dw->second_list))
+                               {
+                               return TRUE;
+                               }
+                       }
                if ((dw->match_mask & DUPE_MATCH_SIM_HIGH ||
                     dw->match_mask & DUPE_MATCH_SIM_MED ||
                     dw->match_mask & DUPE_MATCH_SIM_LOW ||
                     dw->match_mask & DUPE_MATCH_SIM_CUSTOM) &&
                    !(dw->setup_mask & DUPE_MATCH_SIM_MED) )
                        {
+                       /* Similarity only */
                        if (!dw->setup_point) dw->setup_point = dw->list;
 
                        while (dw->setup_point)
@@ -1535,8 +2453,8 @@ static gint dupe_check_cb(gpointer data)
 
                                        dw->img_loader = image_loader_new(di->fd);
                                        image_loader_set_buffer_size(dw->img_loader, 8);
-                                       g_signal_connect (G_OBJECT(dw->img_loader), "error", (GCallback)dupe_loader_done_cb, dw);
-                                       g_signal_connect (G_OBJECT(dw->img_loader), "done", (GCallback)dupe_loader_done_cb, dw);
+                                       g_signal_connect(G_OBJECT(dw->img_loader), "error", (GCallback)dupe_loader_done_cb, dw);
+                                       g_signal_connect(G_OBJECT(dw->img_loader), "done", (GCallback)dupe_loader_done_cb, dw);
 
                                        if (!image_loader_start(dw->img_loader))
                                                {
@@ -1546,7 +2464,7 @@ static gint dupe_check_cb(gpointer data)
                                                dw->img_loader = NULL;
                                                return TRUE;
                                                }
-                                       dw->idle_id = -1;
+                                       dw->idle_id = 0;
                                        return FALSE;
                                        }
 
@@ -1556,21 +2474,74 @@ static gint dupe_check_cb(gpointer data)
                        dw->setup_mask |= DUPE_MATCH_SIM_MED;
                        dupe_setup_reset(dw);
                        }
+
+               /* End of setup not done */
                dupe_window_update_progress(dw, _("Comparing..."), 0.0, FALSE);
                dw->setup_done = TRUE;
                dupe_setup_reset(dw);
                dw->setup_count = g_list_length(dw->list);
                }
 
+       /* Setup done - dw->working set to NULL below
+        * Set before 1st entry: dw->working = g_list_last(dw->list)
+        * Set before 1st entry: dw->setup_count = g_list_length(dw->list)
+        */
        if (!dw->working)
                {
-               if (dw->setup_count > 0)
+               /* Similarity check threads may still be running */
+               if (dw->setup_count > 0 && (dw->match_mask == DUPE_MATCH_SIM_HIGH ||
+                       dw->match_mask == DUPE_MATCH_SIM_MED ||
+                       dw->match_mask == DUPE_MATCH_SIM_LOW ||
+                       dw->match_mask == DUPE_MATCH_SIM_CUSTOM))
                        {
+                       if( dw->thread_count < dw->queue_count)
+                               {
+                               dupe_window_update_progress(dw, _("Comparing..."), 0.0, FALSE);
+
+                               return TRUE;
+                               }
+
+                       if (dw->search_matches_sorted == NULL)
+                               {
+                               dw->search_matches_sorted = g_list_sort(dw->search_matches, sort_func);
+                               dupe_setup_reset(dw);
+                               }
+
+                       while (dw->search_matches_sorted)
+                               {
+                               dw->setup_n++;
+                               dupe_window_update_progress(dw, _("Sorting..."), 0.0, FALSE);
+                               search_match_list_item = dw->search_matches_sorted->data;
+
+                               if (!dupe_match_link_exists(search_match_list_item->a, search_match_list_item->b))
+                                       {
+                                       dupe_match_link(search_match_list_item->a, search_match_list_item->b, search_match_list_item->rank);
+                                       }
+
+                               dw->search_matches_sorted = dw->search_matches_sorted->next;
+
+                               if (dw->search_matches_sorted != NULL)
+                                       {
+                                       return TRUE;
+                                       }
+                               }
+                       g_list_free(dw->search_matches);
+                       dw->search_matches = NULL;
+                       g_list_free(dw->search_matches_sorted);
+                       dw->search_matches_sorted = NULL;
                        dw->setup_count = 0;
-                       dupe_window_update_progress(dw, _("Sorting..."), 1.0, TRUE);
-                       return TRUE;
                        }
-               dw->idle_id = -1;
+               else
+                       {
+                       if (dw->setup_count > 0)
+                               {
+                               dw->setup_count = 0;
+                               dupe_window_update_progress(dw, _("Sorting..."), 1.0, TRUE);
+                               return TRUE;
+                               }
+                       }
+
+               dw->idle_id = 0;
                dupe_window_update_progress(dw, NULL, 0.0, FALSE);
 
                dupe_match_rank(dw);
@@ -1584,13 +2555,32 @@ static gint dupe_check_cb(gpointer data)
                widget_set_cursor(dw->listview, -1);
 
                return FALSE;
+               /* The end */
                }
 
-       dupe_list_check_match(dw, (DupeItem *)dw->working->data, dw->working);
-       dupe_window_update_progress(dw, _("Comparing..."), dw->setup_count == 0 ? 0.0 : (gdouble) dw->setup_n / dw->setup_count, FALSE);
-       dw->setup_n++;
+       /* Setup done - working */
+       if (dw->match_mask == DUPE_MATCH_SIM_HIGH ||
+               dw->match_mask == DUPE_MATCH_SIM_MED ||
+               dw->match_mask == DUPE_MATCH_SIM_LOW ||
+               dw->match_mask == DUPE_MATCH_SIM_CUSTOM)
+               {
+               /* This is the similarity comparison */
+               dupe_list_check_match(dw, (DupeItem *)dw->working->data, dw->working);
+               dupe_window_update_progress(dw, _("Queuing..."), dw->setup_count == 0 ? 0.0 : (gdouble) dw->setup_n / dw->setup_count, FALSE);
+               dw->setup_n++;
+               dw->queue_count++;
 
-       dw->working = dw->working->prev;
+               dw->working = dw->working->prev; /* Is NULL when complete */
+               }
+       else
+               {
+               /* This is the comparison for all other parameters.
+                * dupe_array_check() processes the entire list in one go
+               */
+               dw->working = NULL;
+               dupe_window_update_progress(dw, _("Comparing..."), 0.0, FALSE);
+               dupe_array_check(dw);
+               }
 
        return TRUE;
 }
@@ -1609,10 +2599,23 @@ static void dupe_check_start(DupeWindow *dw)
 
        dupe_window_update_count(dw, TRUE);
        widget_set_cursor(dw->listview, GDK_WATCH);
+       dw->queue_count = 0;
+       dw->thread_count = 0;
+       dw->search_matches_sorted = NULL;
+       dw->abort = FALSE;
+
+       if (dw->idle_id) return;
+
+       dw->idle_id = g_idle_add(dupe_check_cb, dw);
+}
+
+static gboolean dupe_check_start_cb(gpointer data)
+{
+       DupeWindow *dw = data;
 
-       if (dw->idle_id != -1) return;
+       dupe_check_start(dw);
 
-       dw->idle_id = g_idle_add(dupe_check_cb, dw);
+       return FALSE;
 }
 
 /*
@@ -1707,7 +2710,7 @@ static void dupe_item_remove(DupeWindow *dw, DupeItem *di)
        dupe_window_update_count(dw, FALSE);
 }
 
-static gint dupe_item_remove_by_path(DupeWindow *dw, const gchar *path)
+static gboolean dupe_item_remove_by_path(DupeWindow *dw, const gchar *path)
 {
        DupeItem *di;
 
@@ -1719,8 +2722,94 @@ static gint dupe_item_remove_by_path(DupeWindow *dw, const gchar *path)
        return TRUE;
 }
 
+static gboolean dupe_files_add_queue_cb(gpointer data)
+{
+       DupeItem *di = NULL;
+       DupeWindow *dw = data;
+       FileData *fd;
+       GList *queue = dw->add_files_queue;
+
+       gtk_progress_bar_pulse(GTK_PROGRESS_BAR(dw->extra_label));
+
+       if (queue == NULL)
+               {
+               dw->add_files_queue_id = 0;
+               dupe_destroy_list_cache(dw);
+               g_idle_add(dupe_check_start_cb, dw);
+               gtk_widget_set_sensitive(dw->controls_box, TRUE);
+               return FALSE;
+               }
+
+       fd = queue->data;
+       if (fd)
+               {
+               if (isfile(fd->path))
+                       {
+                       di = dupe_item_new(fd);
+                       }
+               else if (isdir(fd->path))
+                       {
+                       GList *f, *d;
+                       dw->add_files_queue = g_list_remove(dw->add_files_queue, g_list_first(dw->add_files_queue)->data);
+
+                       if (filelist_read(fd, &f, &d))
+                               {
+                               f = filelist_filter(f, FALSE);
+                               d = filelist_filter(d, TRUE);
+
+                               dw->add_files_queue = g_list_concat(f, dw->add_files_queue);
+                               dw->add_files_queue = g_list_concat(d, dw->add_files_queue);
+                               }
+                       }
+               else
+                       {
+                       /* Not a file and not a dir */
+                       dw->add_files_queue = g_list_remove(dw->add_files_queue, g_list_first(dw->add_files_queue)->data);
+                       }
+               }
+
+       if (!di)
+               {
+               /* A dir was found. Process the contents on next entry */
+               return TRUE;
+               }
+
+       dw->add_files_queue = g_list_remove(dw->add_files_queue, g_list_first(dw->add_files_queue)->data);
+
+       dupe_item_read_cache(di);
+
+       /* Ensure images in the lists have unique FileDatas */
+       if (!dupe_insert_in_list_cache(dw, di->fd))
+               {
+               dupe_item_free(di);
+               return TRUE;
+               }
+
+       if (dw->second_drop)
+               {
+               dupe_second_add(dw, di);
+               }
+       else
+               {
+               dw->list = g_list_prepend(dw->list, di);
+               }
+
+       if (dw->add_files_queue != NULL)
+               {
+               return TRUE;
+               }
+       else
+               {
+               dw->add_files_queue_id = 0;
+               dupe_destroy_list_cache(dw);
+               g_idle_add(dupe_check_start_cb, dw);
+               gtk_widget_set_sensitive(dw->controls_box, TRUE);
+               return FALSE;
+               }
+}
+
 static void dupe_files_add(DupeWindow *dw, CollectionData *collection, CollectInfo *info,
-                          FileData *fd, gint recurse)
+                          FileData *fd, gboolean recurse)
 {
        DupeItem *di = NULL;
 
@@ -1730,7 +2819,7 @@ static void dupe_files_add(DupeWindow *dw, CollectionData *collection, CollectIn
                }
        else if (fd)
                {
-               if (isfile(fd->path))
+               if (isfile(fd->path) && !g_file_test(fd->path, G_FILE_TEST_IS_SYMLINK))
                        {
                        di = dupe_item_new(fd);
                        }
@@ -1764,6 +2853,42 @@ static void dupe_files_add(DupeWindow *dw, CollectionData *collection, CollectIn
 
        if (!di) return;
 
+       dupe_item_read_cache(di);
+
+       /* Ensure images in the lists have unique FileDatas */
+       GList *work;
+       DupeItem *di_list;
+       work = g_list_first(dw->list);
+       while (work)
+               {
+               di_list = work->data;
+               if (di_list->fd == di->fd)
+                       {
+                       return;
+                       }
+               else
+                       {
+                       work = work->next;
+                       }
+               }
+
+       if (dw->second_list)
+               {
+               work = g_list_first(dw->second_list);
+               while (work)
+                       {
+                       di_list = work->data;
+                       if (di_list->fd == di->fd)
+                               {
+                               return;
+                               }
+                       else
+                               {
+                               work = work->next;
+                               }
+                       }
+               }
+
        if (dw->second_drop)
                {
                dupe_second_add(dw, di);
@@ -1774,6 +2899,51 @@ static void dupe_files_add(DupeWindow *dw, CollectionData *collection, CollectIn
                }
 }
 
+static void dupe_init_list_cache(DupeWindow *dw)
+{
+       dw->list_cache = g_hash_table_new(g_direct_hash, g_direct_equal);
+       dw->second_list_cache = g_hash_table_new(g_direct_hash, g_direct_equal);
+
+       for (GList *i = dw->list; i != NULL; i = i->next)
+               {
+                       DupeItem *di = i->data;
+
+                       g_hash_table_add(dw->list_cache, di->fd);
+               }
+
+       for (GList *i = dw->second_list; i != NULL; i = i->next)
+               {
+                       DupeItem *di = i->data;
+
+                       g_hash_table_add(dw->second_list_cache, di->fd);
+               }
+}
+
+static void dupe_destroy_list_cache(DupeWindow *dw)
+{
+       g_hash_table_destroy(dw->list_cache);
+       g_hash_table_destroy(dw->second_list_cache);
+}
+
+/**
+ * @brief Return true if the fd was not in the cache
+ * @param dw 
+ * @param fd 
+ * @returns 
+ * 
+ * 
+ */
+static gboolean dupe_insert_in_list_cache(DupeWindow *dw, FileData *fd)
+{
+       GHashTable *table =
+               dw->second_drop ? dw->second_list_cache : dw->list_cache;
+       /* We do this as a lookup + add as we don't want to overwrite
+          items as that would leak the old value. */
+       if (g_hash_table_lookup(table, fd) != NULL)
+               return FALSE;
+       return g_hash_table_add(table, fd);
+}
+
 void dupe_window_add_collection(DupeWindow *dw, CollectionData *collection)
 {
        CollectInfo *info;
@@ -1788,7 +2958,7 @@ void dupe_window_add_collection(DupeWindow *dw, CollectionData *collection)
        dupe_check_start(dw);
 }
 
-void dupe_window_add_files(DupeWindow *dw, GList *list, gint recurse)
+void dupe_window_add_files(DupeWindow *dw, GList *list, gboolean recurse)
 {
        GList *work;
 
@@ -1797,11 +2967,40 @@ void dupe_window_add_files(DupeWindow *dw, GList *list, gint recurse)
                {
                FileData *fd = work->data;
                work = work->next;
+               if (isdir(fd->path) && !recurse)
+                       {
+                       GList *f, *d;
 
-               dupe_files_add(dw, NULL, NULL, fd, recurse);
+                       if (filelist_read(fd, &f, &d))
+                               {
+                               GList *work_file;
+                               work_file = f;
+
+                               while (work_file)
+                                       {
+                                       /* Add only the files, ignore the dirs when no recurse */
+                                       dw->add_files_queue = g_list_prepend(dw->add_files_queue, work_file->data);
+                                       work_file = work_file->next;
+                                       }
+                               g_list_free(f);
+                               g_list_free(d);
+                               }
+                       }
+               else
+                       {
+                       dw->add_files_queue = g_list_prepend(dw->add_files_queue, fd);
+                       }
                }
+       if (dw->add_files_queue_id == 0)
+               {
+               gtk_progress_bar_pulse(GTK_PROGRESS_BAR(dw->extra_label));
+               gtk_progress_bar_set_pulse_step(GTK_PROGRESS_BAR(dw->extra_label), DUPE_PROGRESS_PULSE_STEP);
+               gtk_progress_bar_set_text(GTK_PROGRESS_BAR(dw->extra_label), _("Loading file list"));
 
-       dupe_check_start(dw);
+               dupe_init_list_cache(dw);
+               dw->add_files_queue_id = g_idle_add(dupe_files_add_queue_cb, dw);
+               gtk_widget_set_sensitive(dw->controls_box, FALSE);
+               }
 }
 
 static void dupe_item_update(DupeWindow *dw, DupeItem *di)
@@ -1973,6 +3172,7 @@ static void dupe_window_recompare(DupeWindow *dw)
 
        dupe_match_reset_list(dw->list);
        dupe_match_reset_list(dw->second_list);
+       dw->set_count = 0;
 
        dupe_check_start(dw);
 }
@@ -2004,7 +3204,7 @@ static void dupe_menu_view(DupeWindow *dw, DupeItem *di, GtkWidget *listview, gi
                        }
                else
                        {
-                       layout_image_set_fd(NULL, di->fd);
+                       layout_set_fd(NULL, di->fd);
                        }
                }
 }
@@ -2031,7 +3231,7 @@ static void dupe_window_remove_selection(DupeWindow *dw, GtkWidget *listview)
                if (di) list = g_list_prepend(list, di);
                work = work->next;
                }
-       g_list_foreach(slist, (GFunc)gtk_tree_path_free, NULL);
+       g_list_foreach(slist, (GFunc)tree_path_free_wrapper, NULL);
        g_list_free(slist);
 
        dw->color_frozen = TRUE;
@@ -2053,13 +3253,7 @@ static void dupe_window_remove_selection(DupeWindow *dw, GtkWidget *listview)
 
 static void dupe_window_edit_selected(DupeWindow *dw, const gchar *key)
 {
-       GList *list;
-
-       list = dupe_listview_get_selection(dw, dw->listview);
-
-       file_util_start_editor_from_filelist(key, list, dw->window);
-
-       filelist_free(list);
+       file_util_start_editor_from_filelist(key, dupe_listview_get_selection(dw, dw->listview), NULL, dw->window);
 }
 
 static void dupe_window_collection_from_selection(DupeWindow *dw)
@@ -2109,6 +3303,7 @@ static void dupe_menu_select_all_cb(GtkWidget *widget, gpointer data)
        DupeWindow *dw = data;
        GtkTreeSelection *selection;
 
+       options->duplicates_select_type = DUPE_SELECT_NONE;
        selection = gtk_tree_view_get_selection(GTK_TREE_VIEW(dw->listview));
        gtk_tree_selection_select_all(selection);
 }
@@ -2118,6 +3313,7 @@ static void dupe_menu_select_none_cb(GtkWidget *widget, gpointer data)
        DupeWindow *dw = data;
        GtkTreeSelection *selection;
 
+       options->duplicates_select_type = DUPE_SELECT_NONE;
        selection = gtk_tree_view_get_selection(GTK_TREE_VIEW(dw->listview));
        gtk_tree_selection_unselect_all(selection);
 }
@@ -2126,14 +3322,16 @@ static void dupe_menu_select_dupes_set1_cb(GtkWidget *widget, gpointer data)
 {
        DupeWindow *dw = data;
 
-       dupe_listview_select_dupes(dw, TRUE);
+       options->duplicates_select_type = DUPE_SELECT_GROUP1;
+       dupe_listview_select_dupes(dw, DUPE_SELECT_GROUP1);
 }
 
 static void dupe_menu_select_dupes_set2_cb(GtkWidget *widget, gpointer data)
 {
        DupeWindow *dw = data;
 
-       dupe_listview_select_dupes(dw, FALSE);
+       options->duplicates_select_type = DUPE_SELECT_GROUP2;
+       dupe_listview_select_dupes(dw, DUPE_SELECT_GROUP2);
 }
 
 static void dupe_menu_edit_cb(GtkWidget *widget, gpointer data)
@@ -2147,20 +3345,6 @@ static void dupe_menu_edit_cb(GtkWidget *widget, gpointer data)
        dupe_window_edit_selected(dw, key);
 }
 
-static void dupe_menu_info_cb(GtkWidget *widget, gpointer data)
-{
-       DupeWindow *dw = data;
-
-       info_window_new(NULL, dupe_listview_get_selection(dw, dw->listview), NULL);
-}
-
-static void dupe_menu_collection_cb(GtkWidget *widget, gpointer data)
-{
-       DupeWindow *dw = data;
-
-       dupe_window_collection_from_selection(dw);
-}
-
 static void dupe_menu_print_cb(GtkWidget *widget, gpointer data)
 {
        DupeWindow *dw = data;
@@ -2198,14 +3382,30 @@ static void dupe_menu_delete_cb(GtkWidget *widget, gpointer data)
 {
        DupeWindow *dw = data;
 
-       file_util_delete(NULL, dupe_listview_get_selection(dw, dw->listview), dw->window);
+       options->file_ops.safe_delete_enable = FALSE;
+       file_util_delete_notify_done(NULL, dupe_listview_get_selection(dw, dw->listview), dw->window, delete_finished_cb, dw);
+}
+
+static void dupe_menu_move_to_trash_cb(GtkWidget *widget, gpointer data)
+{
+       DupeWindow *dw = data;
+
+       options->file_ops.safe_delete_enable = TRUE;
+       file_util_delete_notify_done(NULL, dupe_listview_get_selection(dw, dw->listview), dw->window, delete_finished_cb, dw);
 }
 
 static void dupe_menu_copy_path_cb(GtkWidget *widget, gpointer data)
 {
        DupeWindow *dw = data;
 
-       file_util_copy_path_list_to_clipboard(dupe_listview_get_selection(dw, dw->listview));
+       file_util_copy_path_list_to_clipboard(dupe_listview_get_selection(dw, dw->listview), TRUE);
+}
+
+static void dupe_menu_copy_path_unquoted_cb(GtkWidget *widget, gpointer data)
+{
+       DupeWindow *dw = data;
+
+       file_util_copy_path_list_to_clipboard(dupe_listview_get_selection(dw, dw->listview), FALSE);
 }
 
 static void dupe_menu_remove_cb(GtkWidget *widget, gpointer data)
@@ -2229,15 +3429,59 @@ static void dupe_menu_close_cb(GtkWidget *widget, gpointer data)
        dupe_window_close(dw);
 }
 
+static void dupe_menu_popup_destroy_cb(GtkWidget *widget, gpointer data)
+{
+       GList *editmenu_fd_list = data;
+
+       filelist_free(editmenu_fd_list);
+}
+
+static GList *dupe_window_get_fd_list(DupeWindow *dw)
+{
+       GList *list;
+
+       if (gtk_widget_has_focus(dw->second_listview))
+               {
+               list = dupe_listview_get_selection(dw, dw->second_listview);
+               }
+       else
+               {
+               list = dupe_listview_get_selection(dw, dw->listview);
+               }
+
+       return list;
+}
+
+/**
+ * @brief Add file selection list to a collection
+ * @param[in] widget 
+ * @param[in] data Index to the collection list menu item selected, or -1 for new collection
+ * 
+ * 
+ */
+static void dupe_pop_menu_collections_cb(GtkWidget *widget, gpointer data)
+{
+       DupeWindow *dw;
+       GList *selection_list;
+
+       dw = submenu_item_get_data(widget);
+       selection_list = dupe_listview_get_selection(dw, dw->listview);
+       pop_menu_collections(selection_list, data);
+
+       filelist_free(selection_list);
+}
+
 static GtkWidget *dupe_menu_popup_main(DupeWindow *dw, DupeItem *di)
 {
        GtkWidget *menu;
        GtkWidget *item;
        gint on_row;
+       GList *editmenu_fd_list;
 
        on_row = (di != NULL);
 
        menu = popup_menu_short_lived();
+
        menu_item_add_sensitive(menu, _("_View"), on_row,
                                G_CALLBACK(dupe_menu_view_cb), dw);
        menu_item_add_stock_sensitive(menu, _("View in _new window"), GTK_STOCK_NEW, on_row,
@@ -2252,12 +3496,21 @@ static GtkWidget *dupe_menu_popup_main(DupeWindow *dw, DupeItem *di)
        menu_item_add_sensitive(menu, _("Select group _2 duplicates"), (dw->dupes != NULL),
                                G_CALLBACK(dupe_menu_select_dupes_set2_cb), dw);
        menu_item_add_divider(menu);
-       submenu_add_edit(menu, &item, G_CALLBACK(dupe_menu_edit_cb), dw);
+
+       submenu_add_export(menu, &item, G_CALLBACK(dupe_pop_menu_export_cb), dw);
+       gtk_widget_set_sensitive(item, on_row);
+       menu_item_add_divider(menu);
+
+       editmenu_fd_list = dupe_window_get_fd_list(dw);
+       g_signal_connect(G_OBJECT(menu), "destroy",
+                        G_CALLBACK(dupe_menu_popup_destroy_cb), editmenu_fd_list);
+       submenu_add_edit(menu, &item, G_CALLBACK(dupe_menu_edit_cb), dw, editmenu_fd_list);
        if (!on_row) gtk_widget_set_sensitive(item, FALSE);
-       menu_item_add_stock_sensitive(menu, _("_Properties"), GTK_STOCK_PROPERTIES, on_row,
-                               G_CALLBACK(dupe_menu_info_cb), dw);
-       menu_item_add_stock_sensitive(menu, _("Add to new collection"), GTK_STOCK_INDEX, on_row,
-                               G_CALLBACK(dupe_menu_collection_cb), dw);
+
+       submenu_add_collections(menu, &item,
+                                                               G_CALLBACK(dupe_pop_menu_collections_cb), dw);
+       gtk_widget_set_sensitive(item, on_row);
+
        menu_item_add_stock_sensitive(menu, _("Print..."), GTK_STOCK_PRINT, on_row,
                                G_CALLBACK(dupe_menu_print_cb), dw);
        menu_item_add_divider(menu);
@@ -2267,11 +3520,21 @@ static GtkWidget *dupe_menu_popup_main(DupeWindow *dw, DupeItem *di)
                                G_CALLBACK(dupe_menu_move_cb), dw);
        menu_item_add_sensitive(menu, _("_Rename..."), on_row,
                                G_CALLBACK(dupe_menu_rename_cb), dw);
-       menu_item_add_stock_sensitive(menu, _("_Delete..."), GTK_STOCK_DELETE, on_row,
+       menu_item_add_sensitive(menu, _("_Copy path"), on_row,
+                               G_CALLBACK(dupe_menu_copy_path_cb), dw);
+       menu_item_add_sensitive(menu, _("_Copy path unquoted"), on_row,
+                               G_CALLBACK(dupe_menu_copy_path_unquoted_cb), dw);
+
+       menu_item_add_divider(menu);
+       menu_item_add_stock_sensitive(menu,
+                               options->file_ops.confirm_move_to_trash ? _("Move to Trash...") :
+                                       _("Move to Trash"), PIXBUF_INLINE_ICON_TRASH, on_row,
+                               G_CALLBACK(dupe_menu_move_to_trash_cb), dw);
+       menu_item_add_stock_sensitive(menu,
+                               options->file_ops.confirm_delete ? _("_Delete...") :
+                                       _("_Delete"), GTK_STOCK_DELETE, on_row,
                                G_CALLBACK(dupe_menu_delete_cb), dw);
-       if (options->show_copy_path)
-               menu_item_add_sensitive(menu, _("_Copy path"), on_row,
-                                       G_CALLBACK(dupe_menu_copy_path_cb), dw);
+
        menu_item_add_divider(menu);
        menu_item_add_stock_sensitive(menu, _("Rem_ove"), GTK_STOCK_REMOVE, on_row,
                                G_CALLBACK(dupe_menu_remove_cb), dw);
@@ -2284,7 +3547,7 @@ static GtkWidget *dupe_menu_popup_main(DupeWindow *dw, DupeItem *di)
        return menu;
 }
 
-static gint dupe_listview_press_cb(GtkWidget *widget, GdkEventButton *bevent, gpointer data)
+static gboolean dupe_listview_press_cb(GtkWidget *widget, GdkEventButton *bevent, gpointer data)
 {
        DupeWindow *dw = data;
        GtkTreeModel *store;
@@ -2367,7 +3630,7 @@ static gint dupe_listview_press_cb(GtkWidget *widget, GdkEventButton *bevent, gp
        return FALSE;
 }
 
-static gint dupe_listview_release_cb(GtkWidget *widget, GdkEventButton *bevent, gpointer data)
+static gboolean dupe_listview_release_cb(GtkWidget *widget, GdkEventButton *bevent, gpointer data)
 {
        DupeWindow *dw = data;
        GtkTreeModel *store;
@@ -2546,11 +3809,8 @@ static void dupe_second_menu_clear_cb(GtkWidget *widget, gpointer data)
 static GtkWidget *dupe_menu_popup_second(DupeWindow *dw, DupeItem *di)
 {
        GtkWidget *menu;
-       gint notempty;
-       gint on_row;
-
-       on_row = (di != NULL);
-       notempty = (dw->second_list != NULL);
+       gboolean notempty = (dw->second_list != NULL);
+       gboolean on_row = (di != NULL);
 
        menu = popup_menu_short_lived();
        menu_item_add_sensitive(menu, _("_View"), on_row,
@@ -2578,7 +3838,7 @@ static void dupe_second_set_toggle_cb(GtkWidget *widget, gpointer data)
 {
        DupeWindow *dw = data;
 
-       dw->second_set = GTK_TOGGLE_BUTTON(widget)->active;
+       dw->second_set = gtk_toggle_button_get_active(GTK_TOGGLE_BUTTON(widget));
 
        if (dw->second_set)
                {
@@ -2596,6 +3856,15 @@ static void dupe_second_set_toggle_cb(GtkWidget *widget, gpointer data)
        dupe_window_recompare(dw);
 }
 
+static void dupe_sort_totals_toggle_cb(GtkWidget *widget, gpointer data)
+{
+       DupeWindow *dw = data;
+
+       options->sort_totals = gtk_toggle_button_get_active(GTK_TOGGLE_BUTTON(widget));
+       dupe_window_recompare(dw);
+
+}
+
 /*
  *-------------------------------------------------------------------
  * match type menu
@@ -2607,6 +3876,8 @@ enum {
        DUPE_MENU_COLUMN_MASK
 };
 
+static void dupe_listview_show_rank(GtkWidget *listview, gboolean rank);
+
 static void dupe_menu_type_cb(GtkWidget *combo, gpointer data)
 {
        DupeWindow *dw = data;
@@ -2617,6 +3888,16 @@ static void dupe_menu_type_cb(GtkWidget *combo, gpointer data)
        if (!gtk_combo_box_get_active_iter(GTK_COMBO_BOX(combo), &iter)) return;
        gtk_tree_model_get(store, &iter, DUPE_MENU_COLUMN_MASK, &dw->match_mask, -1);
 
+       options->duplicates_match = dw->match_mask;
+
+       if (dw->match_mask & (DUPE_MATCH_SIM_HIGH | DUPE_MATCH_SIM_MED | DUPE_MATCH_SIM_LOW | DUPE_MATCH_SIM_CUSTOM))
+               {
+               dupe_listview_show_rank(dw->listview, TRUE);
+               }
+       else
+               {
+               dupe_listview_show_rank(dw->listview, FALSE);
+               }
        dupe_window_recompare(dw);
 }
 
@@ -2652,10 +3933,13 @@ static void dupe_menu_setup(DupeWindow *dw)
        dupe_menu_add_item(store, _("Dimensions"), DUPE_MATCH_DIM, dw);
        dupe_menu_add_item(store, _("Checksum"), DUPE_MATCH_SUM, dw);
        dupe_menu_add_item(store, _("Path"), DUPE_MATCH_PATH, dw);
-       dupe_menu_add_item(store, _("Similarity (high)"), DUPE_MATCH_SIM_HIGH, dw);
-       dupe_menu_add_item(store, _("Similarity"), DUPE_MATCH_SIM_MED, dw);
-       dupe_menu_add_item(store, _("Similarity (low)"), DUPE_MATCH_SIM_LOW, dw);
+       dupe_menu_add_item(store, _("Similarity (high - 95)"), DUPE_MATCH_SIM_HIGH, dw);
+       dupe_menu_add_item(store, _("Similarity (med. - 90)"), DUPE_MATCH_SIM_MED, dw);
+       dupe_menu_add_item(store, _("Similarity (low - 85)"), DUPE_MATCH_SIM_LOW, dw);
        dupe_menu_add_item(store, _("Similarity (custom)"), DUPE_MATCH_SIM_CUSTOM, dw);
+       dupe_menu_add_item(store, _("Name ≠ content"), DUPE_MATCH_NAME_CONTENT, dw);
+       dupe_menu_add_item(store, _("Name case-insensitive ≠ content"), DUPE_MATCH_NAME_CI_CONTENT, dw);
+       dupe_menu_add_item(store, _("Show all"), DUPE_MATCH_ALL, dw);
 
        g_signal_connect(G_OBJECT(dw->combo), "changed",
                         G_CALLBACK(dupe_menu_type_cb), dw);
@@ -2715,7 +3999,7 @@ static void dupe_listview_color_cb(GtkTreeViewColumn *tree_column, GtkCellRender
                     "cell-background-set", set, NULL);
 }
 
-static void dupe_listview_add_column(DupeWindow *dw, GtkWidget *listview, gint n, const gchar *title, gint image, gint right_justify)
+static void dupe_listview_add_column(DupeWindow *dw, GtkWidget *listview, gint n, const gchar *title, gboolean image, gboolean right_justify)
 {
        GtkTreeViewColumn *column;
        GtkCellRenderer *renderer;
@@ -2723,6 +4007,7 @@ static void dupe_listview_add_column(DupeWindow *dw, GtkWidget *listview, gint n
        column = gtk_tree_view_column_new();
        gtk_tree_view_column_set_title(column, title);
        gtk_tree_view_column_set_min_width(column, 4);
+       gtk_tree_view_column_set_sort_column_id(column, n);
 
        if (n != DUPE_COLUMN_RANK &&
            n != DUPE_COLUMN_THUMB)
@@ -2759,7 +4044,7 @@ static void dupe_listview_add_column(DupeWindow *dw, GtkWidget *listview, gint n
        gtk_tree_view_append_column(GTK_TREE_VIEW(listview), column);
 }
 
-static void dupe_listview_set_height(GtkWidget *listview, gint thumb)
+static void dupe_listview_set_height(GtkWidget *listview, gboolean thumb)
 {
        GtkTreeViewColumn *column;
        GtkCellRenderer *cell;
@@ -2769,8 +4054,9 @@ static void dupe_listview_set_height(GtkWidget *listview, gint thumb)
        if (!column) return;
 
        gtk_tree_view_column_set_fixed_width(column, (thumb) ? options->thumbnails.max_width : 4);
+       gtk_tree_view_column_set_visible(column, thumb);
 
-       list = gtk_tree_view_column_get_cell_renderers(column);
+       list = gtk_cell_layout_get_cells(GTK_CELL_LAYOUT(column));
        if (!list) return;
        cell = list->data;
        g_list_free(list);
@@ -2779,6 +4065,15 @@ static void dupe_listview_set_height(GtkWidget *listview, gint thumb)
        gtk_tree_view_columns_autosize(GTK_TREE_VIEW(listview));
 }
 
+static void dupe_listview_show_rank(GtkWidget *listview, gboolean rank)
+{
+       GtkTreeViewColumn *column;
+
+       column = gtk_tree_view_get_column(GTK_TREE_VIEW(listview), DUPE_COLUMN_RANK - 1);
+       if (!column) return;
+
+       gtk_tree_view_column_set_visible(column, rank);
+}
 
 /*
  *-------------------------------------------------------------------
@@ -2790,7 +4085,8 @@ static void dupe_window_show_thumb_cb(GtkWidget *widget, gpointer data)
 {
        DupeWindow *dw = data;
 
-       dw->show_thumbs = GTK_TOGGLE_BUTTON(widget)->active;
+       dw->show_thumbs = gtk_toggle_button_get_active(GTK_TOGGLE_BUTTON(widget));
+       options->duplicates_thumbnails = dw->show_thumbs;
 
        if (dw->show_thumbs)
                {
@@ -2800,7 +4096,7 @@ static void dupe_window_show_thumb_cb(GtkWidget *widget, gpointer data)
                {
                GtkTreeModel *store;
                GtkTreeIter iter;
-               gint valid;
+               gboolean valid;
 
                thumb_loader_free(dw->thumb_loader);
                dw->thumb_loader = NULL;
@@ -2819,6 +4115,41 @@ static void dupe_window_show_thumb_cb(GtkWidget *widget, gpointer data)
        dupe_listview_set_height(dw->listview, dw->show_thumbs);
 }
 
+static void dupe_window_rotation_invariant_cb(GtkWidget *widget, gpointer data)
+{
+       DupeWindow *dw = data;
+
+       options->rot_invariant_sim = gtk_toggle_button_get_active(GTK_TOGGLE_BUTTON(widget));
+       dupe_window_recompare(dw);
+}
+
+static void dupe_window_custom_threshold_cb(GtkWidget *widget, gpointer data)
+{
+       DupeWindow *dw = data;
+       DupeMatchType match_type;
+       GtkTreeModel *store;
+       gboolean valid;
+       GtkTreeIter iter;
+
+       options->duplicates_similarity_threshold = gtk_spin_button_get_value_as_int(GTK_SPIN_BUTTON(widget));
+       dw->match_mask = DUPE_MATCH_SIM_CUSTOM;
+
+       store = gtk_combo_box_get_model(GTK_COMBO_BOX(dw->combo));
+       valid = gtk_tree_model_get_iter_first(store, &iter);
+       while (valid)
+               {
+               gtk_tree_model_get(store, &iter, DUPE_MENU_COLUMN_MASK, &match_type, -1);
+               if (match_type == DUPE_MATCH_SIM_CUSTOM)
+                       {
+                       break;
+                       }
+               valid = gtk_tree_model_iter_next(store, &iter);
+               }
+
+       gtk_combo_box_set_active_iter(GTK_COMBO_BOX(dw->combo), &iter);
+       dupe_window_recompare(dw);
+}
+
 static void dupe_popup_menu_pos_cb(GtkMenu *menu, gint *x, gint *y, gboolean *push_in, gpointer data)
 {
        GtkWidget *view = data;
@@ -2846,18 +4177,18 @@ static void dupe_popup_menu_pos_cb(GtkMenu *menu, gint *x, gint *y, gboolean *pu
        *y = cy;
 }
 
-static gint dupe_window_keypress_cb(GtkWidget *widget, GdkEventKey *event, gpointer data)
+static gboolean dupe_window_keypress_cb(GtkWidget *widget, GdkEventKey *event, gpointer data)
 {
        DupeWindow *dw = data;
-       gint stop_signal = FALSE;
-       gint on_second;
+       gboolean stop_signal = FALSE;
+       gboolean on_second;
        GtkWidget *listview;
        GtkTreeModel *store;
        GtkTreeSelection *selection;
        GList *slist;
        DupeItem *di = NULL;
 
-       on_second = GTK_WIDGET_HAS_FOCUS(dw->second_listview);
+       on_second = gtk_widget_has_focus(dw->second_listview);
 
        if (on_second)
                {
@@ -2883,47 +4214,26 @@ static gint dupe_window_keypress_cb(GtkWidget *widget, GdkEventKey *event, gpoin
                gtk_tree_model_get_iter(store, &iter, tpath);
                gtk_tree_model_get(store, &iter, DUPE_COLUMN_POINTER, &di, -1);
                }
-       g_list_foreach(slist, (GFunc)gtk_tree_path_free, NULL);
+       g_list_foreach(slist, (GFunc)tree_path_free_wrapper, NULL);
        g_list_free(slist);
 
        if (event->state & GDK_CONTROL_MASK)
                {
-               gint edit_val = -1;
-
                if (!on_second)
                        {
                        stop_signal = TRUE;
                        switch (event->keyval)
                                {
                                case '1':
-                                       edit_val = 0;
-                                       break;
                                case '2':
-                                       edit_val = 1;
-                                       break;
                                case '3':
-                                       edit_val = 2;
-                                       break;
                                case '4':
-                                       edit_val = 3;
-                                       break;
                                case '5':
-                                       edit_val = 4;
-                                       break;
                                case '6':
-                                       edit_val = 5;
-                                       break;
                                case '7':
-                                       edit_val = 6;
-                                       break;
                                case '8':
-                                       edit_val = 7;
-                                       break;
                                case '9':
-                                       edit_val = 8;
-                                       break;
                                case '0':
-                                       edit_val = 9;
                                        break;
                                case 'C': case 'c':
                                        file_util_copy(NULL, dupe_listview_get_selection(dw, listview),
@@ -2937,11 +4247,9 @@ static gint dupe_window_keypress_cb(GtkWidget *widget, GdkEventKey *event, gpoin
                                        file_util_rename(NULL, dupe_listview_get_selection(dw, listview), dw->window);
                                        break;
                                case 'D': case 'd':
+                                       options->file_ops.safe_delete_enable = TRUE;
                                        file_util_delete(NULL, dupe_listview_get_selection(dw, listview), dw->window);
                                        break;
-                               case 'P': case 'p':
-                                       info_window_new(NULL, dupe_listview_get_selection(dw, listview), NULL);
-                                       break;
                                default:
                                        stop_signal = FALSE;
                                        break;
@@ -2963,7 +4271,7 @@ static gint dupe_window_keypress_cb(GtkWidget *widget, GdkEventKey *event, gpoin
                                                gtk_tree_selection_select_all(selection);
                                                }
                                        break;
-                               case GDK_Delete: case GDK_KP_Delete:
+                               case GDK_KEY_Delete: case GDK_KEY_KP_Delete:
                                        if (on_second)
                                                {
                                                dupe_second_clear(dw);
@@ -2989,25 +4297,19 @@ static gint dupe_window_keypress_cb(GtkWidget *widget, GdkEventKey *event, gpoin
                                        break;
                                }
                        }
-#if 0
-               if (edit_val >= 0)
-                       {
-                       dupe_window_edit_selected(dw, edit_val);
-                       }
-#endif
                }
        else
                {
                stop_signal = TRUE;
                switch (event->keyval)
                        {
-                       case GDK_Return: case GDK_KP_Enter:
+                       case GDK_KEY_Return: case GDK_KEY_KP_Enter:
                                dupe_menu_view(dw, di, listview, FALSE);
                                break;
                        case 'V': case 'v':
                                dupe_menu_view(dw, di, listview, TRUE);
                                break;
-                       case GDK_Delete: case GDK_KP_Delete:
+                       case GDK_KEY_Delete: case GDK_KEY_KP_Delete:
                                dupe_window_remove_selection(dw, listview);
                                break;
                        case 'C': case 'c':
@@ -3016,14 +4318,20 @@ static gint dupe_window_keypress_cb(GtkWidget *widget, GdkEventKey *event, gpoin
                                        dupe_window_collection_from_selection(dw);
                                        }
                                break;
+                       case '0':
+                               options->duplicates_select_type = DUPE_SELECT_NONE;
+                               dupe_listview_select_dupes(dw, DUPE_SELECT_NONE);
+                               break;
                        case '1':
-                               dupe_listview_select_dupes(dw, TRUE);
+                               options->duplicates_select_type = DUPE_SELECT_GROUP1;
+                               dupe_listview_select_dupes(dw, DUPE_SELECT_GROUP1);
                                break;
                        case '2':
-                               dupe_listview_select_dupes(dw, FALSE);
+                               options->duplicates_select_type = DUPE_SELECT_GROUP2;
+                               dupe_listview_select_dupes(dw, DUPE_SELECT_GROUP2);
                                break;
-                       case GDK_Menu:
-                       case GDK_F10:
+                       case GDK_KEY_Menu:
+                       case GDK_KEY_F10:
                                if (!on_second)
                                        {
                                        GtkWidget *menu;
@@ -3046,6 +4354,11 @@ static gint dupe_window_keypress_cb(GtkWidget *widget, GdkEventKey *event, gpoin
                                break;
                        }
                }
+       if (!stop_signal && is_help_key(event))
+               {
+               help_window_show("GuideImageSearchFindingDuplicates.html");
+               stop_signal = TRUE;
+               }
 
        return stop_signal;
 }
@@ -3066,6 +4379,7 @@ void dupe_window_clear(DupeWindow *dw)
 
        dupe_list_free(dw->list);
        dw->list = NULL;
+       dw->set_count = 0;
 
        dupe_match_reset_list(dw->second_list);
 
@@ -3073,10 +4387,27 @@ void dupe_window_clear(DupeWindow *dw)
        dupe_window_update_progress(dw, NULL, 0.0, FALSE);
 }
 
+static void dupe_window_get_geometry(DupeWindow *dw)
+{
+       GdkWindow *window;
+       LayoutWindow *lw = NULL;
+
+       layout_valid(&lw);
+
+       if (!dw || !lw) return;
+
+       window = gtk_widget_get_window(dw->window);
+       gdk_window_get_position(window, &lw->options.dupe_window.x, &lw->options.dupe_window.y);
+       lw->options.dupe_window.w = gdk_window_get_width(window);
+       lw->options.dupe_window.h = gdk_window_get_height(window);
+}
+
 void dupe_window_close(DupeWindow *dw)
 {
        dupe_check_stop(dw);
 
+       dupe_window_get_geometry(dw);
+
        dupe_window_list = g_list_remove(dupe_window_list, dw);
        gtk_widget_destroy(dw->window);
 
@@ -3087,9 +4418,20 @@ void dupe_window_close(DupeWindow *dw)
 
        file_data_unregister_notify_func(dupe_notify_cb, dw);
 
+       g_thread_pool_free(dw->dupe_comparison_thread_pool, TRUE, TRUE);
+
        g_free(dw);
 }
 
+static gint dupe_window_close_cb(GtkWidget *widget, gpointer data)
+{
+       DupeWindow *dw = data;
+
+       dupe_window_close(dw);
+
+       return TRUE;
+}
+
 static gint dupe_window_delete(GtkWidget *widget, GdkEvent *event, gpointer data)
 {
        DupeWindow *dw = data;
@@ -3098,32 +4440,155 @@ static gint dupe_window_delete(GtkWidget *widget, GdkEvent *event, gpointer data
        return TRUE;
 }
 
+static void dupe_help_cb(GtkAction *action, gpointer data)
+{
+       help_window_show("GuideImageSearchFindingDuplicates.html");
+}
+
+static gint default_sort_cb(GtkTreeModel *model, GtkTreeIter *a, GtkTreeIter *b, gpointer data)
+{
+       return 0;
+}
+
+static gint column_sort_cb(GtkTreeModel *model, GtkTreeIter *a, GtkTreeIter *b, gpointer data)
+{
+       GtkTreeSortable *sortable = data;
+       gint ret = 0;
+       gchar *rank_str_a, *rank_str_b;
+       gint rank_int_a;
+       gint rank_int_b;
+       gint group_a;
+       gint group_b;
+       gint sort_column_id;
+       GtkSortType sort_order;
+       DupeItem *di_a;
+       DupeItem *di_b;
+
+       gtk_tree_sortable_get_sort_column_id(sortable, &sort_column_id, &sort_order);
+
+       gtk_tree_model_get(model, a, DUPE_COLUMN_RANK, &rank_str_a, DUPE_COLUMN_SET, &group_a, DUPE_COLUMN_POINTER, &di_a, -1);
+
+       gtk_tree_model_get(model, b, DUPE_COLUMN_RANK, &rank_str_b, DUPE_COLUMN_SET, &group_b, DUPE_COLUMN_POINTER, &di_b, -1);
+
+       if (group_a == group_b)
+               {
+               switch (sort_column_id)
+                       {
+                       case DUPE_COLUMN_NAME:
+                               ret = utf8_compare(di_a->fd->name, di_b->fd->name, TRUE);
+                               break;
+                       case DUPE_COLUMN_SIZE:
+                               if (di_a->fd->size == di_b->fd->size)
+                                       {
+                                       ret = 0;
+                                       }
+                               else
+                                       {
+                                       ret = (di_a->fd->size > di_b->fd->size) ? 1 : -1;
+                                       }
+                               break;
+                       case DUPE_COLUMN_DATE:
+                               if (di_a->fd->date == di_b->fd->date)
+                                       {
+                                       ret = 0;
+                                       }
+                               else
+                                       {
+                                       ret = (di_a->fd->date > di_b->fd->date) ? 1 : -1;
+                                       }
+                               break;
+                       case DUPE_COLUMN_DIMENSIONS:
+                               if ((di_a->width == di_b->width) && (di_a->height == di_b->height))
+                                       {
+                                       ret = 0;
+                                       }
+                               else
+                                       {
+                                       ret = ((di_a->width * di_a->height) > (di_b->width * di_b->height)) ? 1 : -1;
+                                       }
+                               break;
+                       case DUPE_COLUMN_RANK:
+                               rank_int_a = atoi(rank_str_a);
+                               rank_int_b = atoi(rank_str_b);
+                               if (rank_int_a == 0) rank_int_a = 101;
+                               if (rank_int_b == 0) rank_int_b = 101;
+
+                               if (rank_int_a == rank_int_b)
+                                       {
+                                       ret = 0;
+                                       }
+                               else
+                                       {
+                                       ret = (rank_int_a > rank_int_b) ? 1 : -1;
+                                       }
+                               break;
+                       case DUPE_COLUMN_PATH:
+                               ret = utf8_compare(di_a->fd->path, di_b->fd->path, TRUE);
+                               break;
+                       }
+               }
+       else if (group_a < group_b)
+               {
+               ret = (sort_order == GTK_SORT_ASCENDING) ? 1 : -1;
+               }
+       else
+               {
+               ret = (sort_order == GTK_SORT_ASCENDING) ? -1 : 1;
+               }
+
+       return ret;
+}
+
+static void column_clicked_cb(GtkWidget *widget,  gpointer data)
+{
+       DupeWindow *dw = data;
+
+       options->duplicates_match = DUPE_SELECT_NONE;
+       dupe_listview_select_dupes(dw, DUPE_SELECT_NONE);
+}
+
 /* collection and files can be NULL */
-DupeWindow *dupe_window_new(DupeMatchType match_mask)
+DupeWindow *dupe_window_new()
 {
        DupeWindow *dw;
        GtkWidget *vbox;
+       GtkWidget *hbox;
        GtkWidget *scrolled;
        GtkWidget *frame;
        GtkWidget *status_box;
+       GtkWidget *controls_box;
+       GtkWidget *button_box;
        GtkWidget *label;
        GtkWidget *button;
        GtkListStore *store;
        GtkTreeSelection *selection;
        GdkGeometry geometry;
+       LayoutWindow *lw = NULL;
 
-       dw = g_new0(DupeWindow, 1);
-
-       dw->list = NULL;
-       dw->dupes = NULL;
-       dw->match_mask = match_mask;
-       dw->show_thumbs = FALSE;
-
-       dw->idle_id = -1;
+       layout_valid(&lw);
 
-       dw->second_set = FALSE;
+       dw = g_new0(DupeWindow, 1);
+       dw->add_files_queue = NULL;
+       dw->add_files_queue_id = 0;
+
+       dw->match_mask = DUPE_MATCH_NAME;
+       if (options->duplicates_match == DUPE_MATCH_NAME) dw->match_mask = DUPE_MATCH_NAME;
+       if (options->duplicates_match == DUPE_MATCH_SIZE) dw->match_mask = DUPE_MATCH_SIZE;
+       if (options->duplicates_match == DUPE_MATCH_DATE) dw->match_mask = DUPE_MATCH_DATE;
+       if (options->duplicates_match == DUPE_MATCH_DIM) dw->match_mask = DUPE_MATCH_DIM;
+       if (options->duplicates_match == DUPE_MATCH_SUM) dw->match_mask = DUPE_MATCH_SUM;
+       if (options->duplicates_match == DUPE_MATCH_PATH) dw->match_mask = DUPE_MATCH_PATH;
+       if (options->duplicates_match == DUPE_MATCH_SIM_HIGH) dw->match_mask = DUPE_MATCH_SIM_HIGH;
+       if (options->duplicates_match == DUPE_MATCH_SIM_MED) dw->match_mask = DUPE_MATCH_SIM_MED;
+       if (options->duplicates_match == DUPE_MATCH_SIM_LOW) dw->match_mask = DUPE_MATCH_SIM_LOW;
+       if (options->duplicates_match == DUPE_MATCH_SIM_CUSTOM) dw->match_mask = DUPE_MATCH_SIM_CUSTOM;
+       if (options->duplicates_match == DUPE_MATCH_NAME_CI) dw->match_mask = DUPE_MATCH_NAME_CI;
+       if (options->duplicates_match == DUPE_MATCH_NAME_CONTENT) dw->match_mask = DUPE_MATCH_NAME_CONTENT;
+       if (options->duplicates_match == DUPE_MATCH_NAME_CI_CONTENT) dw->match_mask = DUPE_MATCH_NAME_CI_CONTENT;
+       if (options->duplicates_match == DUPE_MATCH_ALL) dw->match_mask = DUPE_MATCH_ALL;
 
        dw->window = window_new(GTK_WINDOW_TOPLEVEL, "dupe", NULL, NULL, _("Find duplicates"));
+       DEBUG_NAME(dw->window);
 
        geometry.min_width = DEFAULT_MINIMAL_WINDOW_SIZE;
        geometry.min_height = DEFAULT_MINIMAL_WINDOW_SIZE;
@@ -3132,7 +4597,15 @@ DupeWindow *dupe_window_new(DupeMatchType match_mask)
        gtk_window_set_geometry_hints(GTK_WINDOW(dw->window), NULL, &geometry,
                                      GDK_HINT_MIN_SIZE | GDK_HINT_BASE_SIZE);
 
-       gtk_window_set_default_size(GTK_WINDOW(dw->window), DUPE_DEF_WIDTH, DUPE_DEF_HEIGHT);
+       if (lw && options->save_window_positions)
+               {
+               gtk_window_set_default_size(GTK_WINDOW(dw->window), lw->options.dupe_window.w, lw->options.dupe_window.h);
+               gtk_window_move(GTK_WINDOW(dw->window), lw->options.dupe_window.x, lw->options.dupe_window.y);
+               }
+       else
+               {
+               gtk_window_set_default_size(GTK_WINDOW(dw->window), DUPE_DEF_WIDTH, DUPE_DEF_HEIGHT);
+               }
 
        gtk_window_set_resizable(GTK_WINDOW(dw->window), TRUE);
        gtk_container_set_border_width(GTK_CONTAINER(dw->window), 0);
@@ -3156,24 +4629,41 @@ DupeWindow *dupe_window_new(DupeMatchType match_mask)
        gtk_table_attach_defaults(GTK_TABLE(dw->table), scrolled, 0, 2, 0, 1);
        gtk_widget_show(scrolled);
 
-       store = gtk_list_store_new(9, G_TYPE_POINTER, G_TYPE_STRING, GDK_TYPE_PIXBUF,
-                                  G_TYPE_STRING, G_TYPE_STRING, G_TYPE_STRING,
-                                  G_TYPE_STRING, G_TYPE_STRING, G_TYPE_BOOLEAN);
+       store = gtk_list_store_new(DUPE_COLUMN_COUNT, G_TYPE_POINTER, G_TYPE_STRING, GDK_TYPE_PIXBUF, G_TYPE_STRING, G_TYPE_STRING, G_TYPE_STRING, G_TYPE_STRING, G_TYPE_STRING, G_TYPE_BOOLEAN, G_TYPE_INT, G_TYPE_INT);
        dw->listview = gtk_tree_view_new_with_model(GTK_TREE_MODEL(store));
        g_object_unref(store);
 
+       dw->sortable = GTK_TREE_SORTABLE(store);
+
+       gtk_tree_sortable_set_sort_func(dw->sortable, DUPE_COLUMN_RANK, column_sort_cb, dw->sortable, NULL);
+       gtk_tree_sortable_set_sort_func(dw->sortable, DUPE_COLUMN_SET, default_sort_cb, dw->sortable, NULL);
+       gtk_tree_sortable_set_sort_func(dw->sortable, DUPE_COLUMN_THUMB, default_sort_cb, dw->sortable, NULL);
+       gtk_tree_sortable_set_sort_func(dw->sortable, DUPE_COLUMN_NAME, column_sort_cb, dw->sortable, NULL);
+       gtk_tree_sortable_set_sort_func(dw->sortable, DUPE_COLUMN_SIZE, column_sort_cb, dw->sortable, NULL);
+       gtk_tree_sortable_set_sort_func(dw->sortable, DUPE_COLUMN_DATE, column_sort_cb, dw->sortable, NULL);
+       gtk_tree_sortable_set_sort_func(dw->sortable, DUPE_COLUMN_DIMENSIONS, column_sort_cb, dw->sortable, NULL);
+       gtk_tree_sortable_set_sort_func(dw->sortable, DUPE_COLUMN_PATH, column_sort_cb, dw->sortable, NULL);
+
        selection = gtk_tree_view_get_selection(GTK_TREE_VIEW(dw->listview));
        gtk_tree_selection_set_mode(GTK_TREE_SELECTION(selection), GTK_SELECTION_MULTIPLE);
        gtk_tree_view_set_headers_visible(GTK_TREE_VIEW(dw->listview), TRUE);
        gtk_tree_view_set_enable_search(GTK_TREE_VIEW(dw->listview), FALSE);
 
-       dupe_listview_add_column(dw, dw->listview, DUPE_COLUMN_RANK, "", FALSE, TRUE);
-       dupe_listview_add_column(dw, dw->listview, DUPE_COLUMN_THUMB, "", TRUE, FALSE);
+       dupe_listview_add_column(dw, dw->listview, DUPE_COLUMN_RANK, _("Rank"), FALSE, TRUE);
+       dupe_listview_add_column(dw, dw->listview, DUPE_COLUMN_THUMB, _("Thumb"), TRUE, FALSE);
        dupe_listview_add_column(dw, dw->listview, DUPE_COLUMN_NAME, _("Name"), FALSE, FALSE);
        dupe_listview_add_column(dw, dw->listview, DUPE_COLUMN_SIZE, _("Size"), FALSE, TRUE);
        dupe_listview_add_column(dw, dw->listview, DUPE_COLUMN_DATE, _("Date"), FALSE, TRUE);
        dupe_listview_add_column(dw, dw->listview, DUPE_COLUMN_DIMENSIONS, _("Dimensions"), FALSE, FALSE);
        dupe_listview_add_column(dw, dw->listview, DUPE_COLUMN_PATH, _("Path"), FALSE, FALSE);
+       dupe_listview_add_column(dw, dw->listview, DUPE_COLUMN_SET, _("Set"), FALSE, FALSE);
+
+       g_signal_connect(gtk_tree_view_get_column(GTK_TREE_VIEW(dw->listview), DUPE_COLUMN_RANK - 1), "clicked", (GCallback)column_clicked_cb, dw);
+       g_signal_connect(gtk_tree_view_get_column(GTK_TREE_VIEW(dw->listview), DUPE_COLUMN_NAME - 1), "clicked", (GCallback)column_clicked_cb, dw);
+       g_signal_connect(gtk_tree_view_get_column(GTK_TREE_VIEW(dw->listview), DUPE_COLUMN_SIZE - 1), "clicked", (GCallback)column_clicked_cb, dw);
+       g_signal_connect(gtk_tree_view_get_column(GTK_TREE_VIEW(dw->listview), DUPE_COLUMN_DATE - 1), "clicked", (GCallback)column_clicked_cb, dw);
+       g_signal_connect(gtk_tree_view_get_column(GTK_TREE_VIEW(dw->listview), DUPE_COLUMN_DIMENSIONS - 1), "clicked", (GCallback)column_clicked_cb, dw);
+       g_signal_connect(gtk_tree_view_get_column(GTK_TREE_VIEW(dw->listview), DUPE_COLUMN_PATH - 1), "clicked", (GCallback)column_clicked_cb, dw);
 
        gtk_container_add(GTK_CONTAINER(scrolled), dw->listview);
        gtk_widget_show(dw->listview);
@@ -3216,48 +4706,105 @@ DupeWindow *dupe_window_new(DupeMatchType match_mask)
 
        pref_line(dw->second_vbox, GTK_ORIENTATION_HORIZONTAL);
 
-       status_box = pref_box_new(vbox, FALSE, GTK_ORIENTATION_HORIZONTAL, 0);
+       status_box = gtk_hbox_new(FALSE, 0);
+       gtk_box_pack_start(GTK_BOX(vbox), status_box, FALSE, FALSE, 0);
+       gtk_widget_show(status_box);
 
-       label = gtk_label_new(_("Compare by:"));
-       gtk_box_pack_start(GTK_BOX(status_box), label, FALSE, FALSE, PREF_PAD_SPACE);
-       gtk_widget_show(label);
+       frame = gtk_frame_new(NULL);
+       DEBUG_NAME(frame);
+       gtk_frame_set_shadow_type(GTK_FRAME(frame), GTK_SHADOW_IN);
+       gtk_box_pack_start(GTK_BOX(status_box), frame, TRUE, TRUE, 0);
+       gtk_widget_show(frame);
 
-       dupe_menu_setup(dw);
-       gtk_box_pack_start(GTK_BOX(status_box), dw->combo, FALSE, FALSE, 0);
-       gtk_widget_show(dw->combo);
+       dw->status_label = gtk_label_new("");
+       gtk_container_add(GTK_CONTAINER(frame), dw->status_label);
+       gtk_widget_show(dw->status_label);
+
+       dw->extra_label = gtk_progress_bar_new();
+       gtk_progress_bar_set_fraction(GTK_PROGRESS_BAR(dw->extra_label), 0.0);
+#if GTK_CHECK_VERSION(3,0,0)
+       gtk_progress_bar_set_text(GTK_PROGRESS_BAR(dw->extra_label), "");
+       gtk_progress_bar_set_show_text(GTK_PROGRESS_BAR(dw->extra_label), TRUE);
+#endif
+       gtk_box_pack_start(GTK_BOX(status_box), dw->extra_label, FALSE, FALSE, PREF_PAD_SPACE);
+       gtk_widget_show(dw->extra_label);
+
+       controls_box = pref_box_new(vbox, FALSE, GTK_ORIENTATION_HORIZONTAL, 0);
+       dw->controls_box = controls_box;
 
        dw->button_thumbs = gtk_check_button_new_with_label(_("Thumbnails"));
+       dw->show_thumbs = options->duplicates_thumbnails;
        gtk_toggle_button_set_active(GTK_TOGGLE_BUTTON(dw->button_thumbs), dw->show_thumbs);
        g_signal_connect(G_OBJECT(dw->button_thumbs), "toggled",
                         G_CALLBACK(dupe_window_show_thumb_cb), dw);
-       gtk_box_pack_start(GTK_BOX(status_box), dw->button_thumbs, FALSE, FALSE, PREF_PAD_SPACE);
+       gtk_box_pack_start(GTK_BOX(controls_box), dw->button_thumbs, FALSE, FALSE, PREF_PAD_SPACE);
        gtk_widget_show(dw->button_thumbs);
 
+       label = gtk_label_new(_("Compare by:"));
+       gtk_box_pack_start(GTK_BOX(controls_box), label, FALSE, FALSE, PREF_PAD_SPACE);
+       gtk_widget_show(label);
+
+       dupe_menu_setup(dw);
+       gtk_box_pack_start(GTK_BOX(controls_box), dw->combo, FALSE, FALSE, 0);
+       gtk_widget_show(dw->combo);
+
+       label = gtk_label_new(_("Custom Threshold"));
+       gtk_box_pack_start(GTK_BOX(controls_box), label, FALSE, FALSE, PREF_PAD_SPACE);
+       gtk_widget_show(label);
+       dw->custom_threshold = gtk_spin_button_new_with_range(1, 100, 1);
+       gtk_widget_set_tooltip_text(GTK_WIDGET(dw->custom_threshold), "Custom similarity threshold\n(Use tab key to set value)");
+       gtk_spin_button_set_value(GTK_SPIN_BUTTON(dw->custom_threshold), options->duplicates_similarity_threshold);
+       g_signal_connect(G_OBJECT(dw->custom_threshold), "value_changed", G_CALLBACK(dupe_window_custom_threshold_cb), dw);
+       gtk_box_pack_start(GTK_BOX(controls_box), dw->custom_threshold, FALSE, FALSE, PREF_PAD_SPACE);
+       gtk_widget_show(dw->custom_threshold);
+
+       button = gtk_check_button_new_with_label(_("Sort"));
+       gtk_widget_set_tooltip_text(GTK_WIDGET(button), "Sort by group totals");
+       gtk_toggle_button_set_active(GTK_TOGGLE_BUTTON(button), options->sort_totals);
+       g_signal_connect(G_OBJECT(button), "toggled", G_CALLBACK(dupe_sort_totals_toggle_cb), dw);
+       gtk_box_pack_start(GTK_BOX(controls_box), button, FALSE, FALSE, PREF_PAD_SPACE);
+       gtk_widget_show(button);
+
+       dw->button_rotation_invariant = gtk_check_button_new_with_label(_("Ignore Orientation"));
+       gtk_widget_set_tooltip_text(GTK_WIDGET(dw->button_rotation_invariant), "Ignore image orientation");
+       gtk_toggle_button_set_active(GTK_TOGGLE_BUTTON(dw->button_rotation_invariant), options->rot_invariant_sim);
+       g_signal_connect(G_OBJECT(dw->button_rotation_invariant), "toggled",
+                        G_CALLBACK(dupe_window_rotation_invariant_cb), dw);
+       gtk_box_pack_start(GTK_BOX(controls_box), dw->button_rotation_invariant, FALSE, FALSE, PREF_PAD_SPACE);
+       gtk_widget_show(dw->button_rotation_invariant);
+
        button = gtk_check_button_new_with_label(_("Compare two file sets"));
        gtk_toggle_button_set_active(GTK_TOGGLE_BUTTON(button), dw->second_set);
        g_signal_connect(G_OBJECT(button), "toggled",
                         G_CALLBACK(dupe_second_set_toggle_cb), dw);
-       gtk_box_pack_end(GTK_BOX(status_box), button, FALSE, FALSE, PREF_PAD_SPACE);
+       gtk_box_pack_start(GTK_BOX(controls_box), button, FALSE, FALSE, PREF_PAD_SPACE);
        gtk_widget_show(button);
 
-       status_box = gtk_hbox_new(FALSE, 0);
-       gtk_box_pack_start(GTK_BOX(vbox), status_box, FALSE, FALSE, 0);
-       gtk_widget_show(status_box);
+       button_box = gtk_hbox_new(FALSE, 0);
+       gtk_box_pack_start(GTK_BOX(vbox), button_box, FALSE, FALSE, 0);
+       gtk_widget_show(button_box);
 
-       frame = gtk_frame_new(NULL);
-       gtk_frame_set_shadow_type(GTK_FRAME(frame), GTK_SHADOW_IN);
-       gtk_box_pack_start(GTK_BOX(status_box), frame, TRUE, TRUE, 0);
-       gtk_widget_show(frame);
+       hbox = gtk_hbutton_box_new();
+       gtk_button_box_set_layout(GTK_BUTTON_BOX(hbox), GTK_BUTTONBOX_END);
+       gtk_box_set_spacing(GTK_BOX(hbox), PREF_PAD_SPACE);
+       gtk_box_pack_end(GTK_BOX(button_box), hbox, FALSE, FALSE, 0);
+       gtk_widget_show(hbox);
 
-       dw->status_label = gtk_label_new("");
-       gtk_container_add(GTK_CONTAINER(frame), dw->status_label);
-       gtk_widget_show(dw->status_label);
+       button = pref_button_new(NULL, GTK_STOCK_HELP, NULL, FALSE, G_CALLBACK(dupe_help_cb), NULL);
+       gtk_container_add(GTK_CONTAINER(hbox), button);
+       gtk_widget_set_can_default(button, TRUE);
+       gtk_widget_show(button);
 
-       dw->extra_label = gtk_progress_bar_new();
-       gtk_progress_bar_set_fraction(GTK_PROGRESS_BAR(dw->extra_label), 0.0);
-       gtk_box_pack_end(GTK_BOX(status_box), dw->extra_label, FALSE, FALSE, 0);
-       gtk_widget_show(dw->extra_label);
+       button = pref_button_new(NULL, GTK_STOCK_STOP, NULL, FALSE, G_CALLBACK(dupe_check_stop_cb), dw);
+       gtk_container_add(GTK_CONTAINER(hbox), button);
+       gtk_widget_set_can_default(button, TRUE);
+       gtk_widget_show(button);
 
+       button = pref_button_new(NULL, GTK_STOCK_CLOSE, NULL, FALSE, G_CALLBACK(dupe_window_close_cb), dw);
+       gtk_container_add(GTK_CONTAINER(hbox), button);
+       gtk_widget_set_can_default(button, TRUE);
+       gtk_widget_grab_default(button);
+       gtk_widget_show(button);
        dupe_dnd_init(dw);
 
        /* order is important here, dnd_init should be seeing mouse
@@ -3274,6 +4821,9 @@ DupeWindow *dupe_window_new(DupeMatchType match_mask)
 
        gtk_widget_show(dw->window);
 
+       dupe_listview_set_height(dw->listview, dw->show_thumbs);
+       g_signal_emit_by_name(G_OBJECT(dw->combo), "changed");
+
        dupe_window_update_count(dw, TRUE);
        dupe_window_update_progress(dw, NULL, 0.0, FALSE);
 
@@ -3281,6 +4831,10 @@ DupeWindow *dupe_window_new(DupeMatchType match_mask)
 
        file_data_register_notify_func(dupe_notify_cb, dw, NOTIFY_PRIORITY_MEDIUM);
 
+       g_mutex_init(&dw->thread_count_mutex);
+       g_mutex_init(&dw->search_matches_mutex);
+       dw->dupe_comparison_thread_pool = g_thread_pool_new(dupe_comparison_func, dw, -1, FALSE, NULL);
+
        return dw;
 }
 
@@ -3393,8 +4947,6 @@ static void dupe_dnd_data_set(GtkWidget *widget, GdkDragContext *context,
                              guint time, gpointer data)
 {
        DupeWindow *dw = data;
-       gchar *uri_text;
-       gint length;
        GList *list;
 
        switch (info)
@@ -3403,17 +4955,12 @@ static void dupe_dnd_data_set(GtkWidget *widget, GdkDragContext *context,
                case TARGET_TEXT_PLAIN:
                        list = dupe_listview_get_selection(dw, widget);
                        if (!list) return;
-                       uri_text = uri_text_from_filelist(list, &length, (info == TARGET_TEXT_PLAIN));
+                       uri_selection_data_set_uris_from_filelist(selection_data, list);
                        filelist_free(list);
                        break;
                default:
-                       uri_text = NULL;
                        break;
                }
-
-       if (uri_text) gtk_selection_data_set(selection_data, selection_data->target,
-                                            8, (guchar *)uri_text, length);
-       g_free(uri_text);
 }
 
 static void dupe_dnd_data_get(GtkWidget *widget, GdkDragContext *context,
@@ -3426,6 +4973,13 @@ static void dupe_dnd_data_get(GtkWidget *widget, GdkDragContext *context,
        GList *list = NULL;
        GList *work;
 
+       if (dw->add_files_queue_id > 0)
+               {
+               warning_dialog(_("Find duplicates"), _("Please wait for the current file selection to be loaded."), GTK_STOCK_DIALOG_INFO, dw->window);
+
+               return;
+               }
+
        source = gtk_drag_get_source_widget(context);
        if (source == dw->listview || source == dw->second_listview) return;
 
@@ -3434,10 +4988,10 @@ static void dupe_dnd_data_get(GtkWidget *widget, GdkDragContext *context,
        switch (info)
                {
                case TARGET_APP_COLLECTION_MEMBER:
-                       collection_from_dnd_data((gchar *)selection_data->data, &list, NULL);
+                       collection_from_dnd_data((gchar *)gtk_selection_data_get_data(selection_data), &list, NULL);
                        break;
                case TARGET_URI_LIST:
-                       list = uri_filelist_from_text((gchar *)selection_data->data, TRUE);
+                       list = uri_filelist_from_gtk_selection_data(selection_data);
                        work = list;
                        while (work)
                                {
@@ -3464,7 +5018,7 @@ static void dupe_dnd_data_get(GtkWidget *widget, GdkDragContext *context,
                }
 }
 
-static void dupe_dest_set(GtkWidget *widget, gint enable)
+static void dupe_dest_set(GtkWidget *widget, gboolean enable)
 {
        if (enable)
                {
@@ -3568,9 +5122,11 @@ static void dupe_notify_cb(FileData *fd, NotifyType type, gpointer data)
 {
        DupeWindow *dw = data;
 
-       if (type != NOTIFY_TYPE_CHANGE || !fd->change) return;
-       
-       switch(fd->change->type)
+       if (!(type & NOTIFY_CHANGE) || !fd->change) return;
+
+       DEBUG_1("Notify dupe: %s %04x", fd->path, type);
+
+       switch (fd->change->type)
                {
                case FILEDATA_CHANGE_MOVE:
                case FILEDATA_CHANGE_RENAME:
@@ -3579,7 +5135,7 @@ static void dupe_notify_cb(FileData *fd, NotifyType type, gpointer data)
                case FILEDATA_CHANGE_COPY:
                        break;
                case FILEDATA_CHANGE_DELETE:
-                       while (dupe_item_remove_by_path(dw, fd->path));
+                       /* Update the UI only once, after the operation finishes */
                        break;
                case FILEDATA_CHANGE_UNSPECIFIED:
                case FILEDATA_CHANGE_WRITE_METADATA:
@@ -3587,4 +5143,269 @@ static void dupe_notify_cb(FileData *fd, NotifyType type, gpointer data)
                }
 
 }
+
+/**
+ * @brief Refresh window after a file delete operation
+ * @param success (ud->phase != UTILITY_PHASE_CANCEL) #file_util_dialog_run
+ * @param dest_path Not used
+ * @param data #DupeWindow
+ * 
+ * If the window is refreshed after each file of a large set is deleted,
+ * the UI slows to an unacceptable level. The #FileUtilDoneFunc is used
+ * to call this function once, when the entire delete operation is completed.
+ */
+static void delete_finished_cb(gboolean success, const gchar *dest_path, gpointer data)
+{
+       DupeWindow *dw = data;
+       GList *work;
+
+       if (!success)
+               {
+               return;
+               }
+
+       dupe_window_remove_selection(dw, dw->listview);
+}
+
+/*
+ *-------------------------------------------------------------------
+ * Export duplicates data
+ *-------------------------------------------------------------------
+ */
+
+ typedef enum {
+       EXPORT_CSV = 0,
+       EXPORT_TSV
+} SeparatorType;
+
+typedef struct _ExportDupesData ExportDupesData;
+struct _ExportDupesData
+{
+       FileDialog *dialog;
+       SeparatorType separator;
+       DupeWindow *dupewindow;
+};
+
+static void export_duplicates_close(ExportDupesData *edd)
+{
+       if (edd->dialog) file_dialog_close(edd->dialog);
+       edd->dialog = NULL;
+}
+
+static void export_duplicates_data_cancel_cb(FileDialog *fdlg, gpointer data)
+{
+       ExportDupesData *edd = data;
+
+       export_duplicates_close(edd);
+}
+
+static void export_duplicates_data_save_cb(FileDialog *fdlg, gpointer data)
+{
+       ExportDupesData *edd = data;
+       GError *error = NULL;
+       GtkTreeModel *store;
+       GtkTreeIter iter;
+       DupeItem *di;
+       GFileOutputStream *gfstream;
+       GFile *out_file;
+       GString *output_string;
+       gchar *sep;
+       gchar* rank;
+       GList *work;
+       GtkTreeSelection *selection;
+       GList *slist;
+       gchar *thumb_cache;
+       gchar **rank_split;
+       GtkTreePath *tpath;
+       gboolean color_old = FALSE;
+       gboolean color_new = FALSE;
+       gint match_count;
+       gchar *name;
+
+       history_list_add_to_key("export_duplicates", fdlg->dest_path, -1);
+
+       out_file = g_file_new_for_path(fdlg->dest_path);
+
+       gfstream = g_file_replace(out_file, NULL, TRUE, G_FILE_CREATE_NONE, NULL, &error);
+       if (error)
+               {
+               log_printf(_("Error creating Export duplicates data file: Error: %s\n"), error->message);
+               g_error_free(error);
+               return;
+               }
+
+       sep = g_strdup((edd->separator == EXPORT_CSV) ?  "," : "\t");
+       output_string = g_string_new(g_strjoin(sep, _("Match"), _("Group"), _("Similarity"), _("Set"), _("Thumbnail"), _("Name"), _("Size"), _("Date"), _("Width"), _("Height"), _("Path\n"), NULL));
+
+       selection = gtk_tree_view_get_selection(GTK_TREE_VIEW(edd->dupewindow->listview));
+       slist = gtk_tree_selection_get_selected_rows(selection, &store);
+       work = slist;
+
+       tpath = work->data;
+       gtk_tree_model_get_iter(store, &iter, tpath);
+       gtk_tree_model_get(GTK_TREE_MODEL(store), &iter, DUPE_COLUMN_COLOR, &color_new, -1);
+       color_old = !color_new;
+       match_count = 0;
+
+       while (work)
+               {
+               tpath = work->data;
+               gtk_tree_model_get_iter(store, &iter, tpath);
+
+               gtk_tree_model_get(GTK_TREE_MODEL(store), &iter, DUPE_COLUMN_POINTER, &di, -1);
+
+               gtk_tree_model_get(GTK_TREE_MODEL(store), &iter, DUPE_COLUMN_COLOR, &color_new, -1);
+               if (color_new != color_old)
+                       {
+                       match_count++;
+                       }
+               color_old = color_new;
+               output_string = g_string_append(output_string, g_strdup_printf("%d", match_count));
+               output_string = g_string_append(output_string, sep);
+
+               if ((dupe_match_find_parent(edd->dupewindow, di) == di))
+                       {
+                       output_string = g_string_append(output_string, "1");
+                       }
+               else
+                       {
+                       output_string = g_string_append(output_string, "2");
+                       }
+               output_string = g_string_append(output_string, sep);
+
+               gtk_tree_model_get(GTK_TREE_MODEL(store), &iter, DUPE_COLUMN_RANK, &rank, -1);
+               rank_split = g_strsplit_set(rank, " [(", -1);
+               if (rank_split[0] == NULL)
+                       {
+                       output_string = g_string_append(output_string, "");
+                       }
+               else
+                       {
+                       output_string = g_string_append(output_string, g_strdup_printf("%s", rank_split[0]));
+                       }
+               output_string = g_string_append(output_string, sep);
+               g_free(rank);
+               g_strfreev(rank_split);
+
+               output_string = g_string_append(output_string, g_strdup_printf("%d", (di->second + 1)));
+               output_string = g_string_append(output_string, sep);
+
+               thumb_cache = cache_find_location(CACHE_TYPE_THUMB, di->fd->path);
+               if (thumb_cache)
+                       {
+                       output_string = g_string_append(output_string, thumb_cache);
+                       g_free(thumb_cache);
+                       }
+               else
+                       {
+                       output_string = g_string_append(output_string, "");
+                       }
+               output_string = g_string_append(output_string, sep);
+
+               gtk_tree_model_get(GTK_TREE_MODEL(store), &iter, DUPE_COLUMN_NAME, &name, -1);
+               output_string = g_string_append(output_string, name);
+               output_string = g_string_append(output_string, sep);
+               g_free(name);
+
+               output_string = g_string_append(output_string, g_strdup_printf("%"PRIu64, di->fd->size));
+               output_string = g_string_append(output_string, sep);
+               output_string = g_string_append(output_string, text_from_time(di->fd->date));
+               output_string = g_string_append(output_string, sep);
+               output_string = g_string_append(output_string, g_strdup_printf("%d", (di->width ? di->width : 0)));
+               output_string = g_string_append(output_string, sep);
+               output_string = g_string_append(output_string, g_strdup_printf("%d", (di->height ? di->height : 0)));
+               output_string = g_string_append(output_string, sep);
+               output_string = g_string_append(output_string, di->fd->path);
+               output_string = g_string_append_c(output_string, '\n');
+
+               work = work->next;
+               }
+
+       g_output_stream_write(G_OUTPUT_STREAM(gfstream), output_string->str, strlen(output_string->str), NULL, &error);
+
+       g_free(sep);
+       g_string_free(output_string, TRUE);
+       g_object_unref(gfstream);
+       g_object_unref(out_file);
+
+       export_duplicates_close(edd);
+}
+
+static void pop_menu_export(GList *selection_list, gpointer dupe_window, gpointer data)
+{
+       const gint index = GPOINTER_TO_INT(data);
+       DupeWindow *dw = dupe_window;
+       gchar *title = "Export duplicates data";
+       gchar *default_path = "/tmp/";
+       gchar *file_extension;
+       const gchar *stock_id;
+       ExportDupesData *edd;
+       const gchar *previous_path;
+
+       edd = g_new0(ExportDupesData, 1);
+       edd->dialog = file_util_file_dlg(title, "export_duplicates", NULL, export_duplicates_data_cancel_cb, edd);
+
+       switch (index)
+               {
+               case EXPORT_CSV:
+                       edd->separator = EXPORT_CSV;
+                       file_extension = g_strdup(".csv");
+                       break;
+               case EXPORT_TSV:
+                       edd->separator = EXPORT_TSV;
+                       file_extension = g_strdup(".tsv");
+                       break;
+               default:
+                       return;
+               }
+
+       stock_id = GTK_STOCK_SAVE;
+
+       generic_dialog_add_message(GENERIC_DIALOG(edd->dialog), NULL, title, NULL, FALSE);
+       file_dialog_add_button(edd->dialog, stock_id, NULL, export_duplicates_data_save_cb, TRUE);
+
+       previous_path = history_list_find_last_path_by_key("export_duplicates");
+
+       file_dialog_add_path_widgets(edd->dialog, default_path, previous_path, "export_duplicates", file_extension, _("Export Files"));
+
+       edd->dupewindow = dw;
+
+       gtk_widget_show(GENERIC_DIALOG(edd->dialog)->dialog);
+
+       g_free(file_extension);
+}
+
+static void dupe_pop_menu_export_cb(GtkWidget *widget, gpointer data)
+{
+       DupeWindow *dw;
+       GList *selection_list;
+
+       dw = submenu_item_get_data(widget);
+       selection_list = dupe_listview_get_selection(dw, dw->listview);
+       pop_menu_export(selection_list, dw, data);
+
+       filelist_free(selection_list);
+}
+
+static GtkWidget *submenu_add_export(GtkWidget *menu, GtkWidget **menu_item, GCallback func, gpointer data)
+{
+       GtkWidget *item;
+       GtkWidget *submenu;
+
+       item = menu_item_add(menu, _("Export"), NULL, NULL);
+
+       submenu = gtk_menu_new();
+       g_object_set_data(G_OBJECT(submenu), "submenu_data", data);
+
+       menu_item_add_stock_sensitive(submenu, _("Export to csv"),
+                                       GTK_STOCK_INDEX, TRUE, G_CALLBACK(func), GINT_TO_POINTER(0));
+       menu_item_add_stock_sensitive(submenu, _("Export to tab-delimited"),
+                                       GTK_STOCK_INDEX, TRUE, G_CALLBACK(func), GINT_TO_POINTER(1));
+
+       gtk_menu_item_set_submenu(GTK_MENU_ITEM(item), submenu);
+       if (menu_item) *menu_item = item;
+
+       return submenu;
+}
+
 /* vim: set shiftwidth=8 softtabstop=0 cindent cinoptions={1s: */