Fix #477: similarity duplicate search
[geeqie.git] / src / dupe.c
index 620b692..932606e 100644 (file)
@@ -1,15 +1,24 @@
 /*
- * Geeqie
- * (C) 2005 John Ellis
+ * Copyright (C) 2005 John Ellis
+ * Copyright (C) 2008 - 2016 The Geeqie Team
  *
  * Author: John Ellis
  *
- * This software is released under the GNU General Public License (GNU GPL).
- * Please read the included file COPYING for more information.
- * This software comes with no warranty of any kind, use at your own risk!
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  */
 
-
 #include "main.h"
 #include "dupe.h"
 
 #include "collect-table.h"
 #include "dnd.h"
 #include "editors.h"
-#include "filelist.h"
+#include "filedata.h"
 #include "image-load.h"
 #include "img-view.h"
-#include "info.h"
 #include "layout.h"
 #include "layout_image.h"
 #include "md5-util.h"
 #include "menu.h"
+#include "misc.h"
 #include "print.h"
 #include "thumb.h"
-#include "utilops.h"
-#include "ui_bookmark.h"
 #include "ui_fileops.h"
 #include "ui_menu.h"
 #include "ui_misc.h"
 #include "ui_tree_edit.h"
+#include "uri_utils.h"
+#include "utilops.h"
+#include "window.h"
 
 #include <gdk/gdkkeysyms.h> /* for keyboard values */
 
@@ -41,7 +51,7 @@
 #include <math.h>
 
 
-#define DUPE_DEF_WIDTH 600
+#define DUPE_DEF_WIDTH 800
 #define DUPE_DEF_HEIGHT 400
 
 /* column assignment order (simply change them here) */
@@ -80,6 +90,8 @@ static GtkWidget *dupe_menu_popup_second(DupeWindow *dw, DupeItem *di);
 
 static void dupe_dnd_init(DupeWindow *dw);
 
+static void dupe_notify_cb(FileData *fd, NotifyType type, gpointer data);
+
 /*
  * ------------------------------------------------------------------
  * Window updates
@@ -87,7 +99,7 @@ static void dupe_dnd_init(DupeWindow *dw);
  */
 
 
-static void dupe_window_update_count(DupeWindow *dw, gint count_only)
+static void dupe_window_update_count(DupeWindow *dw, gboolean count_only)
 {
        gchar *text;
 
@@ -129,7 +141,7 @@ static gint dupe_iterations(gint n)
        return (n * ((n + 1) / 2));
 }
 
-static void dupe_window_update_progress(DupeWindow *dw, const gchar *status, gdouble value, gint force)
+static void dupe_window_update_progress(DupeWindow *dw, const gchar *status, gdouble value, gboolean force)
 {
        const gchar *status_text;
 
@@ -141,7 +153,7 @@ static void dupe_window_update_progress(DupeWindow *dw, const gchar *status, gdo
                        {
                        new_time = msec_time() - dw->setup_time;
                        }
-               
+
                if (!force &&
                    value != 0.0 &&
                    dw->setup_count > 0 &&
@@ -199,7 +211,7 @@ static void dupe_window_update_progress(DupeWindow *dw, const gchar *status, gdo
                        {
                        status_text = NULL;
                        }
-               }       
+               }
        else
                {
                gtk_progress_bar_set_fraction(GTK_PROGRESS_BAR(dw->extra_label), 0.0);
@@ -213,19 +225,19 @@ static void widget_set_cursor(GtkWidget *widget, gint icon)
 {
        GdkCursor *cursor;
 
-       if (!widget->window) return;
+       if (!gtk_widget_get_window(widget)) return;
+
        if (icon == -1)
                {
                cursor = NULL;
                }
        else
                {
-               cursor = gdk_cursor_new (icon);
+               cursor = gdk_cursor_new(icon);
                }
-       gdk_window_set_cursor(widget->window, cursor);
+
+       gdk_window_set_cursor(gtk_widget_get_window(widget), cursor);
+
        if (cursor) gdk_cursor_unref(cursor);
 }
 
@@ -241,7 +253,7 @@ static void dupe_listview_realign_colors(DupeWindow *dw)
        GtkTreeIter iter;
        gboolean color_set = TRUE;
        DupeItem *parent = NULL;
-       gint valid;
+       gboolean valid;
 
        store = gtk_tree_view_get_model(GTK_TREE_VIEW(dw->listview));
        valid = gtk_tree_model_get_iter_first(store, &iter);
@@ -284,18 +296,8 @@ static DupeItem *dupe_item_new(FileData *fd)
        di = g_new0(DupeItem, 1);
 
        di->fd = file_data_ref(fd);
-
-       di->group = NULL;
        di->group_rank = 0.0;
 
-       di->simd = NULL;
-       di->checksum = 0;
-       di->md5sum = NULL;
-       di->width = 0;
-       di->height = 0;
-
-       di->second = FALSE;
-
        return di;
 }
 
@@ -410,10 +412,6 @@ static void dupe_item_read_cache(DupeItem *di)
                        di->width = cd->width;
                        di->height = cd->height;
                        }
-               if (di->checksum == 0 && cd->have_checksum)
-                       {
-                       di->checksum = cd->checksum;
-                       }
                if (!di->md5sum && cd->have_md5sum)
                        {
                        di->md5sum = md5_digest_to_text(cd->md5sum);
@@ -430,7 +428,7 @@ static void dupe_item_write_cache(DupeItem *di)
        if (!di) return;
 
        base = cache_get_location(CACHE_TYPE_SIM, di->fd->path, FALSE, &mode);
-       if (cache_ensure_dir_exists(base, mode))
+       if (recursive_mkdir_if_not_exists(base, mode))
                {
                CacheData *cd;
 
@@ -438,7 +436,6 @@ static void dupe_item_write_cache(DupeItem *di)
                cd->path = cache_get_location(CACHE_TYPE_SIM, di->fd->path, TRUE, NULL);
 
                if (di->width != 0) cache_sim_data_set_dimensions(cd, di->width, di->height);
-               if (di->checksum != 0) cache_sim_data_set_checksum(cd, di->checksum);
                if (di->md5sum)
                        {
                        guchar digest[16];
@@ -463,7 +460,7 @@ static void dupe_item_write_cache(DupeItem *di)
 
 static gint dupe_listview_find_item(GtkListStore *store, DupeItem *item, GtkTreeIter *iter)
 {
-       gint valid;
+       gboolean valid;
        gint row = 0;
 
        valid = gtk_tree_model_get_iter_first(GTK_TREE_MODEL(store), iter);
@@ -511,7 +508,7 @@ static void dupe_listview_add(DupeWindow *dw, DupeItem *parent, DupeItem *child)
                else
                        {
                        rank = 1;
-                       printf("NULL group in item!\n");
+                       log_printf("NULL group in item!\n");
                        }
                }
        else
@@ -577,6 +574,8 @@ static void dupe_listview_add(DupeWindow *dw, DupeItem *parent, DupeItem *child)
        g_free(text[DUPE_COLUMN_DIMENSIONS]);
 }
 
+static void dupe_listview_select_dupes(DupeWindow *dw, DupeSelectType parents);
+
 static void dupe_listview_populate(DupeWindow *dw)
 {
        GtkListStore *store;
@@ -610,6 +609,16 @@ static void dupe_listview_populate(DupeWindow *dw)
                }
 
        gtk_tree_view_columns_autosize(GTK_TREE_VIEW(dw->listview));
+
+       if (options->duplicates_select_type == DUPE_SELECT_GROUP1)
+               {
+               dupe_listview_select_dupes(dw, DUPE_SELECT_GROUP1);
+               }
+       else if (options->duplicates_select_type == DUPE_SELECT_GROUP2)
+               {
+               dupe_listview_select_dupes(dw, DUPE_SELECT_GROUP2);
+               }
+
 }
 
 static void dupe_listview_remove(DupeWindow *dw, DupeItem *di)
@@ -638,7 +647,7 @@ static GList *dupe_listview_get_filelist(DupeWindow *dw, GtkWidget *listview)
 {
        GtkTreeModel *store;
        GtkTreeIter iter;
-       gint valid;
+       gboolean valid;
        GList *list = NULL;
 
        store = gtk_tree_view_get_model(GTK_TREE_VIEW(listview));
@@ -687,13 +696,13 @@ static GList *dupe_listview_get_selection(DupeWindow *dw, GtkWidget *listview)
        return g_list_reverse(list);
 }
 
-static gint dupe_listview_item_is_selected(DupeWindow *dw, DupeItem *di, GtkWidget *listview)
+static gboolean dupe_listview_item_is_selected(DupeWindow *dw, DupeItem *di, GtkWidget *listview)
 {
        GtkTreeModel *store;
        GtkTreeSelection *selection;
        GList *slist;
        GList *work;
-       gint found = FALSE;
+       gboolean found = FALSE;
 
        selection = gtk_tree_view_get_selection(GTK_TREE_VIEW(listview));
        slist = gtk_tree_selection_get_selected_rows(selection, &store);
@@ -715,12 +724,12 @@ static gint dupe_listview_item_is_selected(DupeWindow *dw, DupeItem *di, GtkWidg
        return found;
 }
 
-static void dupe_listview_select_dupes(DupeWindow *dw, gint parents)
+static void dupe_listview_select_dupes(DupeWindow *dw, DupeSelectType parents)
 {
        GtkTreeModel *store;
        GtkTreeSelection *selection;
        GtkTreeIter iter;
-       gint valid;
+       gboolean valid;
 
        selection = gtk_tree_view_get_selection(GTK_TREE_VIEW(dw->listview));
        gtk_tree_selection_unselect_all(selection);
@@ -732,7 +741,7 @@ static void dupe_listview_select_dupes(DupeWindow *dw, gint parents)
                DupeItem *di;
 
                gtk_tree_model_get(store, &iter, DUPE_COLUMN_POINTER, &di, -1);
-               if ( (dupe_match_find_parent(dw, di) == di) == (parents) )
+               if ((dupe_match_find_parent(dw, di) == di) == (parents == DUPE_SELECT_GROUP1))
                        {
                        gtk_tree_selection_select_iter(selection, &iter);
                        }
@@ -794,10 +803,10 @@ static void dupe_match_unlink(DupeItem *a, DupeItem *b)
        dupe_match_unlink_child(b, a);
 }
 
-static void dupe_match_link_clear(DupeItem *parent, gint unlink_children)
+static void dupe_match_link_clear(DupeItem *parent, gboolean unlink_children)
 {
        GList *work;
-                                                                                                                               
+
        work = parent->group;
        while (work)
                {
@@ -926,7 +935,7 @@ static void dupe_match_print_group(DupeItem *di)
 {
        GList *work;
 
-       printf("+ %f %s\n", di->group_rank, di->fd->name);
+       log_printf("+ %f %s\n", di->group_rank, di->fd->name);
 
        work = di->group;
        while (work)
@@ -934,10 +943,10 @@ static void dupe_match_print_group(DupeItem *di)
                DupeMatch *dm = work->data;
                work = work->next;
 
-               printf("  %f %s\n", dm->rank, dm->di->fd->name);
+               log_printf("  %f %s\n", dm->rank, dm->di->fd->name);
                }
 
-       printf("\n");
+       log_printf("\n");
 }
 
 static void dupe_match_print_list(GList *list)
@@ -964,7 +973,7 @@ static GList *dupe_match_unlink_by_rank(DupeItem *child, DupeItem *parent, GList
                GList *work;
                gdouble rank;
 
-               if (debug > 1) printf("link found %s to %s [%d]\n", child->fd->name, parent->fd->name, g_list_length(parent->group));
+               DEBUG_2("link found %s to %s [%d]", child->fd->name, parent->fd->name, g_list_length(parent->group));
 
                work = parent->group;
                while (work)
@@ -985,7 +994,7 @@ static GList *dupe_match_unlink_by_rank(DupeItem *child, DupeItem *parent, GList
                                list = g_list_remove(list, orphan);
                                }
                        }
-               
+
                rank = dupe_match_link_rank(child, parent);
                dupe_match_link_clear(parent, TRUE);
                dupe_match_link(child, parent, rank);
@@ -993,8 +1002,8 @@ static GList *dupe_match_unlink_by_rank(DupeItem *child, DupeItem *parent, GList
                }
        else
                {
-               if (debug > 1) printf("unlinking %s and %s\n", child->fd->name, parent->fd->name);
-               
+               DEBUG_2("unlinking %s and %s", child->fd->name, parent->fd->name);
+
                dupe_match_unlink(child, parent);
                }
 
@@ -1057,6 +1066,20 @@ static void dupe_match_sort_groups(GList *list)
                }
 }
 
+static gint dupe_match_totals_sort_cb(gconstpointer a, gconstpointer b)
+{
+       DupeItem *da = (DupeItem *)a;
+       DupeItem *db = (DupeItem *)b;
+
+       if (g_list_length(da->group) > g_list_length(db->group)) return -1;
+       if (g_list_length(da->group) < g_list_length(db->group)) return 1;
+
+       if (da->group_rank < db->group_rank) return -1;
+       if (da->group_rank > db->group_rank) return 1;
+
+       return 0;
+}
+
 static gint dupe_match_rank_sort_cb(gconstpointer a, gconstpointer b)
 {
        DupeItem *da = (DupeItem *)a;
@@ -1083,30 +1106,44 @@ static GList *dupe_match_rank_sort(GList *source_list)
                        dupe_match_rank_update(di);
                        list = g_list_prepend(list, di);
                        }
-                       
+
                work = work->next;
                }
 
        return g_list_sort(list, dupe_match_rank_sort_cb);
 }
 
+/* returns allocated GList of dupes sorted by totals */
+static GList *dupe_match_totals_sort(GList *source_list)
+{
+       source_list = g_list_sort(source_list, dupe_match_totals_sort_cb);
+
+       source_list = g_list_first(source_list);
+       return g_list_reverse(source_list);
+}
+
 static void dupe_match_rank(DupeWindow *dw)
 {
        GList *list;
 
        list = dupe_match_rank_sort(dw->list);
 
-       if (debug > 1) dupe_match_print_list(list);
-       
-       if (debug) printf("Similar items: %d\n", g_list_length(list));
+       if (required_debug_level(2)) dupe_match_print_list(list);
+
+       DEBUG_1("Similar items: %d", g_list_length(list));
        list = dupe_match_group_trim(list, dw);
-       if (debug) printf("Unique groups: %d\n", g_list_length(list));
+       DEBUG_1("Unique groups: %d", g_list_length(list));
 
        dupe_match_sort_groups(list);
 
-       if (debug) dupe_match_print_list(list);
+       if (required_debug_level(2)) dupe_match_print_list(list);
 
        list = dupe_match_rank_sort(list);
+       if (options->sort_totals)
+               {
+               list = dupe_match_totals_sort(list);
+               }
+       if (required_debug_level(2)) dupe_match_print_list(list);
 
        g_list_free(dw->dupes);
        dw->dupes = list;
@@ -1118,23 +1155,23 @@ static void dupe_match_rank(DupeWindow *dw)
  * ------------------------------------------------------------------
  */
 
-static gint dupe_match(DupeItem *a, DupeItem *b, DupeMatchType mask, gdouble *rank, gint fast)
+static gboolean dupe_match(DupeItem *a, DupeItem *b, DupeMatchType mask, gdouble *rank, gint fast)
 {
        *rank = 0.0;
 
-       if (a == b) return FALSE;
+       if (a->fd->path == b->fd->path) return FALSE;
 
        if (mask & DUPE_MATCH_PATH)
                {
-               if (strcmp(a->fd->path, b->fd->path) != 0) return FALSE;
+               if (utf8_compare(a->fd->path, b->fd->path, TRUE) != 0) return FALSE;
                }
        if (mask & DUPE_MATCH_NAME)
                {
-               if (strcmp(a->fd->name, b->fd->name) != 0) return FALSE;
+               if (strcmp(a->fd->collate_key_name, b->fd->collate_key_name) != 0) return FALSE;
                }
        if (mask & DUPE_MATCH_NAME_CI)
                {
-               if (strcasecmp(a->fd->name, b->fd->name) != 0) return FALSE;
+               if (strcmp(a->fd->collate_key_name_nocase, b->fd->collate_key_name_nocase) != 0) return FALSE;
                }
        if (mask & DUPE_MATCH_SIZE)
                {
@@ -1168,7 +1205,7 @@ static gint dupe_match(DupeItem *a, DupeItem *b, DupeMatchType mask, gdouble *ra
 
                if (mask & DUPE_MATCH_SIM_HIGH) m = 0.95;
                else if (mask & DUPE_MATCH_SIM_MED) m = 0.90;
-               else if (mask & DUPE_MATCH_SIM_CUSTOM) m = (gdouble)options->dupe_custom_threshold / 100.0;
+               else if (mask & DUPE_MATCH_SIM_CUSTOM) m = (gdouble)options->duplicates_similarity_threshold / 100.0;
                else m = 0.85;
 
                if (fast)
@@ -1184,7 +1221,7 @@ static gint dupe_match(DupeItem *a, DupeItem *b, DupeMatchType mask, gdouble *ra
 
                if (f < m) return FALSE;
 
-               if (debug > 2) printf("similar: %32s %32s = %f\n", a->fd->name, b->fd->name, f);
+               DEBUG_3("similar: %32s %32s = %f", a->fd->name, b->fd->name, f);
                }
 
        return TRUE;
@@ -1260,7 +1297,7 @@ static void dupe_thumb_do(DupeWindow *dw)
        di = dw->thumb_item;
 
        if (di->pixbuf) g_object_unref(di->pixbuf);
-       di->pixbuf = thumb_loader_get_pixbuf(dw->thumb_loader, TRUE);
+       di->pixbuf = thumb_loader_get_pixbuf(dw->thumb_loader);
 
        dupe_listview_set_thumb(dw, di, NULL);
 }
@@ -1286,7 +1323,7 @@ static void dupe_thumb_step(DupeWindow *dw)
        GtkTreeModel *store;
        GtkTreeIter iter;
        DupeItem *di = NULL;
-       gint valid;
+       gboolean valid;
        gint row = 0;
        gint length = 0;
 
@@ -1327,7 +1364,7 @@ static void dupe_thumb_step(DupeWindow *dw)
 
        dw->thumb_item = di;
        thumb_loader_free(dw->thumb_loader);
-       dw->thumb_loader = thumb_loader_new(options->thumb_max_width, options->thumb_max_height);
+       dw->thumb_loader = thumb_loader_new(options->thumbnails.max_width, options->thumbnails.max_height);
 
        thumb_loader_set_callbacks(dw->thumb_loader,
                                   dupe_thumb_done_cb,
@@ -1336,10 +1373,10 @@ static void dupe_thumb_step(DupeWindow *dw)
                                   dw);
 
        /* start it */
-       if (!thumb_loader_start(dw->thumb_loader, di->fd->path))
+       if (!thumb_loader_start(dw->thumb_loader, di->fd))
                {
                /* error, handle it, do next */
-               if (debug) printf("error loading thumb for %s\n", di->fd->path);
+               DEBUG_1("error loading thumb for %s", di->fd->path);
                dupe_thumb_do(dw);
                dupe_thumb_step(dw);
                }
@@ -1353,10 +1390,10 @@ static void dupe_thumb_step(DupeWindow *dw)
 
 static void dupe_check_stop(DupeWindow *dw)
 {
-       if (dw->idle_id != -1 || dw->img_loader || dw->thumb_loader)
+       if (dw->idle_id || dw->img_loader || dw->thumb_loader)
                {
                g_source_remove(dw->idle_id);
-               dw->idle_id = -1;
+               dw->idle_id = 0;
                dupe_window_update_progress(dw, NULL, 0.0, FALSE);
                widget_set_cursor(dw->listview, -1);
                }
@@ -1393,7 +1430,7 @@ static void dupe_loader_done_cb(ImageLoader *il, gpointer data)
                        di->width = gdk_pixbuf_get_width(pixbuf);
                        di->height = gdk_pixbuf_get_height(pixbuf);
                        }
-               if (options->enable_thumb_caching)
+               if (options->thumbnails.enable_caching)
                        {
                        dupe_item_write_cache(di);
                        }
@@ -1426,11 +1463,11 @@ static GList *dupe_setup_point_step(DupeWindow *dw, GList *p)
        return NULL;
 }
 
-static gint dupe_check_cb(gpointer data)
+static gboolean dupe_check_cb(gpointer data)
 {
        DupeWindow *dw = data;
 
-       if (dw->idle_id == -1) return FALSE;
+       if (!dw->idle_id) return FALSE;
 
        if (!dw->setup_done)
                {
@@ -1451,14 +1488,14 @@ static gint dupe_check_cb(gpointer data)
                                        dupe_window_update_progress(dw, _("Reading checksums..."),
                                                dw->setup_count == 0 ? 0.0 : (gdouble)(dw->setup_n - 1) / dw->setup_count, FALSE);
 
-                                       if (options->enable_thumb_caching)
+                                       if (options->thumbnails.enable_caching)
                                                {
                                                dupe_item_read_cache(di);
                                                if (di->md5sum) return TRUE;
                                                }
 
                                        di->md5sum = md5_text_from_file_utf8(di->fd->path, "");
-                                       if (options->enable_thumb_caching)
+                                       if (options->thumbnails.enable_caching)
                                                {
                                                dupe_item_write_cache(di);
                                                }
@@ -1484,14 +1521,14 @@ static gint dupe_check_cb(gpointer data)
                                        dupe_window_update_progress(dw, _("Reading dimensions..."),
                                                dw->setup_count == 0 ? 0.0 : (gdouble)(dw->setup_n - 1) / dw->setup_count, FALSE);
 
-                                       if (options->enable_thumb_caching)
+                                       if (options->thumbnails.enable_caching)
                                                {
                                                dupe_item_read_cache(di);
                                                if (di->width != 0 || di->height != 0) return TRUE;
                                                }
 
                                        image_load_dimensions(di->fd, &di->width, &di->height);
-                                       if (options->enable_thumb_caching)
+                                       if (options->thumbnails.enable_caching)
                                                {
                                                dupe_item_write_cache(di);
                                                }
@@ -1518,7 +1555,7 @@ static gint dupe_check_cb(gpointer data)
                                        dupe_window_update_progress(dw, _("Reading similarity data..."),
                                                dw->setup_count == 0 ? 0.0 : (gdouble)dw->setup_n / dw->setup_count, FALSE);
 
-                                       if (options->enable_thumb_caching)
+                                       if (options->thumbnails.enable_caching)
                                                {
                                                dupe_item_read_cache(di);
                                                if (cache_sim_data_filled(di->simd))
@@ -1530,9 +1567,10 @@ static gint dupe_check_cb(gpointer data)
 
                                        dw->img_loader = image_loader_new(di->fd);
                                        image_loader_set_buffer_size(dw->img_loader, 8);
-                                       image_loader_set_error_func(dw->img_loader, dupe_loader_done_cb, dw);
+                                       g_signal_connect(G_OBJECT(dw->img_loader), "error", (GCallback)dupe_loader_done_cb, dw);
+                                       g_signal_connect(G_OBJECT(dw->img_loader), "done", (GCallback)dupe_loader_done_cb, dw);
 
-                                       if (!image_loader_start(dw->img_loader, dupe_loader_done_cb, dw))
+                                       if (!image_loader_start(dw->img_loader))
                                                {
                                                image_sim_free(di->simd);
                                                di->simd = image_sim_new();
@@ -1540,7 +1578,7 @@ static gint dupe_check_cb(gpointer data)
                                                dw->img_loader = NULL;
                                                return TRUE;
                                                }
-                                       dw->idle_id = -1;
+                                       dw->idle_id = 0;
                                        return FALSE;
                                        }
 
@@ -1564,7 +1602,7 @@ static gint dupe_check_cb(gpointer data)
                        dupe_window_update_progress(dw, _("Sorting..."), 1.0, TRUE);
                        return TRUE;
                        }
-               dw->idle_id = -1;
+               dw->idle_id = 0;
                dupe_window_update_progress(dw, NULL, 0.0, FALSE);
 
                dupe_match_rank(dw);
@@ -1604,7 +1642,7 @@ static void dupe_check_start(DupeWindow *dw)
        dupe_window_update_count(dw, TRUE);
        widget_set_cursor(dw->listview, GDK_WATCH);
 
-       if (dw->idle_id != -1) return;
+       if (dw->idle_id) return;
 
        dw->idle_id = g_idle_add(dupe_check_cb, dw);
 }
@@ -1663,7 +1701,7 @@ static void dupe_item_remove(DupeWindow *dw, DupeItem *di)
                                {
                                DupeItem *new_parent;
                                DupeMatch *dm;
-                               
+
                                dm = parent->group->data;
                                new_parent = dm->di;
                                dupe_match_reparent(dw, parent, new_parent);
@@ -1701,7 +1739,7 @@ static void dupe_item_remove(DupeWindow *dw, DupeItem *di)
        dupe_window_update_count(dw, FALSE);
 }
 
-static gint dupe_item_remove_by_path(DupeWindow *dw, const gchar *path)
+static gboolean dupe_item_remove_by_path(DupeWindow *dw, const gchar *path)
 {
        DupeItem *di;
 
@@ -1714,7 +1752,7 @@ static gint dupe_item_remove_by_path(DupeWindow *dw, const gchar *path)
 }
 
 static void dupe_files_add(DupeWindow *dw, CollectionData *collection, CollectInfo *info,
-                          FileData *fd, gint recurse)
+                          FileData *fd, gboolean recurse)
 {
        DupeItem *di = NULL;
 
@@ -1731,7 +1769,7 @@ static void dupe_files_add(DupeWindow *dw, CollectionData *collection, CollectIn
                else if (isdir(fd->path) && recurse)
                        {
                        GList *f, *d;
-                       if (filelist_read(fd->path, &f, &d))
+                       if (filelist_read(fd, &f, &d))
                                {
                                GList *work;
 
@@ -1739,14 +1777,14 @@ static void dupe_files_add(DupeWindow *dw, CollectionData *collection, CollectIn
                                d = filelist_filter(d, TRUE);
 
                                work = f;
-                               while(work)
+                               while (work)
                                        {
                                        dupe_files_add(dw, NULL, NULL, (FileData *)work->data, TRUE);
                                        work = work->next;
                                        }
                                filelist_free(f);
                                work = d;
-                               while(work)
+                               while (work)
                                        {
                                        dupe_files_add(dw, NULL, NULL, (FileData *)work->data, TRUE);
                                        work = work->next;
@@ -1758,6 +1796,40 @@ static void dupe_files_add(DupeWindow *dw, CollectionData *collection, CollectIn
 
        if (!di) return;
 
+       /* Ensure images in the lists have unique FileDatas */
+       GList *work;
+       DupeItem *di_list;
+       work = g_list_first(dw->list);
+       while (work)
+               {
+               di_list = work->data;
+               if (di_list->fd == di->fd)
+                       {
+                       return;
+                       }
+               else
+                       {
+                       work = work->next;
+                       }
+               }
+
+       if (dw->second_list)
+               {
+               work = g_list_first(dw->second_list);
+               while (work)
+                       {
+                       di_list = work->data;
+                       if (di_list->fd == di->fd)
+                               {
+                               return;
+                               }
+                       else
+                               {
+                               work = work->next;
+                               }
+                       }
+               }
+
        if (dw->second_drop)
                {
                dupe_second_add(dw, di);
@@ -1773,7 +1845,7 @@ void dupe_window_add_collection(DupeWindow *dw, CollectionData *collection)
        CollectInfo *info;
 
        info = collection_get_first(collection);
-       while(info)
+       while (info)
                {
                dupe_files_add(dw, collection, info, NULL, FALSE);
                info = collection_next_by_info(collection, info);
@@ -1782,12 +1854,12 @@ void dupe_window_add_collection(DupeWindow *dw, CollectionData *collection)
        dupe_check_start(dw);
 }
 
-void dupe_window_add_files(DupeWindow *dw, GList *list, gint recurse)
+void dupe_window_add_files(DupeWindow *dw, GList *list, gboolean recurse)
 {
        GList *work;
 
        work = list;
-       while(work)
+       while (work)
                {
                FileData *fd = work->data;
                work = work->next;
@@ -1813,7 +1885,7 @@ static void dupe_item_update(DupeWindow *dw, DupeItem *di)
                dw->second_drop = second;
                dupe_files_add(dw, NULL, NULL, fd, FALSE);
                dw->second_drop = FALSE;
-               
+
                file_data_unref(fd);
 */
                dupe_check_start(dw);
@@ -1853,7 +1925,7 @@ static void dupe_item_update_fd_in_list(DupeWindow *dw, FileData *fd, GList *wor
                {
                DupeItem *di = work->data;
 
-               if (di->fd == fd) 
+               if (di->fd == fd)
                        dupe_item_update(dw, di);
 
                work = work->next;
@@ -1887,7 +1959,7 @@ static GtkWidget *dupe_display_label(GtkWidget *vbox, const gchar *description,
        label = gtk_label_new(text);
        gtk_box_pack_start(GTK_BOX(hbox), label, FALSE, FALSE, 0);
        gtk_widget_show(label);
-       
+
        gtk_box_pack_start(GTK_BOX(vbox), hbox, FALSE, FALSE, 0);
        gtk_widget_show(hbox);
 
@@ -1898,10 +1970,10 @@ static void dupe_display_stats(DupeWindow *dw, DupeItem *di)
 {
        GenericDialog *gd;
        gchar *buf;
-       
+
        if (!di) return;
 
-       gd = file_util_gen_dlg("Image thumbprint debug info", GQ_WMCLASS, "thumbprint",
+       gd = file_util_gen_dlg("Image thumbprint debug info", "thumbprint",
                               dw->window, TRUE,
                               NULL, NULL);
        generic_dialog_add_button(gd, GTK_STOCK_CLOSE, NULL, NULL, TRUE);
@@ -1926,7 +1998,7 @@ static void dupe_display_stats(DupeWindow *dw, DupeItem *di)
                guchar *dp;
                gint rs;
                gint sp;
-               
+
                pixbuf = gdk_pixbuf_new(GDK_COLORSPACE_RGB, FALSE, 8, 32, 32);
                rs = gdk_pixbuf_get_rowstride(pixbuf);
                d_pix = gdk_pixbuf_get_pixels(pixbuf);
@@ -1946,10 +2018,10 @@ static void dupe_display_stats(DupeWindow *dw, DupeItem *di)
                image = gtk_image_new_from_pixbuf(pixbuf);
                gtk_box_pack_start(GTK_BOX(gd->vbox), image, FALSE, FALSE, 0);
                gtk_widget_show(image);
-               
-               gdk_pixbuf_unref(pixbuf);
+
+               g_object_unref(pixbuf);
                }
-       
+
        gtk_widget_show(gd->dialog);
 }
 
@@ -1998,7 +2070,7 @@ static void dupe_menu_view(DupeWindow *dw, DupeItem *di, GtkWidget *listview, gi
                        }
                else
                        {
-                       layout_image_set_fd(NULL, di->fd);
+                       layout_set_fd(NULL, di->fd);
                        }
                }
 }
@@ -2045,15 +2117,9 @@ static void dupe_window_remove_selection(DupeWindow *dw, GtkWidget *listview)
        dupe_listview_realign_colors(dw);
 }
 
-static void dupe_window_edit_selected(DupeWindow *dw, gint n)
+static void dupe_window_edit_selected(DupeWindow *dw, const gchar *key)
 {
-       GList *list;
-
-       list = dupe_listview_get_selection(dw, dw->listview);
-
-       start_editor_from_filelist(n, list);
-
-       filelist_free(list);
+       file_util_start_editor_from_filelist(key, dupe_listview_get_selection(dw, dw->listview), NULL, dw->window);
 }
 
 static void dupe_window_collection_from_selection(DupeWindow *dw)
@@ -2103,6 +2169,7 @@ static void dupe_menu_select_all_cb(GtkWidget *widget, gpointer data)
        DupeWindow *dw = data;
        GtkTreeSelection *selection;
 
+       options->duplicates_select_type = DUPE_SELECT_NONE;
        selection = gtk_tree_view_get_selection(GTK_TREE_VIEW(dw->listview));
        gtk_tree_selection_select_all(selection);
 }
@@ -2112,6 +2179,7 @@ static void dupe_menu_select_none_cb(GtkWidget *widget, gpointer data)
        DupeWindow *dw = data;
        GtkTreeSelection *selection;
 
+       options->duplicates_select_type = DUPE_SELECT_NONE;
        selection = gtk_tree_view_get_selection(GTK_TREE_VIEW(dw->listview));
        gtk_tree_selection_unselect_all(selection);
 }
@@ -2120,33 +2188,27 @@ static void dupe_menu_select_dupes_set1_cb(GtkWidget *widget, gpointer data)
 {
        DupeWindow *dw = data;
 
-       dupe_listview_select_dupes(dw, TRUE);
+       options->duplicates_select_type = DUPE_SELECT_GROUP1;
+       dupe_listview_select_dupes(dw, DUPE_SELECT_GROUP1);
 }
 
 static void dupe_menu_select_dupes_set2_cb(GtkWidget *widget, gpointer data)
 {
        DupeWindow *dw = data;
 
-       dupe_listview_select_dupes(dw, FALSE);
+       options->duplicates_select_type = DUPE_SELECT_GROUP2;
+       dupe_listview_select_dupes(dw, DUPE_SELECT_GROUP2);
 }
 
 static void dupe_menu_edit_cb(GtkWidget *widget, gpointer data)
 {
        DupeWindow *dw;
-       gint n;
+       const gchar *key = data;
 
        dw = submenu_item_get_data(widget);
-       n = GPOINTER_TO_INT(data);
        if (!dw) return;
 
-       dupe_window_edit_selected(dw, n);
-}
-
-static void dupe_menu_info_cb(GtkWidget *widget, gpointer data)
-{
-       DupeWindow *dw = data;
-
-       info_window_new(NULL, dupe_listview_get_selection(dw, dw->listview));
+       dupe_window_edit_selected(dw, key);
 }
 
 static void dupe_menu_collection_cb(GtkWidget *widget, gpointer data)
@@ -2196,6 +2258,13 @@ static void dupe_menu_delete_cb(GtkWidget *widget, gpointer data)
        file_util_delete(NULL, dupe_listview_get_selection(dw, dw->listview), dw->window);
 }
 
+static void dupe_menu_copy_path_cb(GtkWidget *widget, gpointer data)
+{
+       DupeWindow *dw = data;
+
+       file_util_copy_path_list_to_clipboard(dupe_listview_get_selection(dw, dw->listview));
+}
+
 static void dupe_menu_remove_cb(GtkWidget *widget, gpointer data)
 {
        DupeWindow *dw = data;
@@ -2217,15 +2286,40 @@ static void dupe_menu_close_cb(GtkWidget *widget, gpointer data)
        dupe_window_close(dw);
 }
 
+static void dupe_menu_popup_destroy_cb(GtkWidget *widget, gpointer data)
+{
+       GList *editmenu_fd_list = data;
+
+       filelist_free(editmenu_fd_list);
+}
+
+static GList *dupe_window_get_fd_list(DupeWindow *dw)
+{
+       GList *list;
+
+       if (gtk_widget_has_focus(dw->second_listview))
+               {
+               list = dupe_listview_get_selection(dw, dw->second_listview);
+               }
+       else
+               {
+               list = dupe_listview_get_selection(dw, dw->listview);
+               }
+
+       return list;
+}
+
 static GtkWidget *dupe_menu_popup_main(DupeWindow *dw, DupeItem *di)
 {
        GtkWidget *menu;
        GtkWidget *item;
        gint on_row;
+       GList *editmenu_fd_list;
 
        on_row = (di != NULL);
 
        menu = popup_menu_short_lived();
+
        menu_item_add_sensitive(menu, _("_View"), on_row,
                                G_CALLBACK(dupe_menu_view_cb), dw);
        menu_item_add_stock_sensitive(menu, _("View in _new window"), GTK_STOCK_NEW, on_row,
@@ -2240,10 +2334,12 @@ static GtkWidget *dupe_menu_popup_main(DupeWindow *dw, DupeItem *di)
        menu_item_add_sensitive(menu, _("Select group _2 duplicates"), (dw->dupes != NULL),
                                G_CALLBACK(dupe_menu_select_dupes_set2_cb), dw);
        menu_item_add_divider(menu);
-       submenu_add_edit(menu, &item, G_CALLBACK(dupe_menu_edit_cb), dw);
+
+       editmenu_fd_list = dupe_window_get_fd_list(dw);
+       g_signal_connect(G_OBJECT(menu), "destroy",
+                        G_CALLBACK(dupe_menu_popup_destroy_cb), editmenu_fd_list);
+       submenu_add_edit(menu, &item, G_CALLBACK(dupe_menu_edit_cb), dw, editmenu_fd_list);
        if (!on_row) gtk_widget_set_sensitive(item, FALSE);
-       menu_item_add_stock_sensitive(menu, _("_Properties"), GTK_STOCK_PROPERTIES, on_row,
-                               G_CALLBACK(dupe_menu_info_cb), dw);
        menu_item_add_stock_sensitive(menu, _("Add to new collection"), GTK_STOCK_INDEX, on_row,
                                G_CALLBACK(dupe_menu_collection_cb), dw);
        menu_item_add_stock_sensitive(menu, _("Print..."), GTK_STOCK_PRINT, on_row,
@@ -2257,6 +2353,8 @@ static GtkWidget *dupe_menu_popup_main(DupeWindow *dw, DupeItem *di)
                                G_CALLBACK(dupe_menu_rename_cb), dw);
        menu_item_add_stock_sensitive(menu, _("_Delete..."), GTK_STOCK_DELETE, on_row,
                                G_CALLBACK(dupe_menu_delete_cb), dw);
+       menu_item_add_sensitive(menu, _("_Copy path"), on_row,
+                               G_CALLBACK(dupe_menu_copy_path_cb), dw);
        menu_item_add_divider(menu);
        menu_item_add_stock_sensitive(menu, _("Rem_ove"), GTK_STOCK_REMOVE, on_row,
                                G_CALLBACK(dupe_menu_remove_cb), dw);
@@ -2269,7 +2367,7 @@ static GtkWidget *dupe_menu_popup_main(DupeWindow *dw, DupeItem *di)
        return menu;
 }
 
-static gint dupe_listview_press_cb(GtkWidget *widget, GdkEventButton *bevent, gpointer data)
+static gboolean dupe_listview_press_cb(GtkWidget *widget, GdkEventButton *bevent, gpointer data)
 {
        DupeWindow *dw = data;
        GtkTreeModel *store;
@@ -2289,7 +2387,7 @@ static gint dupe_listview_press_cb(GtkWidget *widget, GdkEventButton *bevent, gp
 
        dw->click_item = di;
 
-       if (bevent->button == 3)
+       if (bevent->button == MOUSE_BUTTON_RIGHT)
                {
                /* right click menu */
                GtkWidget *menu;
@@ -2312,14 +2410,15 @@ static gint dupe_listview_press_cb(GtkWidget *widget, GdkEventButton *bevent, gp
 
        if (!di) return FALSE;
 
-       if (bevent->button == 1 && bevent->type == GDK_2BUTTON_PRESS)
+       if (bevent->button == MOUSE_BUTTON_LEFT &&
+           bevent->type == GDK_2BUTTON_PRESS)
                {
                dupe_menu_view(dw, di, widget, FALSE);
                }
 
-       if (bevent->button == 2) return TRUE;
+       if (bevent->button == MOUSE_BUTTON_MIDDLE) return TRUE;
 
-       if (bevent->button == 3)
+       if (bevent->button == MOUSE_BUTTON_RIGHT)
                {
                if (!dupe_listview_item_is_selected(dw, di, widget))
                        {
@@ -2337,7 +2436,8 @@ static gint dupe_listview_press_cb(GtkWidget *widget, GdkEventButton *bevent, gp
                return TRUE;
                }
 
-       if (bevent->button == 1 && bevent->type == GDK_BUTTON_PRESS &&
+       if (bevent->button == MOUSE_BUTTON_LEFT &&
+           bevent->type == GDK_BUTTON_PRESS &&
            !(bevent->state & GDK_SHIFT_MASK ) &&
            !(bevent->state & GDK_CONTROL_MASK ) &&
            dupe_listview_item_is_selected(dw, di, widget))
@@ -2350,7 +2450,7 @@ static gint dupe_listview_press_cb(GtkWidget *widget, GdkEventButton *bevent, gp
        return FALSE;
 }
 
-static gint dupe_listview_release_cb(GtkWidget *widget, GdkEventButton *bevent, gpointer data)
+static gboolean dupe_listview_release_cb(GtkWidget *widget, GdkEventButton *bevent, gpointer data)
 {
        DupeWindow *dw = data;
        GtkTreeModel *store;
@@ -2358,7 +2458,7 @@ static gint dupe_listview_release_cb(GtkWidget *widget, GdkEventButton *bevent,
        GtkTreeIter iter;
        DupeItem *di = NULL;
 
-       if (bevent->button != 1 && bevent->button != 2) return TRUE;
+       if (bevent->button != MOUSE_BUTTON_LEFT && bevent->button != MOUSE_BUTTON_MIDDLE) return TRUE;
 
        store = gtk_tree_view_get_model(GTK_TREE_VIEW(widget));
 
@@ -2371,7 +2471,7 @@ static gint dupe_listview_release_cb(GtkWidget *widget, GdkEventButton *bevent,
                gtk_tree_path_free(tpath);
                }
 
-       if (bevent->button == 2)
+       if (bevent->button == MOUSE_BUTTON_MIDDLE)
                {
                if (di && dw->click_item == di)
                        {
@@ -2529,11 +2629,8 @@ static void dupe_second_menu_clear_cb(GtkWidget *widget, gpointer data)
 static GtkWidget *dupe_menu_popup_second(DupeWindow *dw, DupeItem *di)
 {
        GtkWidget *menu;
-       gint notempty;
-       gint on_row;
-
-       on_row = (di != NULL);
-       notempty = (dw->second_list != NULL);
+       gboolean notempty = (dw->second_list != NULL);
+       gboolean on_row = (di != NULL);
 
        menu = popup_menu_short_lived();
        menu_item_add_sensitive(menu, _("_View"), on_row,
@@ -2561,7 +2658,7 @@ static void dupe_second_set_toggle_cb(GtkWidget *widget, gpointer data)
 {
        DupeWindow *dw = data;
 
-       dw->second_set = GTK_TOGGLE_BUTTON(widget)->active;
+       dw->second_set = gtk_toggle_button_get_active(GTK_TOGGLE_BUTTON(widget));
 
        if (dw->second_set)
                {
@@ -2579,6 +2676,15 @@ static void dupe_second_set_toggle_cb(GtkWidget *widget, gpointer data)
        dupe_window_recompare(dw);
 }
 
+static void dupe_sort_totals_toggle_cb(GtkWidget *widget, gpointer data)
+{
+       DupeWindow *dw = data;
+
+       options->sort_totals = gtk_toggle_button_get_active(GTK_TOGGLE_BUTTON(widget));
+       dupe_window_recompare(dw);
+
+}
+
 /*
  *-------------------------------------------------------------------
  * match type menu
@@ -2600,6 +2706,8 @@ static void dupe_menu_type_cb(GtkWidget *combo, gpointer data)
        if (!gtk_combo_box_get_active_iter(GTK_COMBO_BOX(combo), &iter)) return;
        gtk_tree_model_get(store, &iter, DUPE_MENU_COLUMN_MASK, &dw->match_mask, -1);
 
+       options->duplicates_match = dw->match_mask;
+
        dupe_window_recompare(dw);
 }
 
@@ -2653,7 +2761,7 @@ static void dupe_menu_setup(DupeWindow *dw)
 /* this overrides the low default of a GtkCellRenderer from 100 to CELL_HEIGHT_OVERRIDE, something sane for our purposes */
 
 #define CELL_HEIGHT_OVERRIDE 512
-                                                                                                                               
+
 void cell_renderer_height_override(GtkCellRenderer *renderer)
 {
        GParamSpec *spec;
@@ -2698,7 +2806,7 @@ static void dupe_listview_color_cb(GtkTreeViewColumn *tree_column, GtkCellRender
                     "cell-background-set", set, NULL);
 }
 
-static void dupe_listview_add_column(DupeWindow *dw, GtkWidget *listview, gint n, const gchar *title, gint image, gint right_justify)
+static void dupe_listview_add_column(DupeWindow *dw, GtkWidget *listview, gint n, const gchar *title, gboolean image, gboolean right_justify)
 {
        GtkTreeViewColumn *column;
        GtkCellRenderer *renderer;
@@ -2742,7 +2850,7 @@ static void dupe_listview_add_column(DupeWindow *dw, GtkWidget *listview, gint n
        gtk_tree_view_append_column(GTK_TREE_VIEW(listview), column);
 }
 
-static void dupe_listview_set_height(GtkWidget *listview, gint thumb)
+static void dupe_listview_set_height(GtkWidget *listview, gboolean thumb)
 {
        GtkTreeViewColumn *column;
        GtkCellRenderer *cell;
@@ -2751,14 +2859,14 @@ static void dupe_listview_set_height(GtkWidget *listview, gint thumb)
        column = gtk_tree_view_get_column(GTK_TREE_VIEW(listview), DUPE_COLUMN_THUMB - 1);
        if (!column) return;
 
-       gtk_tree_view_column_set_fixed_width(column, (thumb) ? options->thumb_max_width : 4);
-       
-       list = gtk_tree_view_column_get_cell_renderers(column);
+       gtk_tree_view_column_set_fixed_width(column, (thumb) ? options->thumbnails.max_width : 4);
+
+       list = gtk_cell_layout_get_cells(GTK_CELL_LAYOUT(column));
        if (!list) return;
        cell = list->data;
        g_list_free(list);
 
-       g_object_set(G_OBJECT(cell), "height", (thumb) ? options->thumb_max_height : -1, NULL);
+       g_object_set(G_OBJECT(cell), "height", (thumb) ? options->thumbnails.max_height : -1, NULL);
        gtk_tree_view_columns_autosize(GTK_TREE_VIEW(listview));
 }
 
@@ -2773,7 +2881,8 @@ static void dupe_window_show_thumb_cb(GtkWidget *widget, gpointer data)
 {
        DupeWindow *dw = data;
 
-       dw->show_thumbs = GTK_TOGGLE_BUTTON(widget)->active;
+       dw->show_thumbs = gtk_toggle_button_get_active(GTK_TOGGLE_BUTTON(widget));
+       options->duplicates_thumbnails = dw->show_thumbs;
 
        if (dw->show_thumbs)
                {
@@ -2783,7 +2892,7 @@ static void dupe_window_show_thumb_cb(GtkWidget *widget, gpointer data)
                {
                GtkTreeModel *store;
                GtkTreeIter iter;
-               gint valid;
+               gboolean valid;
 
                thumb_loader_free(dw->thumb_loader);
                dw->thumb_loader = NULL;
@@ -2802,6 +2911,41 @@ static void dupe_window_show_thumb_cb(GtkWidget *widget, gpointer data)
        dupe_listview_set_height(dw->listview, dw->show_thumbs);
 }
 
+static void dupe_window_rotation_invariant_cb(GtkWidget *widget, gpointer data)
+{
+       DupeWindow *dw = data;
+
+       options->rot_invariant_sim = gtk_toggle_button_get_active(GTK_TOGGLE_BUTTON(widget));
+       dupe_window_recompare(dw);
+}
+
+static void dupe_window_custom_threshold_cb(GtkWidget *widget, gpointer data)
+{
+       DupeWindow *dw = data;
+       DupeMatchType match_type;
+       GtkTreeModel *store;
+       gboolean valid;
+       GtkTreeIter iter;
+
+       options->duplicates_similarity_threshold = gtk_spin_button_get_value_as_int(GTK_SPIN_BUTTON(widget));
+       dw->match_mask = DUPE_MATCH_SIM_CUSTOM;
+
+       store = gtk_combo_box_get_model(GTK_COMBO_BOX(dw->combo));
+       valid = gtk_tree_model_get_iter_first(store, &iter);
+       while (valid)
+               {
+               gtk_tree_model_get(store, &iter, DUPE_MENU_COLUMN_MASK, &match_type, -1);
+               if (match_type == DUPE_MATCH_SIM_CUSTOM)
+                       {
+                       break;
+                       }
+               valid = gtk_tree_model_iter_next(store, &iter);
+               }
+
+       gtk_combo_box_set_active_iter(GTK_COMBO_BOX(dw->combo), &iter);
+       dupe_window_recompare(dw);
+}
+
 static void dupe_popup_menu_pos_cb(GtkMenu *menu, gint *x, gint *y, gboolean *push_in, gpointer data)
 {
        GtkWidget *view = data;
@@ -2829,18 +2973,18 @@ static void dupe_popup_menu_pos_cb(GtkMenu *menu, gint *x, gint *y, gboolean *pu
        *y = cy;
 }
 
-static gint dupe_window_keypress_cb(GtkWidget *widget, GdkEventKey *event, gpointer data)
+static gboolean dupe_window_keypress_cb(GtkWidget *widget, GdkEventKey *event, gpointer data)
 {
        DupeWindow *dw = data;
-       gint stop_signal = FALSE;
-       gint on_second;
+       gboolean stop_signal = FALSE;
+       gboolean on_second;
        GtkWidget *listview;
        GtkTreeModel *store;
        GtkTreeSelection *selection;
        GList *slist;
        DupeItem *di = NULL;
 
-       on_second = GTK_WIDGET_HAS_FOCUS(dw->second_listview);
+       on_second = gtk_widget_has_focus(dw->second_listview);
 
        if (on_second)
                {
@@ -2871,42 +3015,21 @@ static gint dupe_window_keypress_cb(GtkWidget *widget, GdkEventKey *event, gpoin
 
        if (event->state & GDK_CONTROL_MASK)
                {
-               gint edit_val = -1;
-
                if (!on_second)
                        {
                        stop_signal = TRUE;
                        switch (event->keyval)
                                {
                                case '1':
-                                       edit_val = 0;
-                                       break;
                                case '2':
-                                       edit_val = 1;
-                                       break;
                                case '3':
-                                       edit_val = 2;
-                                       break;
                                case '4':
-                                       edit_val = 3;
-                                       break;
                                case '5':
-                                       edit_val = 4;
-                                       break;
                                case '6':
-                                       edit_val = 5;
-                                       break;
                                case '7':
-                                       edit_val = 6;
-                                       break;
                                case '8':
-                                       edit_val = 7;
-                                       break;
                                case '9':
-                                       edit_val = 8;
-                                       break;
                                case '0':
-                                       edit_val = 9;
                                        break;
                                case 'C': case 'c':
                                        file_util_copy(NULL, dupe_listview_get_selection(dw, listview),
@@ -2922,9 +3045,6 @@ static gint dupe_window_keypress_cb(GtkWidget *widget, GdkEventKey *event, gpoin
                                case 'D': case 'd':
                                        file_util_delete(NULL, dupe_listview_get_selection(dw, listview), dw->window);
                                        break;
-                               case 'P': case 'p':
-                                       info_window_new(NULL, dupe_listview_get_selection(dw, listview));
-                                       break;
                                default:
                                        stop_signal = FALSE;
                                        break;
@@ -2946,7 +3066,7 @@ static gint dupe_window_keypress_cb(GtkWidget *widget, GdkEventKey *event, gpoin
                                                gtk_tree_selection_select_all(selection);
                                                }
                                        break;
-                               case GDK_Delete: case GDK_KP_Delete:
+                               case GDK_KEY_Delete: case GDK_KEY_KP_Delete:
                                        if (on_second)
                                                {
                                                dupe_second_clear(dw);
@@ -2972,24 +3092,19 @@ static gint dupe_window_keypress_cb(GtkWidget *widget, GdkEventKey *event, gpoin
                                        break;
                                }
                        }
-
-               if (edit_val >= 0)
-                       {
-                       dupe_window_edit_selected(dw, edit_val);
-                       }
                }
        else
                {
                stop_signal = TRUE;
                switch (event->keyval)
                        {
-                       case GDK_Return: case GDK_KP_Enter:
+                       case GDK_KEY_Return: case GDK_KEY_KP_Enter:
                                dupe_menu_view(dw, di, listview, FALSE);
                                break;
                        case 'V': case 'v':
                                dupe_menu_view(dw, di, listview, TRUE);
                                break;
-                       case GDK_Delete: case GDK_KP_Delete:
+                       case GDK_KEY_Delete: case GDK_KEY_KP_Delete:
                                dupe_window_remove_selection(dw, listview);
                                break;
                        case 'C': case 'c':
@@ -2999,13 +3114,15 @@ static gint dupe_window_keypress_cb(GtkWidget *widget, GdkEventKey *event, gpoin
                                        }
                                break;
                        case '1':
-                               dupe_listview_select_dupes(dw, TRUE);
+                               options->duplicates_select_type = DUPE_SELECT_GROUP1;
+                               dupe_listview_select_dupes(dw, DUPE_SELECT_GROUP1);
                                break;
                        case '2':
-                               dupe_listview_select_dupes(dw, FALSE);
+                               options->duplicates_select_type = DUPE_SELECT_GROUP2;
+                               dupe_listview_select_dupes(dw, DUPE_SELECT_GROUP2);
                                break;
-                       case GDK_Menu:
-                       case GDK_F10:
+                       case GDK_KEY_Menu:
+                       case GDK_KEY_F10:
                                if (!on_second)
                                        {
                                        GtkWidget *menu;
@@ -3023,6 +3140,9 @@ static gint dupe_window_keypress_cb(GtkWidget *widget, GdkEventKey *event, gpoin
                                                       dupe_popup_menu_pos_cb, listview, 0, GDK_CURRENT_TIME);
                                        }
                                break;
+                       case GDK_KEY_F1:
+                               help_window_show("GuideReferenceKeyboardShortcuts.html#DuplicatesKeyboardShortcuts");
+                               break;
                        default:
                                stop_signal = FALSE;
                                break;
@@ -3067,6 +3187,8 @@ void dupe_window_close(DupeWindow *dw)
 
        dupe_list_free(dw->second_list);
 
+       file_data_unregister_notify_func(dupe_notify_cb, dw);
+
        g_free(dw);
 }
 
@@ -3079,7 +3201,7 @@ static gint dupe_window_delete(GtkWidget *widget, GdkEvent *event, gpointer data
 }
 
 /* collection and files can be NULL */
-DupeWindow *dupe_window_new(DupeMatchType match_mask)
+DupeWindow *dupe_window_new()
 {
        DupeWindow *dw;
        GtkWidget *vbox;
@@ -3094,19 +3216,23 @@ DupeWindow *dupe_window_new(DupeMatchType match_mask)
 
        dw = g_new0(DupeWindow, 1);
 
-       dw->list = NULL;
-       dw->dupes = NULL;
-       dw->match_mask = match_mask;
-       dw->show_thumbs = FALSE;
-
-       dw->idle_id = -1;
-
-       dw->second_set = FALSE;
+       dw->match_mask = DUPE_MATCH_NAME;
+       if (options->duplicates_match == DUPE_MATCH_NAME) dw->match_mask = DUPE_MATCH_NAME;
+       if (options->duplicates_match == DUPE_MATCH_SIZE) dw->match_mask = DUPE_MATCH_SIZE;
+       if (options->duplicates_match == DUPE_MATCH_DATE) dw->match_mask = DUPE_MATCH_DATE;
+       if (options->duplicates_match == DUPE_MATCH_DIM) dw->match_mask = DUPE_MATCH_DIM;
+       if (options->duplicates_match == DUPE_MATCH_SUM) dw->match_mask = DUPE_MATCH_SUM;
+       if (options->duplicates_match == DUPE_MATCH_PATH) dw->match_mask = DUPE_MATCH_PATH;
+       if (options->duplicates_match == DUPE_MATCH_SIM_HIGH) dw->match_mask = DUPE_MATCH_SIM_HIGH;
+       if (options->duplicates_match == DUPE_MATCH_SIM_MED) dw->match_mask = DUPE_MATCH_SIM_MED;
+       if (options->duplicates_match == DUPE_MATCH_SIM_LOW) dw->match_mask = DUPE_MATCH_SIM_LOW;
+       if (options->duplicates_match == DUPE_MATCH_SIM_CUSTOM) dw->match_mask = DUPE_MATCH_SIM_CUSTOM;
+       if (options->duplicates_match == DUPE_MATCH_NAME_CI) dw->match_mask = DUPE_MATCH_NAME_CI;
 
        dw->window = window_new(GTK_WINDOW_TOPLEVEL, "dupe", NULL, NULL, _("Find duplicates"));
-      
-       geometry.min_width = 32;
-       geometry.min_height = 32;
+
+       geometry.min_width = DEFAULT_MINIMAL_WINDOW_SIZE;
+       geometry.min_height = DEFAULT_MINIMAL_WINDOW_SIZE;
        geometry.base_width = DUPE_DEF_WIDTH;
        geometry.base_height = DUPE_DEF_HEIGHT;
        gtk_window_set_geometry_hints(GTK_WINDOW(dw->window), NULL, &geometry,
@@ -3115,9 +3241,9 @@ DupeWindow *dupe_window_new(DupeMatchType match_mask)
        gtk_window_set_default_size(GTK_WINDOW(dw->window), DUPE_DEF_WIDTH, DUPE_DEF_HEIGHT);
 
        gtk_window_set_resizable(GTK_WINDOW(dw->window), TRUE);
-        gtk_container_set_border_width (GTK_CONTAINER (dw->window), 0);
+       gtk_container_set_border_width(GTK_CONTAINER(dw->window), 0);
 
-        g_signal_connect(G_OBJECT(dw->window), "delete_event",
+       g_signal_connect(G_OBJECT(dw->window), "delete_event",
                         G_CALLBACK(dupe_window_delete), dw);
        g_signal_connect(G_OBJECT(dw->window), "key_press_event",
                         G_CALLBACK(dupe_window_keypress_cb), dw);
@@ -3134,7 +3260,7 @@ DupeWindow *dupe_window_new(DupeMatchType match_mask)
        gtk_scrolled_window_set_shadow_type(GTK_SCROLLED_WINDOW(scrolled), GTK_SHADOW_IN);
        gtk_scrolled_window_set_policy(GTK_SCROLLED_WINDOW(scrolled), GTK_POLICY_AUTOMATIC, GTK_POLICY_AUTOMATIC);
        gtk_table_attach_defaults(GTK_TABLE(dw->table), scrolled, 0, 2, 0, 1);
-        gtk_widget_show(scrolled);
+       gtk_widget_show(scrolled);
 
        store = gtk_list_store_new(9, G_TYPE_POINTER, G_TYPE_STRING, GDK_TYPE_PIXBUF,
                                   G_TYPE_STRING, G_TYPE_STRING, G_TYPE_STRING,
@@ -3174,7 +3300,7 @@ DupeWindow *dupe_window_new(DupeMatchType match_mask)
        gtk_scrolled_window_set_shadow_type(GTK_SCROLLED_WINDOW(scrolled), GTK_SHADOW_IN);
        gtk_scrolled_window_set_policy(GTK_SCROLLED_WINDOW(scrolled), GTK_POLICY_AUTOMATIC, GTK_POLICY_AUTOMATIC);
        gtk_box_pack_start(GTK_BOX(dw->second_vbox), scrolled, TRUE, TRUE, 0);
-        gtk_widget_show(scrolled);
+       gtk_widget_show(scrolled);
 
        store = gtk_list_store_new(2, G_TYPE_POINTER, G_TYPE_STRING);
        dw->second_listview = gtk_tree_view_new_with_model(GTK_TREE_MODEL(store));
@@ -3188,7 +3314,7 @@ DupeWindow *dupe_window_new(DupeMatchType match_mask)
        dupe_listview_add_column(dw, dw->second_listview, 1, _("Compare to:"), FALSE, FALSE);
 
        gtk_container_add(GTK_CONTAINER(scrolled), dw->second_listview);
-        gtk_widget_show(dw->second_listview);
+       gtk_widget_show(dw->second_listview);
 
        dw->second_status_label = gtk_label_new("");
        gtk_box_pack_start(GTK_BOX(dw->second_vbox), dw->second_status_label, FALSE, FALSE, 0);
@@ -3207,12 +3333,21 @@ DupeWindow *dupe_window_new(DupeMatchType match_mask)
        gtk_widget_show(dw->combo);
 
        dw->button_thumbs = gtk_check_button_new_with_label(_("Thumbnails"));
+       dw->show_thumbs = options->duplicates_thumbnails;
        gtk_toggle_button_set_active(GTK_TOGGLE_BUTTON(dw->button_thumbs), dw->show_thumbs);
        g_signal_connect(G_OBJECT(dw->button_thumbs), "toggled",
                         G_CALLBACK(dupe_window_show_thumb_cb), dw);
        gtk_box_pack_start(GTK_BOX(status_box), dw->button_thumbs, FALSE, FALSE, PREF_PAD_SPACE);
        gtk_widget_show(dw->button_thumbs);
 
+       dw->button_rotation_invariant = gtk_check_button_new_with_label(_("Ignore Rotation"));
+       gtk_widget_set_tooltip_text(GTK_WIDGET(dw->button_rotation_invariant), "Ignore image orientation");
+       gtk_toggle_button_set_active(GTK_TOGGLE_BUTTON(dw->button_rotation_invariant), options->rot_invariant_sim);
+       g_signal_connect(G_OBJECT(dw->button_rotation_invariant), "toggled",
+                        G_CALLBACK(dupe_window_rotation_invariant_cb), dw);
+       gtk_box_pack_start(GTK_BOX(status_box), dw->button_rotation_invariant, FALSE, FALSE, PREF_PAD_SPACE);
+       gtk_widget_show(dw->button_rotation_invariant);
+
        button = gtk_check_button_new_with_label(_("Compare two file sets"));
        gtk_toggle_button_set_active(GTK_TOGGLE_BUTTON(button), dw->second_set);
        g_signal_connect(G_OBJECT(button), "toggled",
@@ -3221,8 +3356,8 @@ DupeWindow *dupe_window_new(DupeMatchType match_mask)
        gtk_widget_show(button);
 
        status_box = gtk_hbox_new(FALSE, 0);
-        gtk_box_pack_start(GTK_BOX(vbox), status_box, FALSE, FALSE, 0);
-        gtk_widget_show(status_box);
+       gtk_box_pack_start(GTK_BOX(vbox), status_box, FALSE, FALSE, 0);
+       gtk_widget_show(status_box);
 
        frame = gtk_frame_new(NULL);
        gtk_frame_set_shadow_type(GTK_FRAME(frame), GTK_SHADOW_IN);
@@ -3233,8 +3368,31 @@ DupeWindow *dupe_window_new(DupeMatchType match_mask)
        gtk_container_add(GTK_CONTAINER(frame), dw->status_label);
        gtk_widget_show(dw->status_label);
 
+       button = gtk_check_button_new_with_label(_("Sort"));
+       gtk_widget_set_tooltip_text(GTK_WIDGET(button), "Sort by group totals");
+       gtk_toggle_button_set_active(GTK_TOGGLE_BUTTON(button), options->sort_totals);
+       g_signal_connect(G_OBJECT(button), "toggled",
+                        G_CALLBACK(dupe_sort_totals_toggle_cb), dw);
+       gtk_box_pack_start(GTK_BOX(status_box), button, FALSE, FALSE, PREF_PAD_SPACE);
+       gtk_widget_show(button);
+
+       label = gtk_label_new(_("Custom Threshold"));
+       gtk_box_pack_start(GTK_BOX(status_box), label, FALSE, FALSE, PREF_PAD_SPACE);
+       gtk_widget_show(label);
+       dw->custom_threshold = gtk_spin_button_new_with_range(1, 100, 1);
+       gtk_widget_set_tooltip_text(GTK_WIDGET(dw->custom_threshold), "Custom similarity threshold");
+       gtk_spin_button_set_value(GTK_SPIN_BUTTON(dw->custom_threshold), options->duplicates_similarity_threshold);
+       g_signal_connect(G_OBJECT(dw->custom_threshold), "value_changed",
+                                                                                                       G_CALLBACK(dupe_window_custom_threshold_cb), dw);
+       gtk_box_pack_start(GTK_BOX(status_box), dw->custom_threshold, FALSE, FALSE, PREF_PAD_SPACE);
+       gtk_widget_show(dw->custom_threshold);
+
        dw->extra_label = gtk_progress_bar_new();
        gtk_progress_bar_set_fraction(GTK_PROGRESS_BAR(dw->extra_label), 0.0);
+#if GTK_CHECK_VERSION(3,0,0)
+       gtk_progress_bar_set_text(GTK_PROGRESS_BAR(dw->extra_label), "");
+       gtk_progress_bar_set_show_text(GTK_PROGRESS_BAR(dw->extra_label), TRUE);
+#endif
        gtk_box_pack_end(GTK_BOX(status_box), dw->extra_label, FALSE, FALSE, 0);
        gtk_widget_show(dw->extra_label);
 
@@ -3259,6 +3417,8 @@ DupeWindow *dupe_window_new(DupeMatchType match_mask)
 
        dupe_window_list = g_list_append(dupe_window_list, dw);
 
+       file_data_register_notify_func(dupe_notify_cb, dw, NOTIFY_PRIORITY_MEDIUM);
+
        return dw;
 }
 
@@ -3292,9 +3452,9 @@ static void confirm_dir_list_add(GtkWidget *widget, gpointer data)
                work = work->next;
                if (isdir(fd->path))
                        {
-                       GList *list = NULL;
+                       GList *list;
 
-                       filelist_read(fd->path, &list, NULL);
+                       filelist_read(fd, &list, NULL);
                        list = filelist_filter(list, FALSE);
                        if (list)
                                {
@@ -3355,14 +3515,14 @@ static GtkWidget *dupe_confirm_dir_list(DupeWindow *dw, GList *list)
  */
 
 static GtkTargetEntry dupe_drag_types[] = {
-        { "text/uri-list", 0, TARGET_URI_LIST },
-        { "text/plain", 0, TARGET_TEXT_PLAIN }
+       { "text/uri-list", 0, TARGET_URI_LIST },
+       { "text/plain", 0, TARGET_TEXT_PLAIN }
 };
 static gint n_dupe_drag_types = 2;
 
 static GtkTargetEntry dupe_drop_types[] = {
-        { TARGET_APP_COLLECTION_MEMBER_STRING, 0, TARGET_APP_COLLECTION_MEMBER },
-        { "text/uri-list", 0, TARGET_URI_LIST }
+       { TARGET_APP_COLLECTION_MEMBER_STRING, 0, TARGET_APP_COLLECTION_MEMBER },
+       { "text/uri-list", 0, TARGET_URI_LIST }
 };
 static gint n_dupe_drop_types = 2;
 
@@ -3371,8 +3531,6 @@ static void dupe_dnd_data_set(GtkWidget *widget, GdkDragContext *context,
                              guint time, gpointer data)
 {
        DupeWindow *dw = data;
-       gchar *uri_text;
-       gint length;
        GList *list;
 
        switch (info)
@@ -3381,17 +3539,12 @@ static void dupe_dnd_data_set(GtkWidget *widget, GdkDragContext *context,
                case TARGET_TEXT_PLAIN:
                        list = dupe_listview_get_selection(dw, widget);
                        if (!list) return;
-                       uri_text = uri_text_from_filelist(list, &length, (info == TARGET_TEXT_PLAIN));
+                       uri_selection_data_set_uris_from_filelist(selection_data, list);
                        filelist_free(list);
                        break;
                default:
-                       uri_text = NULL;
                        break;
                }
-
-       if (uri_text) gtk_selection_data_set(selection_data, selection_data->target,
-                                            8, (guchar *)uri_text, length);
-       g_free(uri_text);
 }
 
 static void dupe_dnd_data_get(GtkWidget *widget, GdkDragContext *context,
@@ -3412,12 +3565,12 @@ static void dupe_dnd_data_get(GtkWidget *widget, GdkDragContext *context,
        switch (info)
                {
                case TARGET_APP_COLLECTION_MEMBER:
-                       collection_from_dnd_data((gchar *)selection_data->data, &list, NULL);
+                       collection_from_dnd_data((gchar *)gtk_selection_data_get_data(selection_data), &list, NULL);
                        break;
                case TARGET_URI_LIST:
-                       list = uri_filelist_from_text((gchar *)selection_data->data, TRUE);
+                       list = uri_filelist_from_gtk_selection_data(selection_data);
                        work = list;
-                       while(work)
+                       while (work)
                                {
                                FileData *fd = work->data;
                                if (isdir(fd->path))
@@ -3442,7 +3595,7 @@ static void dupe_dnd_data_get(GtkWidget *widget, GdkDragContext *context,
                }
 }
 
-static void dupe_dest_set(GtkWidget *widget, gint enable)
+static void dupe_dest_set(GtkWidget *widget, gboolean enable)
 {
        if (enable)
                {
@@ -3542,32 +3695,29 @@ static void dupe_dnd_init(DupeWindow *dw)
  *-------------------------------------------------------------------
  */
 
-void dupe_maint_removed(FileData *fd)
+static void dupe_notify_cb(FileData *fd, NotifyType type, gpointer data)
 {
-       GList *work;
-
-       work = dupe_window_list;
-       while (work)
-               {
-               DupeWindow *dw = work->data;
-               work = work->next;
+       DupeWindow *dw = data;
 
-               while (dupe_item_remove_by_path(dw, fd->path));
-               }
-}
+       if (!(type & NOTIFY_CHANGE) || !fd->change) return;
 
-void dupe_maint_renamed(FileData *fd)
-{
-       GList *work;
+       DEBUG_1("Notify dupe: %s %04x", fd->path, type);
 
-       work = dupe_window_list;
-       while (work)
+       switch (fd->change->type)
                {
-               DupeWindow *dw = work->data;
-               work = work->next;
-
-               dupe_item_update_fd(dw, fd);
+               case FILEDATA_CHANGE_MOVE:
+               case FILEDATA_CHANGE_RENAME:
+                       dupe_item_update_fd(dw, fd);
+                       break;
+               case FILEDATA_CHANGE_COPY:
+                       break;
+               case FILEDATA_CHANGE_DELETE:
+                       while (dupe_item_remove_by_path(dw, fd->path));
+                       break;
+               case FILEDATA_CHANGE_UNSPECIFIED:
+               case FILEDATA_CHANGE_WRITE_METADATA:
+                       break;
                }
 
 }
-
+/* vim: set shiftwidth=8 softtabstop=0 cindent cinoptions={1s: */