Find duplicates - blocking when loading file list

Geeqie blocks when loading a list of files for find duplicates.
Using recurse can result in thousands of files being loaded, which
creates a problem.
This patch loads the file list in the idle loop.

diff --git a/src/dupe.c b/src/dupe.c
index d64f69d..23f6000 100644 (file)
--- a/src/dupe.c
+++ b/src/dupe.c
@@ -58,6 +58,7 @@
 
 #define DUPE_DEF_WIDTH 800
 #define DUPE_DEF_HEIGHT 400
+#define DUPE_PROGRESS_PULSE_STEP 0.0001
 
 /* column assignment order (simply change them here) */
 enum {
@@ -1469,6 +1470,20 @@ static void dupe_check_stop(DupeWindow *dw)
                widget_set_cursor(dw->listview, -1);
                }
 
+       if (dw->add_files_queue_id)
+               {
+               g_source_remove(dw->add_files_queue_id);
+               dw->add_files_queue_id = 0;
+               gtk_widget_set_sensitive(dw->controls_box, TRUE);
+               if (g_list_length(dw->add_files_queue) > 0)
+                       {
+                       filelist_free(dw->add_files_queue);
+                       }
+               dw->add_files_queue = NULL;
+               dupe_window_update_progress(dw, NULL, 0.0, FALSE);
+               widget_set_cursor(dw->listview, -1);
+               }
+
        thumb_loader_free(dw->thumb_loader);
        dw->thumb_loader = NULL;
 
@@ -1725,6 +1740,15 @@ static void dupe_check_start(DupeWindow *dw)
        dw->idle_id = g_idle_add(dupe_check_cb, dw);
 }
 
+static gboolean dupe_check_start_cb(gpointer data)
+{
+       DupeWindow *dw = data;
+
+       dupe_check_start(dw);
+
+       return FALSE;
+}
+
 /*
  * ------------------------------------------------------------------
  * Item addition, removal
@@ -1829,6 +1853,131 @@ static gboolean dupe_item_remove_by_path(DupeWindow *dw, const gchar *path)
        return TRUE;
 }
 
+static gboolean dupe_files_add_queue_cb(gpointer data)
+{
+       DupeItem *di = NULL;
+       DupeWindow *dw = data;
+       FileData *fd;
+       GList *queue = dw->add_files_queue;
+
+       gtk_progress_bar_pulse(GTK_PROGRESS_BAR(dw->extra_label));
+
+       if (g_list_length(queue) == 0)
+               {
+               dw->add_files_queue_id = 0;
+               g_idle_add(dupe_check_start_cb, dw);
+               gtk_widget_set_sensitive(dw->controls_box, TRUE);
+               return FALSE;
+               }
+
+       fd = queue->data;
+       if (fd)
+               {
+               if (isfile(fd->path))
+                       {
+                       di = dupe_item_new(fd);
+                       }
+               else if (isdir(fd->path))
+                       {
+                       GList *f, *d;
+                       dw->add_files_queue = g_list_remove(dw->add_files_queue, g_list_first(dw->add_files_queue)->data);
+
+                       if (filelist_read(fd, &f, &d))
+                               {
+                               GList *work;
+
+                               f = filelist_filter(f, FALSE);
+                               d = filelist_filter(d, TRUE);
+
+                               work = f;
+                               while (work)
+                                       {
+                                       dw->add_files_queue = g_list_prepend(dw->add_files_queue, work->data);
+                                       work = work->next;
+                                       }
+                               g_list_free(f);
+                               work = d;
+                               while (work)
+                                       {
+                                       dw->add_files_queue = g_list_prepend(dw->add_files_queue, work->data);
+                                       work = work->next;
+                                       }
+                               g_list_free(d);
+                               }
+                       }
+               else
+                       {
+                       /* Not a file and not a dir */
+                       dw->add_files_queue = g_list_remove(dw->add_files_queue, g_list_first(dw->add_files_queue)->data);
+                       }
+               }
+
+       if (!di)
+               {
+               /* A dir was found. Process the contents on next entry */
+               return TRUE;
+               }
+
+       dw->add_files_queue = g_list_remove(dw->add_files_queue, g_list_first(dw->add_files_queue)->data);
+
+       dupe_item_read_cache(di);
+
+       /* Ensure images in the lists have unique FileDatas */
+       GList *work;
+       DupeItem *di_list;
+       work = g_list_first(dw->list);
+       while (work)
+               {
+               di_list = work->data;
+               if (di_list->fd == di->fd)
+                       {
+                       return TRUE;
+                       }
+               else
+                       {
+                       work = work->next;
+                       }
+               }
+
+       if (dw->second_list)
+               {
+               work = g_list_first(dw->second_list);
+               while (work)
+                       {
+                       di_list = work->data;
+                       if (di_list->fd == di->fd)
+                               {
+                               return TRUE;
+                               }
+                       else
+                               {
+                               work = work->next;
+                               }
+                       }
+               }
+
+       if (dw->second_drop)
+               {
+               dupe_second_add(dw, di);
+               }
+       else
+               {
+               dw->list = g_list_prepend(dw->list, di);
+               }
+
+       if (g_list_length(dw->add_files_queue) > 0)
+               {
+               return TRUE;
+               }
+       else
+               {
+               dw->add_files_queue_id = 0;
+               g_idle_add(dupe_check_start_cb, dw);
+               gtk_widget_set_sensitive(dw->controls_box, TRUE);
+               return FALSE;
+               }
+}
+
 static void dupe_files_add(DupeWindow *dw, CollectionData *collection, CollectInfo *info,
                           FileData *fd, gboolean recurse)
 {
@@ -1943,11 +2092,39 @@ void dupe_window_add_files(DupeWindow *dw, GList *list, gboolean recurse)
                {
                FileData *fd = work->data;
                work = work->next;
+               if (isdir(fd->path) && !recurse)
+                       {
+                       GList *f, *d;
+
+                       if (filelist_read(fd, &f, &d))
+                               {
+                               GList *work_file;
+                               work_file = f;
 
-               dupe_files_add(dw, NULL, NULL, fd, recurse);
+                               while (work_file)
+                                       {
+                                       /* Add only the files, ignore the dirs when no recurse */
+                                       dw->add_files_queue = g_list_prepend(dw->add_files_queue, work_file->data);
+                                       work_file = work_file->next;
+                                       }
+                               g_list_free(f);
+                               g_list_free(d);
+                               }
+                       }
+               else
+                       {
+                       dw->add_files_queue = g_list_prepend(dw->add_files_queue, fd);
+                       }
                }
+       if (dw->add_files_queue_id == 0)
+               {
+               gtk_progress_bar_pulse(GTK_PROGRESS_BAR(dw->extra_label));
+               gtk_progress_bar_set_pulse_step(GTK_PROGRESS_BAR(dw->extra_label), DUPE_PROGRESS_PULSE_STEP);
+               gtk_progress_bar_set_text(GTK_PROGRESS_BAR(dw->extra_label), _("Loading file list"));
 
-       dupe_check_start(dw);
+               dw->add_files_queue_id = g_idle_add(dupe_files_add_queue_cb, dw);
+               gtk_widget_set_sensitive(dw->controls_box, FALSE);
+               }
 }
 
 static void dupe_item_update(DupeWindow *dw, DupeItem *di)
@@ -3501,6 +3678,7 @@ DupeWindow *dupe_window_new()
        GtkWidget *scrolled;
        GtkWidget *frame;
        GtkWidget *status_box;
+       GtkWidget *controls_box;
        GtkWidget *button_box;
        GtkWidget *label;
        GtkWidget *button;
@@ -3512,6 +3690,8 @@ DupeWindow *dupe_window_new()
        layout_valid(&lw);
 
        dw = g_new0(DupeWindow, 1);
+       dw->add_files_queue = NULL;
+       dw->add_files_queue_id = 0;
 
        dw->match_mask = DUPE_MATCH_NAME;
        if (options->duplicates_match == DUPE_MATCH_NAME) dw->match_mask = DUPE_MATCH_NAME;
@@ -3671,39 +3851,40 @@ DupeWindow *dupe_window_new()
        gtk_box_pack_start(GTK_BOX(status_box), dw->extra_label, FALSE, FALSE, PREF_PAD_SPACE);
        gtk_widget_show(dw->extra_label);
 
-       status_box = pref_box_new(vbox, FALSE, GTK_ORIENTATION_HORIZONTAL, 0);
+       controls_box = pref_box_new(vbox, FALSE, GTK_ORIENTATION_HORIZONTAL, 0);
+       dw->controls_box = controls_box;
 
        dw->button_thumbs = gtk_check_button_new_with_label(_("Thumbnails"));
        dw->show_thumbs = options->duplicates_thumbnails;
        gtk_toggle_button_set_active(GTK_TOGGLE_BUTTON(dw->button_thumbs), dw->show_thumbs);
        g_signal_connect(G_OBJECT(dw->button_thumbs), "toggled",
                         G_CALLBACK(dupe_window_show_thumb_cb), dw);
-       gtk_box_pack_start(GTK_BOX(status_box), dw->button_thumbs, FALSE, FALSE, PREF_PAD_SPACE);
+       gtk_box_pack_start(GTK_BOX(controls_box), dw->button_thumbs, FALSE, FALSE, PREF_PAD_SPACE);
        gtk_widget_show(dw->button_thumbs);
 
        label = gtk_label_new(_("Compare by:"));
-       gtk_box_pack_start(GTK_BOX(status_box), label, FALSE, FALSE, PREF_PAD_SPACE);
+       gtk_box_pack_start(GTK_BOX(controls_box), label, FALSE, FALSE, PREF_PAD_SPACE);
        gtk_widget_show(label);
 
        dupe_menu_setup(dw);
-       gtk_box_pack_start(GTK_BOX(status_box), dw->combo, FALSE, FALSE, 0);
+       gtk_box_pack_start(GTK_BOX(controls_box), dw->combo, FALSE, FALSE, 0);
        gtk_widget_show(dw->combo);
 
        label = gtk_label_new(_("Custom Threshold"));
-       gtk_box_pack_start(GTK_BOX(status_box), label, FALSE, FALSE, PREF_PAD_SPACE);
+       gtk_box_pack_start(GTK_BOX(controls_box), label, FALSE, FALSE, PREF_PAD_SPACE);
        gtk_widget_show(label);
        dw->custom_threshold = gtk_spin_button_new_with_range(1, 100, 1);
        gtk_widget_set_tooltip_text(GTK_WIDGET(dw->custom_threshold), "Custom similarity threshold");
        gtk_spin_button_set_value(GTK_SPIN_BUTTON(dw->custom_threshold), options->duplicates_similarity_threshold);
        g_signal_connect(G_OBJECT(dw->custom_threshold), "value_changed", G_CALLBACK(dupe_window_custom_threshold_cb), dw);
-       gtk_box_pack_start(GTK_BOX(status_box), dw->custom_threshold, FALSE, FALSE, PREF_PAD_SPACE);
+       gtk_box_pack_start(GTK_BOX(controls_box), dw->custom_threshold, FALSE, FALSE, PREF_PAD_SPACE);
        gtk_widget_show(dw->custom_threshold);
 
        button = gtk_check_button_new_with_label(_("Sort"));
        gtk_widget_set_tooltip_text(GTK_WIDGET(button), "Sort by group totals");
        gtk_toggle_button_set_active(GTK_TOGGLE_BUTTON(button), options->sort_totals);
        g_signal_connect(G_OBJECT(button), "toggled", G_CALLBACK(dupe_sort_totals_toggle_cb), dw);
-       gtk_box_pack_start(GTK_BOX(status_box), button, FALSE, FALSE, PREF_PAD_SPACE);
+       gtk_box_pack_start(GTK_BOX(controls_box), button, FALSE, FALSE, PREF_PAD_SPACE);
        gtk_widget_show(button);
 
        dw->button_rotation_invariant = gtk_check_button_new_with_label(_("Ignore Orientation"));
@@ -3711,14 +3892,14 @@ DupeWindow *dupe_window_new()
        gtk_toggle_button_set_active(GTK_TOGGLE_BUTTON(dw->button_rotation_invariant), options->rot_invariant_sim);
        g_signal_connect(G_OBJECT(dw->button_rotation_invariant), "toggled",
                         G_CALLBACK(dupe_window_rotation_invariant_cb), dw);
-       gtk_box_pack_start(GTK_BOX(status_box), dw->button_rotation_invariant, FALSE, FALSE, PREF_PAD_SPACE);
+       gtk_box_pack_start(GTK_BOX(controls_box), dw->button_rotation_invariant, FALSE, FALSE, PREF_PAD_SPACE);
        gtk_widget_show(dw->button_rotation_invariant);
 
        button = gtk_check_button_new_with_label(_("Compare two file sets"));
        gtk_toggle_button_set_active(GTK_TOGGLE_BUTTON(button), dw->second_set);
        g_signal_connect(G_OBJECT(button), "toggled",
                         G_CALLBACK(dupe_second_set_toggle_cb), dw);
-       gtk_box_pack_start(GTK_BOX(status_box), button, FALSE, FALSE, PREF_PAD_SPACE);
+       gtk_box_pack_start(GTK_BOX(controls_box), button, FALSE, FALSE, PREF_PAD_SPACE);
        gtk_widget_show(button);
 
        button_box = gtk_hbox_new(FALSE, 0);
@@ -3910,6 +4091,13 @@ static void dupe_dnd_data_get(GtkWidget *widget, GdkDragContext *context,
        GList *list = NULL;
        GList *work;
 
+       if (dw->add_files_queue_id > 0)
+               {
+               warning_dialog(_("Find duplicates"), _("Please wait for the current file selection to be loaded."), GTK_STOCK_DIALOG_INFO, dw->window);
+
+               return;
+               }
+
        source = gtk_drag_get_source_widget(context);
        if (source == dw->listview || source == dw->second_listview) return;
 

diff --git a/src/dupe.h b/src/dupe.h
index ec33b2a..d897022 100644 (file)
--- a/src/dupe.h
+++ b/src/dupe.h
@@ -97,6 +97,9 @@ struct _DupeWindow
        GtkWidget *button_thumbs;
        GtkWidget *button_rotation_invariant;
        GtkWidget *custom_threshold;
+       GList *add_files_queue;
+       guint add_files_queue_id;
+       GtkWidget *controls_box;
 
        gboolean show_thumbs;