From: Colin Clark <colin.clark@cclark.uk>
Date: Mon, 24 Aug 2020 14:43:07 +0000 (+0100)
Subject: Find duplicates - blocking when loading file list
X-Git-Tag: v1.6~24
X-Git-Url: http://geeqie.org/cgi-bin/gitweb.cgi?p=geeqie.git;a=commitdiff_plain;h=5f2afc4b57f94823105e343a844450fcf175a6db

Find duplicates - blocking when loading file list

Geeqie blocks when loading a list of files for find duplicates.
Using recurse can result in thousands of files being loaded, which
creates a problem.
This patch loads the file list in the idle loop.
---

diff --git a/src/dupe.c b/src/dupe.c
index d64f69de..23f60000 100644
--- a/src/dupe.c
+++ b/src/dupe.c
@@ -58,6 +58,7 @@
 
 #define DUPE_DEF_WIDTH 800
 #define DUPE_DEF_HEIGHT 400
+#define DUPE_PROGRESS_PULSE_STEP 0.0001
 
 /* column assignment order (simply change them here) */
 enum {
@@ -1469,6 +1470,20 @@ static void dupe_check_stop(DupeWindow *dw)
 		widget_set_cursor(dw->listview, -1);
 		}
 
+	if (dw->add_files_queue_id)
+		{
+		g_source_remove(dw->add_files_queue_id);
+		dw->add_files_queue_id = 0;
+		gtk_widget_set_sensitive(dw->controls_box, TRUE);
+		if (g_list_length(dw->add_files_queue) > 0)
+			{
+			filelist_free(dw->add_files_queue);
+			}
+		dw->add_files_queue = NULL;
+		dupe_window_update_progress(dw, NULL, 0.0, FALSE);
+		widget_set_cursor(dw->listview, -1);
+		}
+
 	thumb_loader_free(dw->thumb_loader);
 	dw->thumb_loader = NULL;
 
@@ -1725,6 +1740,15 @@ static void dupe_check_start(DupeWindow *dw)
 	dw->idle_id = g_idle_add(dupe_check_cb, dw);
 }
 
+static gboolean dupe_check_start_cb(gpointer data)
+{
+	DupeWindow *dw = data;
+
+	dupe_check_start(dw);
+
+	return FALSE;
+}
+
 /*
  * ------------------------------------------------------------------
  * Item addition, removal
@@ -1829,6 +1853,131 @@ static gboolean dupe_item_remove_by_path(DupeWindow *dw, const gchar *path)
 	return TRUE;
 }
 
+static gboolean dupe_files_add_queue_cb(gpointer data)
+{
+	DupeItem *di = NULL;
+	DupeWindow *dw = data;
+	FileData *fd;
+	GList *queue = dw->add_files_queue;
+
+	gtk_progress_bar_pulse(GTK_PROGRESS_BAR(dw->extra_label));
+
+	if (g_list_length(queue) == 0)
+		{
+		dw->add_files_queue_id = 0;
+		g_idle_add(dupe_check_start_cb, dw);
+		gtk_widget_set_sensitive(dw->controls_box, TRUE);
+		return FALSE;
+		}
+
+	fd = queue->data;
+	if (fd)
+		{
+		if (isfile(fd->path))
+			{
+			di = dupe_item_new(fd);
+			}
+		else if (isdir(fd->path))
+			{
+			GList *f, *d;
+			dw->add_files_queue = g_list_remove(dw->add_files_queue, g_list_first(dw->add_files_queue)->data);
+
+			if (filelist_read(fd, &f, &d))
+				{
+				GList *work;
+
+				f = filelist_filter(f, FALSE);
+				d = filelist_filter(d, TRUE);
+
+				work = f;
+				while (work)
+					{
+					dw->add_files_queue = g_list_prepend(dw->add_files_queue, work->data);
+					work = work->next;
+					}
+				g_list_free(f);
+				work = d;
+				while (work)
+					{
+					dw->add_files_queue = g_list_prepend(dw->add_files_queue, work->data);
+					work = work->next;
+					}
+				g_list_free(d);
+				}
+			}
+		else
+			{
+			/* Not a file and not a dir */
+			dw->add_files_queue = g_list_remove(dw->add_files_queue, g_list_first(dw->add_files_queue)->data);
+			}
+		}
+
+	if (!di)
+		{
+		/* A dir was found. Process the contents on next entry */
+		return TRUE;
+		}
+
+	dw->add_files_queue = g_list_remove(dw->add_files_queue, g_list_first(dw->add_files_queue)->data);
+
+	dupe_item_read_cache(di);
+
+	/* Ensure images in the lists have unique FileDatas */
+	GList *work;
+	DupeItem *di_list;
+	work = g_list_first(dw->list);
+	while (work)
+		{
+		di_list = work->data;
+		if (di_list->fd == di->fd)
+			{
+			return TRUE;
+			}
+		else
+			{
+			work = work->next;
+			}
+		}
+
+	if (dw->second_list)
+		{
+		work = g_list_first(dw->second_list);
+		while (work)
+			{
+			di_list = work->data;
+			if (di_list->fd == di->fd)
+				{
+				return TRUE;
+				}
+			else
+				{
+				work = work->next;
+				}
+			}
+		}
+
+	if (dw->second_drop)
+		{
+		dupe_second_add(dw, di);
+		}
+	else
+		{
+		dw->list = g_list_prepend(dw->list, di);
+		}
+
+	if (g_list_length(dw->add_files_queue) > 0)
+		{
+		return TRUE;
+		}
+	else
+		{
+		dw->add_files_queue_id = 0;
+		g_idle_add(dupe_check_start_cb, dw);
+		gtk_widget_set_sensitive(dw->controls_box, TRUE);
+		return FALSE;
+		}
+}
+
 static void dupe_files_add(DupeWindow *dw, CollectionData *collection, CollectInfo *info,
 			   FileData *fd, gboolean recurse)
 {
@@ -1943,11 +2092,39 @@ void dupe_window_add_files(DupeWindow *dw, GList *list, gboolean recurse)
 		{
 		FileData *fd = work->data;
 		work = work->next;
+		if (isdir(fd->path) && !recurse)
+			{
+			GList *f, *d;
+
+			if (filelist_read(fd, &f, &d))
+				{
+				GList *work_file;
+				work_file = f;
 
-		dupe_files_add(dw, NULL, NULL, fd, recurse);
+				while (work_file)
+					{
+					/* Add only the files, ignore the dirs when no recurse */
+					dw->add_files_queue = g_list_prepend(dw->add_files_queue, work_file->data);
+					work_file = work_file->next;
+					}
+				g_list_free(f);
+				g_list_free(d);
+				}
+			}
+		else
+			{
+			dw->add_files_queue = g_list_prepend(dw->add_files_queue, fd);
+			}
 		}
+	if (dw->add_files_queue_id == 0)
+		{
+		gtk_progress_bar_pulse(GTK_PROGRESS_BAR(dw->extra_label));
+		gtk_progress_bar_set_pulse_step(GTK_PROGRESS_BAR(dw->extra_label), DUPE_PROGRESS_PULSE_STEP);
+		gtk_progress_bar_set_text(GTK_PROGRESS_BAR(dw->extra_label), _("Loading file list"));
 
-	dupe_check_start(dw);
+		dw->add_files_queue_id = g_idle_add(dupe_files_add_queue_cb, dw);
+		gtk_widget_set_sensitive(dw->controls_box, FALSE);
+		}
 }
 
 static void dupe_item_update(DupeWindow *dw, DupeItem *di)
@@ -3501,6 +3678,7 @@ DupeWindow *dupe_window_new()
 	GtkWidget *scrolled;
 	GtkWidget *frame;
 	GtkWidget *status_box;
+	GtkWidget *controls_box;
 	GtkWidget *button_box;
 	GtkWidget *label;
 	GtkWidget *button;
@@ -3512,6 +3690,8 @@ DupeWindow *dupe_window_new()
 	layout_valid(&lw);
 
 	dw = g_new0(DupeWindow, 1);
+	dw->add_files_queue = NULL;
+	dw->add_files_queue_id = 0;
 
 	dw->match_mask = DUPE_MATCH_NAME;
 	if (options->duplicates_match == DUPE_MATCH_NAME) dw->match_mask = DUPE_MATCH_NAME;
@@ -3671,39 +3851,40 @@ DupeWindow *dupe_window_new()
 	gtk_box_pack_start(GTK_BOX(status_box), dw->extra_label, FALSE, FALSE, PREF_PAD_SPACE);
 	gtk_widget_show(dw->extra_label);
 
-	status_box = pref_box_new(vbox, FALSE, GTK_ORIENTATION_HORIZONTAL, 0);
+	controls_box = pref_box_new(vbox, FALSE, GTK_ORIENTATION_HORIZONTAL, 0);
+	dw->controls_box = controls_box;
 
 	dw->button_thumbs = gtk_check_button_new_with_label(_("Thumbnails"));
 	dw->show_thumbs = options->duplicates_thumbnails;
 	gtk_toggle_button_set_active(GTK_TOGGLE_BUTTON(dw->button_thumbs), dw->show_thumbs);
 	g_signal_connect(G_OBJECT(dw->button_thumbs), "toggled",
 			 G_CALLBACK(dupe_window_show_thumb_cb), dw);
-	gtk_box_pack_start(GTK_BOX(status_box), dw->button_thumbs, FALSE, FALSE, PREF_PAD_SPACE);
+	gtk_box_pack_start(GTK_BOX(controls_box), dw->button_thumbs, FALSE, FALSE, PREF_PAD_SPACE);
 	gtk_widget_show(dw->button_thumbs);
 
 	label = gtk_label_new(_("Compare by:"));
-	gtk_box_pack_start(GTK_BOX(status_box), label, FALSE, FALSE, PREF_PAD_SPACE);
+	gtk_box_pack_start(GTK_BOX(controls_box), label, FALSE, FALSE, PREF_PAD_SPACE);
 	gtk_widget_show(label);
 
 	dupe_menu_setup(dw);
-	gtk_box_pack_start(GTK_BOX(status_box), dw->combo, FALSE, FALSE, 0);
+	gtk_box_pack_start(GTK_BOX(controls_box), dw->combo, FALSE, FALSE, 0);
 	gtk_widget_show(dw->combo);
 
 	label = gtk_label_new(_("Custom Threshold"));
-	gtk_box_pack_start(GTK_BOX(status_box), label, FALSE, FALSE, PREF_PAD_SPACE);
+	gtk_box_pack_start(GTK_BOX(controls_box), label, FALSE, FALSE, PREF_PAD_SPACE);
 	gtk_widget_show(label);
 	dw->custom_threshold = gtk_spin_button_new_with_range(1, 100, 1);
 	gtk_widget_set_tooltip_text(GTK_WIDGET(dw->custom_threshold), "Custom similarity threshold");
 	gtk_spin_button_set_value(GTK_SPIN_BUTTON(dw->custom_threshold), options->duplicates_similarity_threshold);
 	g_signal_connect(G_OBJECT(dw->custom_threshold), "value_changed", G_CALLBACK(dupe_window_custom_threshold_cb), dw);
-	gtk_box_pack_start(GTK_BOX(status_box), dw->custom_threshold, FALSE, FALSE, PREF_PAD_SPACE);
+	gtk_box_pack_start(GTK_BOX(controls_box), dw->custom_threshold, FALSE, FALSE, PREF_PAD_SPACE);
 	gtk_widget_show(dw->custom_threshold);
 
 	button = gtk_check_button_new_with_label(_("Sort"));
 	gtk_widget_set_tooltip_text(GTK_WIDGET(button), "Sort by group totals");
 	gtk_toggle_button_set_active(GTK_TOGGLE_BUTTON(button), options->sort_totals);
 	g_signal_connect(G_OBJECT(button), "toggled", G_CALLBACK(dupe_sort_totals_toggle_cb), dw);
-	gtk_box_pack_start(GTK_BOX(status_box), button, FALSE, FALSE, PREF_PAD_SPACE);
+	gtk_box_pack_start(GTK_BOX(controls_box), button, FALSE, FALSE, PREF_PAD_SPACE);
 	gtk_widget_show(button);
 
 	dw->button_rotation_invariant = gtk_check_button_new_with_label(_("Ignore Orientation"));
@@ -3711,14 +3892,14 @@ DupeWindow *dupe_window_new()
 	gtk_toggle_button_set_active(GTK_TOGGLE_BUTTON(dw->button_rotation_invariant), options->rot_invariant_sim);
 	g_signal_connect(G_OBJECT(dw->button_rotation_invariant), "toggled",
 			 G_CALLBACK(dupe_window_rotation_invariant_cb), dw);
-	gtk_box_pack_start(GTK_BOX(status_box), dw->button_rotation_invariant, FALSE, FALSE, PREF_PAD_SPACE);
+	gtk_box_pack_start(GTK_BOX(controls_box), dw->button_rotation_invariant, FALSE, FALSE, PREF_PAD_SPACE);
 	gtk_widget_show(dw->button_rotation_invariant);
 
 	button = gtk_check_button_new_with_label(_("Compare two file sets"));
 	gtk_toggle_button_set_active(GTK_TOGGLE_BUTTON(button), dw->second_set);
 	g_signal_connect(G_OBJECT(button), "toggled",
 			 G_CALLBACK(dupe_second_set_toggle_cb), dw);
-	gtk_box_pack_start(GTK_BOX(status_box), button, FALSE, FALSE, PREF_PAD_SPACE);
+	gtk_box_pack_start(GTK_BOX(controls_box), button, FALSE, FALSE, PREF_PAD_SPACE);
 	gtk_widget_show(button);
 
 	button_box = gtk_hbox_new(FALSE, 0);
@@ -3910,6 +4091,13 @@ static void dupe_dnd_data_get(GtkWidget *widget, GdkDragContext *context,
 	GList *list = NULL;
 	GList *work;
 
+	if (dw->add_files_queue_id > 0)
+		{
+		warning_dialog(_("Find duplicates"), _("Please wait for the current file selection to be loaded."), GTK_STOCK_DIALOG_INFO, dw->window);
+
+		return;
+		}
+
 	source = gtk_drag_get_source_widget(context);
 	if (source == dw->listview || source == dw->second_listview) return;
 
diff --git a/src/dupe.h b/src/dupe.h
index ec33b2a7..d8970221 100644
--- a/src/dupe.h
+++ b/src/dupe.h
@@ -97,6 +97,9 @@ struct _DupeWindow
 	GtkWidget *button_thumbs;
 	GtkWidget *button_rotation_invariant;
 	GtkWidget *custom_threshold;
+	GList *add_files_queue;
+	guint add_files_queue_id;
+	GtkWidget *controls_box;
 
 	gboolean show_thumbs;