diff --git a/src/manager.c b/src/manager.c
index 2c1c32e..cc2a378 100644
--- a/src/manager.c
+++ b/src/manager.c
@@ -95,10 +95,12 @@ fprint_manager_in_use_notified (FprintDevice *rdev, GParamSpec *spec, FprintMana
 	GSList *l;
 	gboolean in_use;
 
-	if (priv->timeout_id > 0) {
+	if (priv->timeout_id > 0 || !priv->no_timeout) {
 		g_source_remove (priv->timeout_id);
 		priv->timeout_id = 0;
 	}
+	if (!priv->no_timeout)
+		return;
 
 	for (l = priv->dev_registry; l != NULL; l = l->next) {
 		FprintDevice *dev = l->data;
@@ -142,9 +144,6 @@ fprint_manager_init (FprintManager *manager)
 			G_OBJECT(rdev));
 		g_free(path);
 	}
-
-	if (!priv->no_timeout)
-		priv->timeout_id = g_timeout_add_seconds (TIMEOUT, (GSourceFunc) fprint_manager_timeout_cb, manager);
 }
 
 FprintManager *fprint_manager_new(gboolean no_timeout)
@@ -156,6 +155,9 @@ FprintManager *fprint_manager_new(gboolean no_timeout)
 	priv = FPRINT_MANAGER_GET_PRIVATE (object);
 	priv->no_timeout = no_timeout;
 
+	if (!priv->no_timeout)
+		priv->timeout_id = g_timeout_add_seconds (TIMEOUT, (GSourceFunc) fprint_manager_timeout_cb, object);
+
 	return FPRINT_MANAGER (object);
 }