diff --git a/src/device.c b/src/device.c
index 010bfc5..49cbdd5 100644
--- a/src/device.c
+++ b/src/device.c
@@ -54,6 +54,11 @@ static gboolean action_authorization_handler (GDBusInterfaceSkeleton *,
                                               GDBusMethodInvocation *,
                                               gpointer user_data);
 
+static gboolean delete_enrolled_fingers (FprintDevice *rdev,
+                                         const char   *user,
+                                         FpFinger      finger,
+                                         GError      **error);
+
 static GQuark quark_auth_user = 0;
 
 typedef enum {
@@ -1979,22 +1984,21 @@ fprint_device_enroll_start (FprintDBusDevice      *dbus_dev,
   store.print_data_load (priv->dev, finger,
                          session->username, &existing_print);
 
-  if (existing_print)
-    {
-      g_set_error (&error, FPRINT_ERROR, FPRINT_ERROR_FINGER_ALREADY_ENROLLED,
-                   "Finger %d has already been enrolled for user %s", finger, session->username);
-      g_dbus_method_invocation_return_gerror (invocation,
-                                              error);
-      return TRUE;
-    }
-
-
   if (!can_start_action (rdev, &error))
     {
       g_dbus_method_invocation_return_gerror (invocation, error);
       return TRUE;
     }
 
+  if (existing_print)
+    {
+      if (!delete_enrolled_fingers (rdev, session->username, finger, &error))
+        {
+          g_dbus_method_invocation_return_gerror (invocation, error);
+          return TRUE;
+        }
+    }
+
   g_debug ("start enrollment device %d finger %d", priv->id, finger);
 
   priv->current_cancellable = g_cancellable_new ();
diff --git a/tests/fprintd.py b/tests/fprintd.py
index e35ac88..1f8faee 100644
--- a/tests/fprintd.py
+++ b/tests/fprintd.py
@@ -2315,9 +2315,21 @@ class FPrintdVirtualDeviceEnrollTests(FPrintdVirtualDeviceBaseTest):
     def test_enroll_already_enrolled_finger(self):
         self.enroll_image('whorl', start=False)
 
+        # We can enroll a new image deleting the first
+        self.device.EnrollStart('(s)', 'left-middle-finger')
+        self.enroll_image('arch', start=False)
         self.stop_on_teardown = False
-        with self.assertFprintError('FingerAlreadyEnrolled'):
-            self.device.EnrollStart('(s)', 'left-middle-finger')
+
+        # If we verify, 'arch' will match, 'whorl' will not match
+        self.device.VerifyStart('(s)', 'any')
+        self.send_image('whorl')
+        self.assertVerifyNoMatch()
+        self.device.VerifyStop()
+
+        self.device.VerifyStart('(s)', 'any')
+        self.send_image('arch')
+        self.assertVerifyMatch()
+        self.device.VerifyStop()
 
     def test_enroll_duplicate_image(self):
         self.enroll_image('whorl', finger='left-thumb', start=False)