diff --git a/pam/pam_fprintd.c b/pam/pam_fprintd.c
index a4278f7..5dd7e98 100644
--- a/pam/pam_fprintd.c
+++ b/pam/pam_fprintd.c
@@ -455,7 +455,6 @@ do_verify (sd_bus *bus, verify_data *data)
 
   sigemptyset (&signals);
   sigaddset (&signals, SIGINT);
-  signal (SIGUSR1, handle_sigusr1);
   sigaddset (&signals, SIGUSR1);
   signal_fd = signalfd (signal_fd, &signals, SFD_NONBLOCK);
 
@@ -786,6 +785,8 @@ do_auth (pam_handle_t *pamh, const char *username)
       data->stop_got_pw = false;
       data->ppid = getpid();
 
+      signal (SIGUSR1, handle_sigusr1);
+
       pthread_t pw_prompt_thread;
       if (pthread_create (&pw_prompt_thread, NULL, (void*) &prompt_pw, data) != 0)
         send_err_msg (pamh, _("Failed to create thread"));