diff --git a/tests/start-stop-agent b/tests/start-stop-agent
index 9514e0a1..79013741 100755
--- a/tests/start-stop-agent
+++ b/tests/start-stop-agent
@@ -39,6 +39,7 @@ do
       mv "$GNUPGHOME/$F" "$GNUPGHOME/$F~"
       sed -e "s#^agent-program.*#agent-program ${GPG_AGENT}|--debug-quick-random#" \
           >"$GNUPGHOME/$F" <"$GNUPGHOME/$F~"
+      rm "$GNUPGHOME/$F~"
   fi
 done