core: Support direct signing of file with gpg

* src/gpgme.h.in (GPGME_SIG_MODE_FILE): New signature mode flag.
* src/engine-gpg.c (gpg_sign): Separate signature mode from additional
flags. Check for incompatible flags. Explicitly set output to stdout if
no output file is used. Pass filename instead of fd to gpg when new flag
is set.
* src/engine-gpgsm.c (gpgsm_sign): Return error if new flag is set.
* src/engine-uiserver.c (uiserver_sign): Ditto.
* src/sign.c (sign_start): Consider new flag on check for invalid flags.

* tests/run-sign.c (show_usage): New options --detach and
--direct-file-io.
(main): Parse new options. Create a detached signature if --detach is
given. Make gpg read the input file itself if --direct-file-io is given.
--

With this change the gpgme_op_sign* functions gain the possibility to
make gpg read the data to sign directly from a file instead of from an
input FD to which it is written by gpgme.

GnuPG-bug-id: 6550

5 files changed, 41 insertions, 19 deletions

diff --git a/src/engine-gpg.c b/src/engine-gpg.c
index 123760be..cc249e7b 100644
--- a/src/engine-gpg.c
+++ b/src/engine-gpg.c
@@ -3615,25 +3615,32 @@ gpg_sign (void *engine, gpgme_data_t in, gpgme_data_t out,
 {
   engine_gpg_t gpg = engine;
   gpgme_error_t err;
-  const char *output = NULL;
+  const char *file_name = NULL;
+  unsigned int mode = flags & (GPGME_SIG_MODE_NORMAL
+                               |GPGME_SIG_MODE_DETACH
+                               |GPGME_SIG_MODE_CLEAR
+                               |GPGME_SIG_MODE_ARCHIVE);
 
   (void)include_certs;
 
-  if ((flags != GPGME_SIG_MODE_NORMAL) && (flags != GPGME_SIG_MODE_DETACH)
-      && (flags != GPGME_SIG_MODE_CLEAR) && (flags != GPGME_SIG_MODE_ARCHIVE))
+  if ((mode != GPGME_SIG_MODE_NORMAL) && (mode != GPGME_SIG_MODE_DETACH)
+      && (mode != GPGME_SIG_MODE_CLEAR) && (mode != GPGME_SIG_MODE_ARCHIVE))
     return gpg_error (GPG_ERR_INV_VALUE);
 
-  gpg->flags.use_gpgtar = !!(flags & GPGME_SIG_MODE_ARCHIVE);
+  gpg->flags.use_gpgtar = !!(mode == GPGME_SIG_MODE_ARCHIVE);
 
   if (gpg->flags.use_gpgtar && !have_usable_gpgtar (gpg))
     return gpg_error (GPG_ERR_NOT_SUPPORTED);
 
-  if (flags & GPGME_SIG_MODE_CLEAR)
+  if (gpg->flags.use_gpgtar && (flags & GPGME_SIG_MODE_FILE))
+    return gpg_error (GPG_ERR_INV_VALUE);
+
+  if (mode == GPGME_SIG_MODE_CLEAR)
     err = add_arg (gpg, "--clearsign");
   else
     {
       err = add_arg (gpg, "--sign");
-      if (!err && (flags & GPGME_SIG_MODE_DETACH))
+      if (!err && (mode == GPGME_SIG_MODE_DETACH))
 	err = add_arg (gpg, "--detach");
       if (!err && use_armor)
 	err = add_gpg_arg (gpg, "--armor");
@@ -3657,20 +3664,24 @@ gpg_sign (void *engine, gpgme_data_t in, gpgme_data_t out,
     err = append_args_from_sig_notations (gpg, ctx, NOTATION_FLAG_SIG);
 
   if (!err)
+    err = add_arg (gpg, "--output");
+  if (!err)
     {
-      output = gpgme_data_get_file_name (out);
+      const char *output = gpgme_data_get_file_name (out);
       if (output)
+        err = add_arg (gpg, output);
+      else
         {
-          err = add_arg (gpg, "--output");
+          err = add_arg (gpg, "-");
           if (!err)
-            err = add_arg (gpg, output);
+            err = add_data (gpg, out, 1, 1);
         }
     }
-
+  if (!err)
+    file_name = gpgme_data_get_file_name (in);
   /* Tell the gpg object about the data.  */
   if (gpg->flags.use_gpgtar)
     {
-      const char *file_name = gpgme_data_get_file_name (in);
       if (!err && file_name)
         {
           err = add_arg (gpg, "--directory");
@@ -3689,9 +3700,17 @@ gpg_sign (void *engine, gpgme_data_t in, gpgme_data_t out,
       if (!err)
         err = add_data (gpg, in, 0, 0);
     }
+  else if (flags & GPGME_SIG_MODE_FILE)
+    {
+      if (!err)
+        err = add_arg (gpg, "--");
+      if (!err && (!file_name || !*file_name))
+        err = gpg_error (GPG_ERR_INV_VALUE);
+      if (!err)
+	err = add_arg (gpg, file_name);
+    }
   else
     {
-      const char *file_name = gpgme_data_get_file_name (in);
       if (!err && file_name)
 	err = add_gpg_arg_with_value (gpg, "--set-filename=", file_name, 0);
       if (!err)
@@ -3702,9 +3721,6 @@ gpg_sign (void *engine, gpgme_data_t in, gpgme_data_t out,
         err = add_data (gpg, in, -1, 0);
     }
 
-  if (!err && !output)
-    err = add_data (gpg, out, 1, 1);
-
   if (!err)
     err = start (gpg);
 
diff --git a/src/engine-gpgsm.c b/src/engine-gpgsm.c
index 014507d4..fa89ae3f 100644
--- a/src/engine-gpgsm.c
+++ b/src/engine-gpgsm.c
@@ -2128,7 +2128,9 @@ gpgsm_sign (void *engine, gpgme_data_t in, gpgme_data_t out,
   if (!gpgsm)
     return gpg_error (GPG_ERR_INV_VALUE);
 
-  if (flags & (GPGME_SIG_MODE_CLEAR | GPGME_SIG_MODE_ARCHIVE))
+  if (flags & (GPGME_SIG_MODE_CLEAR
+               | GPGME_SIG_MODE_ARCHIVE
+               | GPGME_SIG_MODE_FILE))
     return gpg_error (GPG_ERR_INV_VALUE);
 
   /* FIXME: This does not work as RESET does not reset it so we can't
diff --git a/src/engine-uiserver.c b/src/engine-uiserver.c
index a116feb9..3e9836af 100644
--- a/src/engine-uiserver.c
+++ b/src/engine-uiserver.c
@@ -1237,7 +1237,9 @@ uiserver_sign (void *engine, gpgme_data_t in, gpgme_data_t out,
   else
     return gpgme_error (GPG_ERR_UNSUPPORTED_PROTOCOL);
 
-  if (flags & (GPGME_SIG_MODE_CLEAR | GPGME_SIG_MODE_ARCHIVE))
+  if (flags & (GPGME_SIG_MODE_CLEAR
+               | GPGME_SIG_MODE_ARCHIVE
+               | GPGME_SIG_MODE_FILE))
     return gpg_error (GPG_ERR_INV_VALUE);
 
   if (gpgrt_asprintf (&cmd, "SIGN%s%s", protocol,
diff --git a/src/gpgme.h.in b/src/gpgme.h.in
index ccc7b026..c8eba0a5 100644
--- a/src/gpgme.h.in
+++ b/src/gpgme.h.in
@@ -309,7 +309,8 @@ typedef enum
     GPGME_SIG_MODE_NORMAL = 0,
     GPGME_SIG_MODE_DETACH = 1,
     GPGME_SIG_MODE_CLEAR  = 2,
-    GPGME_SIG_MODE_ARCHIVE = 4
+    GPGME_SIG_MODE_ARCHIVE = 4,
+    GPGME_SIG_MODE_FILE = 8
   }
 gpgme_sig_mode_t;
 
diff --git a/src/sign.c b/src/sign.c
index 1563c1dd..56b2d5fb 100644
--- a/src/sign.c
+++ b/src/sign.c
@@ -450,7 +450,8 @@ sign_start (gpgme_ctx_t ctx, int synchronous, gpgme_data_t plain,
 
   if (flags & ~(GPGME_SIG_MODE_DETACH
                 |GPGME_SIG_MODE_CLEAR
-                |GPGME_SIG_MODE_ARCHIVE))
+                |GPGME_SIG_MODE_ARCHIVE
+                |GPGME_SIG_MODE_FILE))
     return gpg_error (GPG_ERR_INV_VALUE);
 
   if (!plain)