core: Use signature modes as flags

* src/engine-backend.h (engine_ops.sign): Rename argument mode to
flags.
* src/engine-gpg.c (gpg_sign): Rename argument mode to flags. Check
for invalid combination of flags.
* src/engine-gpgsm.c (gpgsm_sign): Rename argument mode to flags.
Check for unsupported flags.
* src/engine-uiserver.c (gpgsm_sign): Rename argument mode to flags.
Check for unsupported flags.
* src/engine.c, src/engine.h (_gpgme_engine_op_sign): Rename argument
mode to flags.
* src/gpgme.h.in (GPGME_SIG_MODE_ARCHIVE): Change value to 4.
(gpgme_op_sign_start, gpgme_op_sign): Rename argument mode to flags.
* src/sign.c (sign_start): Rename argument mode to flags. Adjust
check for invalid flags.
(gpgme_op_sign_start, gpgme_op_sign): Rename argument mode to flags.
--

Using the signature mode constants as flags is more natural, even if
currently all flags are mutually exclusive, because archives are
signed with a normal signature.

GnuPG-bug-id: 6342

8 files changed, 37 insertions, 26 deletions

diff --git a/src/engine-backend.h b/src/engine-backend.h
index 07ea8f14..d430620b 100644
--- a/src/engine-backend.h
+++ b/src/engine-backend.h
@@ -119,7 +119,7 @@ struct engine_ops
                                 gpgme_key_t key,
                                 gpgme_tofu_policy_t policy);
   gpgme_error_t (*sign) (void *engine, gpgme_data_t in, gpgme_data_t out,
-			 gpgme_sig_mode_t mode, int use_armor,
+			 gpgme_sig_mode_t flags, int use_armor,
 			 int use_textmode, int include_certs,
 			 gpgme_ctx_t ctx /* FIXME */);
   gpgme_error_t (*verify) (void *engine, gpgme_verify_flags_t flags,
diff --git a/src/engine-gpg.c b/src/engine-gpg.c
index e921df0a..41f24d1e 100644
--- a/src/engine-gpg.c
+++ b/src/engine-gpg.c
@@ -3580,7 +3580,7 @@ gpg_tofu_policy (void *engine, gpgme_key_t key, gpgme_tofu_policy_t policy)
 
 static gpgme_error_t
 gpg_sign (void *engine, gpgme_data_t in, gpgme_data_t out,
-	  gpgme_sig_mode_t mode, int use_armor, int use_textmode,
+	  gpgme_sig_mode_t flags, int use_armor, int use_textmode,
 	  int include_certs, gpgme_ctx_t ctx /* FIXME */)
 {
   engine_gpg_t gpg = engine;
@@ -3588,17 +3588,21 @@ gpg_sign (void *engine, gpgme_data_t in, gpgme_data_t out,
 
   (void)include_certs;
 
-  gpg->flags.use_gpgtar = mode == GPGME_SIG_MODE_ARCHIVE;
+  if ((flags != GPGME_SIG_MODE_NORMAL) && (flags != GPGME_SIG_MODE_DETACH)
+      && (flags != GPGME_SIG_MODE_CLEAR) && (flags != GPGME_SIG_MODE_ARCHIVE))
+    return gpg_error (GPG_ERR_INV_VALUE);
+
+  gpg->flags.use_gpgtar = !!(flags & GPGME_SIG_MODE_ARCHIVE);
 
   if (gpg->flags.use_gpgtar && !have_gpg_version (gpg, "2.4.1"))
     return gpg_error (GPG_ERR_NOT_SUPPORTED);
 
-  if (mode == GPGME_SIG_MODE_CLEAR)
+  if (flags & GPGME_SIG_MODE_CLEAR)
     err = add_arg (gpg, "--clearsign");
   else
     {
       err = add_arg (gpg, "--sign");
-      if (!err && mode == GPGME_SIG_MODE_DETACH)
+      if (!err && (flags & GPGME_SIG_MODE_DETACH))
 	err = add_arg (gpg, "--detach");
       if (!err && use_armor)
 	err = add_gpg_arg (gpg, "--armor");
diff --git a/src/engine-gpgsm.c b/src/engine-gpgsm.c
index fdc85dd6..7ac4f2db 100644
--- a/src/engine-gpgsm.c
+++ b/src/engine-gpgsm.c
@@ -2043,7 +2043,7 @@ gpgsm_keylist_ext (void *engine, const char *pattern[], int secret_only,
 
 static gpgme_error_t
 gpgsm_sign (void *engine, gpgme_data_t in, gpgme_data_t out,
-	    gpgme_sig_mode_t mode, int use_armor, int use_textmode,
+	    gpgme_sig_mode_t flags, int use_armor, int use_textmode,
 	    int include_certs, gpgme_ctx_t ctx /* FIXME */)
 {
   engine_gpgsm_t gpgsm = engine;
@@ -2057,6 +2057,9 @@ gpgsm_sign (void *engine, gpgme_data_t in, gpgme_data_t out,
   if (!gpgsm)
     return gpg_error (GPG_ERR_INV_VALUE);
 
+  if (flags & (GPGME_SIG_MODE_CLEAR | GPGME_SIG_MODE_ARCHIVE))
+    return gpg_error (GPG_ERR_INV_VALUE);
+
   /* FIXME: This does not work as RESET does not reset it so we can't
      revert back to default.  */
   if (include_certs != GPGME_INCLUDE_CERTS_DEFAULT)
@@ -2105,7 +2108,7 @@ gpgsm_sign (void *engine, gpgme_data_t in, gpgme_data_t out,
   gpgsm_clear_fd (gpgsm, MESSAGE_FD);
   gpgsm->inline_data = NULL;
 
-  err = start (gpgsm, mode == GPGME_SIG_MODE_DETACH
+  err = start (gpgsm, (flags & GPGME_SIG_MODE_DETACH)
 	       ? "SIGN --detached" : "SIGN");
   return err;
 }
diff --git a/src/engine-uiserver.c b/src/engine-uiserver.c
index 7aa9cfe3..a298bec6 100644
--- a/src/engine-uiserver.c
+++ b/src/engine-uiserver.c
@@ -1214,7 +1214,7 @@ uiserver_encrypt (void *engine, gpgme_key_t recp[], const char *recpstring,
 
 static gpgme_error_t
 uiserver_sign (void *engine, gpgme_data_t in, gpgme_data_t out,
-	       gpgme_sig_mode_t mode, int use_armor, int use_textmode,
+	       gpgme_sig_mode_t flags, int use_armor, int use_textmode,
 	       int include_certs, gpgme_ctx_t ctx /* FIXME */)
 {
   engine_uiserver_t uiserver = engine;
@@ -1237,8 +1237,11 @@ uiserver_sign (void *engine, gpgme_data_t in, gpgme_data_t out,
   else
     return gpgme_error (GPG_ERR_UNSUPPORTED_PROTOCOL);
 
+  if (flags & (GPGME_SIG_MODE_CLEAR | GPGME_SIG_MODE_ARCHIVE))
+    return gpg_error (GPG_ERR_INV_VALUE);
+
   if (gpgrt_asprintf (&cmd, "SIGN%s%s", protocol,
-		(mode == GPGME_SIG_MODE_DETACH) ? " --detached" : "") < 0)
+		(flags & GPGME_SIG_MODE_DETACH) ? " --detached" : "") < 0)
     return gpg_error_from_syserror ();
 
   key = gpgme_signers_enum (ctx, 0);
diff --git a/src/engine.c b/src/engine.c
index 2397f88d..ab399e73 100644
--- a/src/engine.c
+++ b/src/engine.c
@@ -912,7 +912,7 @@ _gpgme_engine_op_keylist_data (engine_t engine, gpgme_keylist_mode_t mode,
 
 gpgme_error_t
 _gpgme_engine_op_sign (engine_t engine, gpgme_data_t in, gpgme_data_t out,
-		       gpgme_sig_mode_t mode, int use_armor,
+		       gpgme_sig_mode_t flags, int use_armor,
 		       int use_textmode, int include_certs,
 		       gpgme_ctx_t ctx /* FIXME */)
 {
@@ -922,7 +922,7 @@ _gpgme_engine_op_sign (engine_t engine, gpgme_data_t in, gpgme_data_t out,
   if (!engine->ops->sign)
     return gpg_error (GPG_ERR_NOT_IMPLEMENTED);
 
-  return (*engine->ops->sign) (engine->engine, in, out, mode, use_armor,
+  return (*engine->ops->sign) (engine->engine, in, out, flags, use_armor,
 			       use_textmode, include_certs, ctx);
 }
 
diff --git a/src/engine.h b/src/engine.h
index e5654047..59d159a4 100644
--- a/src/engine.h
+++ b/src/engine.h
@@ -160,7 +160,7 @@ gpgme_error_t _gpgme_engine_op_keylist_data (engine_t engine,
 					     gpgme_keylist_mode_t mode,
 					     gpgme_data_t data);
 gpgme_error_t _gpgme_engine_op_sign (engine_t engine, gpgme_data_t in,
-				     gpgme_data_t out, gpgme_sig_mode_t mode,
+				     gpgme_data_t out, gpgme_sig_mode_t flags,
 				     int use_armor, int use_textmode,
 				     int include_certs,
 				     gpgme_ctx_t ctx /* FIXME */);
diff --git a/src/gpgme.h.in b/src/gpgme.h.in
index a337a124..3ea07a81 100644
--- a/src/gpgme.h.in
+++ b/src/gpgme.h.in
@@ -303,13 +303,13 @@ typedef enum
 gpgme_hash_algo_t;
 
 
-/* The available signature modes.  */
+/* The available signature mode flags.  */
 typedef enum
   {
     GPGME_SIG_MODE_NORMAL = 0,
     GPGME_SIG_MODE_DETACH = 1,
     GPGME_SIG_MODE_CLEAR  = 2,
-    GPGME_SIG_MODE_ARCHIVE = 3
+    GPGME_SIG_MODE_ARCHIVE = 4
   }
 gpgme_sig_mode_t;
 
@@ -1522,10 +1522,10 @@ gpgme_sign_result_t gpgme_op_sign_result (gpgme_ctx_t ctx);
 /* Sign the plaintext PLAIN and store the signature in SIG.  */
 gpgme_error_t gpgme_op_sign_start (gpgme_ctx_t ctx,
 				   gpgme_data_t plain, gpgme_data_t sig,
-				   gpgme_sig_mode_t mode);
+				   gpgme_sig_mode_t flags);
 gpgme_error_t gpgme_op_sign (gpgme_ctx_t ctx,
 			     gpgme_data_t plain, gpgme_data_t sig,
-			     gpgme_sig_mode_t mode);
+			     gpgme_sig_mode_t flags);
 
 
 /*
diff --git a/src/sign.c b/src/sign.c
index 7617d27f..7198d934 100644
--- a/src/sign.c
+++ b/src/sign.c
@@ -431,7 +431,7 @@ _gpgme_op_sign_init_result (gpgme_ctx_t ctx)
 
 static gpgme_error_t
 sign_start (gpgme_ctx_t ctx, int synchronous, gpgme_data_t plain,
-	    gpgme_data_t sig, gpgme_sig_mode_t mode)
+	    gpgme_data_t sig, gpgme_sig_mode_t flags)
 {
   gpgme_error_t err;
 
@@ -446,8 +446,9 @@ sign_start (gpgme_ctx_t ctx, int synchronous, gpgme_data_t plain,
   if (err)
     return err;
 
-  if (mode != GPGME_SIG_MODE_NORMAL && mode != GPGME_SIG_MODE_DETACH
-      && mode != GPGME_SIG_MODE_CLEAR && mode != GPGME_SIG_MODE_ARCHIVE)
+  if (flags & ~(GPGME_SIG_MODE_DETACH
+                |GPGME_SIG_MODE_CLEAR
+                |GPGME_SIG_MODE_ARCHIVE))
     return gpg_error (GPG_ERR_INV_VALUE);
 
   if (!plain)
@@ -466,7 +467,7 @@ sign_start (gpgme_ctx_t ctx, int synchronous, gpgme_data_t plain,
   _gpgme_engine_set_status_handler (ctx->engine, sign_status_handler,
 				    ctx);
 
-  return _gpgme_engine_op_sign (ctx->engine, plain, sig, mode, ctx->use_armor,
+  return _gpgme_engine_op_sign (ctx->engine, plain, sig, flags, ctx->use_armor,
 				ctx->use_textmode, ctx->include_certs,
 				ctx /* FIXME */);
 }
@@ -475,16 +476,16 @@ sign_start (gpgme_ctx_t ctx, int synchronous, gpgme_data_t plain,
 /* Sign the plaintext PLAIN and store the signature in SIG.  */
 gpgme_error_t
 gpgme_op_sign_start (gpgme_ctx_t ctx, gpgme_data_t plain, gpgme_data_t sig,
-		     gpgme_sig_mode_t mode)
+		     gpgme_sig_mode_t flags)
 {
   gpg_error_t err;
   TRACE_BEG  (DEBUG_CTX, "gpgme_op_sign_start", ctx,
-	      "plain=%p, sig=%p, mode=%i", plain, sig, mode);
+	      "plain=%p, sig=%p, flags=%i", plain, sig, flags);
 
   if (!ctx)
     return TRACE_ERR (gpg_error (GPG_ERR_INV_VALUE));
 
-  err = sign_start (ctx, 0, plain, sig, mode);
+  err = sign_start (ctx, 0, plain, sig, flags);
   return TRACE_ERR (err);
 }
 
@@ -492,17 +493,17 @@ gpgme_op_sign_start (gpgme_ctx_t ctx, gpgme_data_t plain, gpgme_data_t sig,
 /* Sign the plaintext PLAIN and store the signature in SIG.  */
 gpgme_error_t
 gpgme_op_sign (gpgme_ctx_t ctx, gpgme_data_t plain, gpgme_data_t sig,
-	       gpgme_sig_mode_t mode)
+	       gpgme_sig_mode_t flags)
 {
   gpgme_error_t err;
 
   TRACE_BEG  (DEBUG_CTX, "gpgme_op_sign", ctx,
-	      "plain=%p, sig=%p, mode=%i", plain, sig, mode);
+	      "plain=%p, sig=%p, flags=%i", plain, sig, flags);
 
   if (!ctx)
     return TRACE_ERR (gpg_error (GPG_ERR_INV_VALUE));
 
-  err = sign_start (ctx, 1, plain, sig, mode);
+  err = sign_start (ctx, 1, plain, sig, flags);
   if (!err)
     err = _gpgme_wait_one (ctx);
   return TRACE_ERR (err);