gpg: Fix the encrypt+sign hash algo preference selection for ECDSA.

* g10/keydb.h (pref_hint): Change from union to struct and add field
'exact'.  Adjust callers.
* g10/pkclist.c (algo_available): Take care of the exact hint.
* g10/sign.c (sign_file): Rework the hash detection from
recipient prefs.
--

This fixes a encrypt+sign case like: One recipient key has SHA512 as
highest ranked hash preference but the the signing key is a 256 bit
curve.  Because we don't want to use a truncated hash with ECDSA, we
need to have an exact match - this is in particular important for
smartcard which check that the hash matches the curves.

Signed-off-by: Werner Koch <[email protected]>

Ported-from-stable: aeed0b93ff660fe271d8f98f8d5ce60aa5bf3ebe

1 files changed, 15 insertions, 5 deletions

diff --git a/g10/pkclist.c b/g10/pkclist.c
index 55cafcacb..643a0fb03 100644
--- a/g10/pkclist.c
+++ b/g10/pkclist.c
@@ -1444,7 +1444,7 @@ build_pk_list (ctrl_t ctrl, strlist_t rcpts, PK_LIST *ret_pk_list)
    preference list, so I'm including it. -dms */
 
 int
-algo_available( preftype_t preftype, int algo, const union pref_hint *hint)
+algo_available( preftype_t preftype, int algo, const struct pref_hint *hint)
 {
   if( preftype == PREFTYPE_SYM )
     {
@@ -1465,16 +1465,26 @@ algo_available( preftype_t preftype, int algo, const union pref_hint *hint)
     {
       if (hint && hint->digest_length)
 	{
-	  if (hint->digest_length!=20 || opt.flags.dsa2)
+          unsigned int n = gcry_md_get_algo_dlen (algo);
+
+          if (hint->exact)
+            {
+              /* For example ECDSA requires an exact hash value so
+               * that we do not truncate.  For DSA we allow truncation
+               * and thus exact is not set.  */
+              if (hint->digest_length != n)
+                return 0;
+            }
+	  else if (hint->digest_length!=20 || opt.flags.dsa2)
 	    {
 	      /* If --enable-dsa2 is set or the hash isn't 160 bits
 		 (which implies DSA2), then we'll accept a hash that
 		 is larger than we need.  Otherwise we won't accept
 		 any hash that isn't exactly the right size. */
-	      if (hint->digest_length > gcry_md_get_algo_dlen (algo))
+	      if (hint->digest_length > n)
 		return 0;
 	    }
-	  else if (hint->digest_length != gcry_md_get_algo_dlen (algo))
+	  else if (hint->digest_length != n)
 	    return 0;
 	}
 
@@ -1511,7 +1521,7 @@ algo_available( preftype_t preftype, int algo, const union pref_hint *hint)
  */
 int
 select_algo_from_prefs(PK_LIST pk_list, int preftype,
-		       int request, const union pref_hint *hint)
+		       int request, const struct pref_hint *hint)
 {
   PK_LIST pkr;
   u32 bits[8];