gpgsm: Support creation of verification of ML-DSA signatures.

* common/sexputil.c (get_pk_algo_from_key): Enlarge buffer for algo
name so that "dilithium3" can be detected.
* agent/pksign.c (do_encode_mldsa): New.
(agent_pksign_do): Call it.

* sm/certcheck.c (pk_algo_from_sexp): Detected ML-DSA.
(gpgsm_check_cms_signature): Support ML-DSA.  Also rework error
cleanup.

* sm/sign.c (gpgsm_sign): Support ML-DSA.
* sm/verify.c (gpgsm_verify): Fix a comment.
--

Libgcrypt 1.12.5 and libksba 1.9.0 are required to work properly.

GnuPG-bug-id: 8517

4 files changed, 46 insertions, 7 deletions

diff --git a/sm/certcheck.c b/sm/certcheck.c
index 7d7cbe1e6..3aafd6028 100644
--- a/sm/certcheck.c
+++ b/sm/certcheck.c
@@ -217,6 +217,8 @@ pk_algo_from_sexp (gcry_sexp_t pkey)
     algo = GCRY_PK_ECC;
   else if (n==13 && !memcmp (name, "ambiguous-rsa", 13))
     algo = GCRY_PK_RSA;
+  else if (n==10 && !memcmp (name, "dilithium", 9)) /* "dilithium<N>" */
+    algo = GCRY_PK_MLDSA;
   else
     algo = 0;
   gcry_sexp_release (l2);
@@ -672,10 +674,11 @@ gpgsm_check_cms_signature (ksba_cert_t cert, gcry_sexp_t s_sig,
 {
   int rc;
   ksba_sexp_t p;
-  gcry_sexp_t s_hash, s_pkey;
+  gcry_sexp_t s_hash = NULL;
+  gcry_sexp_t s_pkey = NULL;
   size_t n;
   int pkalgo;
-  int use_pss;
+  int use_pss = 0;
   unsigned int saltlen = 0;
 
   if (r_pkalgo)
@@ -735,7 +738,24 @@ gpgsm_check_cms_signature (ksba_cert_t cert, gcry_sexp_t s_sig,
                             gcry_md_read (md, mdalgo),
                             saltlen);
       if (rc)
-        BUG ();
+        {
+          log_error ("building data for RSA-PSS failed: %s\n",
+                     gpg_strerror (rc));
+          goto leave;
+        }
+    }
+  else if (pkalgo == GCRY_PK_MLDSA)
+    {
+      rc = gcry_sexp_build (&s_hash, NULL,
+                            "(data(raw)(value %b)(label %s))",
+                            (int)gcry_md_get_algo_dlen (mdalgo),
+                            gcry_md_read (md, mdalgo),
+                            "");
+      if (rc)
+        {
+          log_error ("building data for mldsa failed: %s\n", gpg_strerror (rc));
+          goto leave;
+        }
     }
   else
     {
@@ -746,18 +766,22 @@ gpgsm_check_cms_signature (ksba_cert_t cert, gcry_sexp_t s_sig,
                          gcry_pk_get_nbits (s_pkey), s_pkey, &frame);
       if (rc)
         {
-          gcry_sexp_release (s_pkey);
-          return rc;
+          log_error ("building data for RSA or DSA failed: %s\n",
+                     gpg_strerror (rc));
+          goto leave;
         }
-      /* put hash into the S-Exp s_hash */
+      /* Put hash into the S-Exp s_hash */
       if ( gcry_sexp_build (&s_hash, NULL, "%m", frame) )
         BUG ();
       gcry_mpi_release (frame);
     }
 
   rc = gcry_pk_verify (s_sig, s_hash, s_pkey);
+
   if (DBG_X509)
     log_debug ("gcry_pk_verify: %s\n", gpg_strerror (rc));
+
+ leave:
   gcry_sexp_release (s_hash);
   gcry_sexp_release (s_pkey);
   return rc;
diff --git a/sm/keylist.c b/sm/keylist.c
index cd7db65cc..232dcdc1a 100644
--- a/sm/keylist.c
+++ b/sm/keylist.c
@@ -279,6 +279,14 @@ print_key_data (ksba_cert_t cert, estream_t fp)
       putchar('\n');
     }
 #else
+  /* unsigned char *p; */
+  /* size_t n; */
+
+  /* p = ksba_cert_get_public_key (cert); */
+  /* n = gcry_sexp_canon_len (p, 0, NULL, NULL); */
+  /* log_printhex (p, n, "public key"); */
+
+
   (void)cert;
   (void)fp;
 #endif
diff --git a/sm/sign.c b/sm/sign.c
index 4a15f91ea..0e3ad980c 100644
--- a/sm/sign.c
+++ b/sm/sign.c
@@ -866,6 +866,12 @@ gpgsm_sign (ctrl_t ctrl, certlist_t signerlist,
               else
                 oid = "2.16.840.1.101.3.4.2.3"; /* sha512 */
             }
+          else if (pk_algo == GCRY_PK_MLDSA)
+            {
+              /* RFC9882 section 3.3 and table 1 suggest the use of
+               * SHA512 for best interoperability.  */
+              oid = "2.16.840.1.101.3.4.2.3"; /* sha512 */
+            }
           else
             {
               /* For RSA we reuse the hash algo used by the certificate.  */
diff --git a/sm/verify.c b/sm/verify.c
index fe5d63f5c..39906793a 100644
--- a/sm/verify.c
+++ b/sm/verify.c
@@ -422,7 +422,8 @@ gpgsm_verify (ctrl_t ctrl, estream_t in_fp, estream_t data_fp,
                      signer, sigval_hash_algo, pkalgoflags);
         }
       if (!sigval_hash_algo)
-        sigval_hash_algo = algo; /* Fallback used e.g. with old libksba. */
+        sigval_hash_algo = algo; /* Fallback when the hash algo is not
+                                  * part of the pubkey algo oid. */
 
       /* Find the certificate of the signer */
       keydb_search_reset (kh);