agent: Fix for smartcard decryption, checking compressed format.

* agent/agent.h (agent_card_pkdecrypt): Fix the type of R_BUF.
* agent/call-scd.c (agent_card_pkdecrypt): Likewise.
* agent/divert-scd.c (divert_pkdecrypt): Follow the change.
* agent/divert-scd.c (agent_card_ecc_kem): The prefix may be
0x02 or 0x03 depending on hidden y-coordinate.

--

Fixes-commit: 4ad08a8998fa16f8329fb57b0a47fe7a9b973700
GnuPG-bug-id: 7709
Signed-off-by: NIIBE Yutaka <[email protected]>

3 files changed, 7 insertions, 5 deletions

diff --git a/agent/agent.h b/agent/agent.h
index e891981b2..d1192f505 100644
--- a/agent/agent.h
+++ b/agent/agent.h
@@ -769,7 +769,8 @@ int agent_card_pkdecrypt (ctrl_t ctrl,
                           void *getpin_cb_arg,
                           const char *desc_text,
                           const unsigned char *indata, size_t indatalen,
-                          char **r_buf, size_t *r_buflen, int *r_padding);
+                          unsigned char **r_buf, size_t *r_buflen,
+                          int *r_padding);
 
 int agent_card_readcert (ctrl_t ctrl,
                          const char *id, char **r_buf, size_t *r_buflen);
diff --git a/agent/call-scd.c b/agent/call-scd.c
index 28669206c..f47deb83e 100644
--- a/agent/call-scd.c
+++ b/agent/call-scd.c
@@ -609,7 +609,7 @@ agent_card_pkdecrypt (ctrl_t ctrl,
                       void *getpin_cb_arg,
                       const char *desc_text,
                       const unsigned char *indata, size_t indatalen,
-                      char **r_buf, size_t *r_buflen, int *r_padding)
+                      unsigned char **r_buf, size_t *r_buflen, int *r_padding)
 {
   int rc;
   char line[ASSUAN_LINELENGTH];
diff --git a/agent/divert-scd.c b/agent/divert-scd.c
index 15d1abd8d..e0b5164b5 100644
--- a/agent/divert-scd.c
+++ b/agent/divert-scd.c
@@ -394,7 +394,7 @@ divert_pkdecrypt (ctrl_t ctrl,
   int depth;
   const unsigned char *ciphertext;
   size_t ciphertextlen;
-  char *plaintext;
+  unsigned char *plaintext;
   size_t plaintextlen;
 
   bin2hex (grip, 20, hexgrip);
@@ -504,7 +504,7 @@ agent_card_ecc_kem (ctrl_t ctrl, const unsigned char *ecc_ct,
                     size_t ecc_point_len, unsigned char *ecc_ecdh)
 {
   gpg_error_t err = 0;
-  char *ecdh = NULL;
+  unsigned char *ecdh = NULL;
   size_t len;
   int rc;
   char hexgrip[KEYGRIP_LEN*2+1];
@@ -517,7 +517,8 @@ agent_card_ecc_kem (ctrl_t ctrl, const unsigned char *ecc_ct,
 
   if (len == ecc_point_len)
     memcpy (ecc_ecdh, ecdh, len);
-  else if (len && (len - 1) * 2 == ecc_point_len - 1 && ecdh[0] == 0x02)
+  else if (len && (len - 1) * 2 == ecc_point_len - 1
+           && (ecdh[0] & ~1) == 0x02)
     {
       /* It's x-coordinate-only (compressed) point representation.  */
       memcpy (ecc_ecdh, ecdh, len);