core: Enhance gpgme_data_identify to detect binary PGP messages.

* src/gpgme.h.in (GPGME_DATA_TYPE_PGP_ENCRYPTED): New.
(GPGME_DATA_TYPE_PGP_SIGNATURE): New.
* src/data-identify.c: Add enum for OpenPGP packet types.
(buf32_to_ulong): New.
(next_openpgp_packet): New.  Based on the gnupg/kbx/keybox-openpgp.c
implementation and relicensed to LGPL by g10 Code.
(pgp_binary_detection): New.
(basic_detection): Call pgp_binary_detection instead of returning
unknown.

Signed-off-by: Werner Koch <wk@gnupg.org>

NEWS

@@ -18,6 +18,8 @@ Noteworthy changes in version 1.7.0 (unreleased) [C25/A14/R_]
  GPGME_STATUS_TOFU_STATS        NEW.
  GPGME_STATUS_TOFU_STATS_LONG   NEW.
  GPGME_STATUS_NOTATION_FLAGS    NEW.
+ GPGME_DATA_TYPE_PGP_ENCRYPTED  NEW.
+ GPGME_DATA_TYPE_PGP_SIGNATURE  NEW.
 
 
 Noteworthy changes in version 1.6.0 (2015-08-26) [C25/A14/R0]

src/data-identify.c

@@ -29,10 +29,238 @@
 #include "util.h"
 #include "parsetlv.h"
 
+
 /* The size of the sample data we take for detection.  */
 #define SAMPLE_SIZE 2048
 
 
+/* OpenPGP packet types.  */
+enum
+  {
+    PKT_NONE	      = 0,
+    PKT_PUBKEY_ENC    = 1,  /* Public key encrypted packet. */
+    PKT_SIGNATURE     = 2,  /* Secret key encrypted packet. */
+    PKT_SYMKEY_ENC    = 3,  /* Session key packet. */
+    PKT_ONEPASS_SIG   = 4,  /* One pass sig packet. */
+    PKT_SECRET_KEY    = 5,  /* Secret key. */
+    PKT_PUBLIC_KEY    = 6,  /* Public key. */
+    PKT_SECRET_SUBKEY = 7,  /* Secret subkey. */
+    PKT_COMPRESSED    = 8,  /* Compressed data packet. */
+    PKT_ENCRYPTED     = 9,  /* Conventional encrypted data. */
+    PKT_MARKER	      = 10, /* Marker packet. */
+    PKT_PLAINTEXT     = 11, /* Literal data packet. */
+    PKT_RING_TRUST    = 12, /* Keyring trust packet. */
+    PKT_USER_ID	      = 13, /* User id packet. */
+    PKT_PUBLIC_SUBKEY = 14, /* Public subkey. */
+    PKT_OLD_COMMENT   = 16, /* Comment packet from an OpenPGP draft. */
+    PKT_ATTRIBUTE     = 17, /* PGP's attribute packet. */
+    PKT_ENCRYPTED_MDC = 18, /* Integrity protected encrypted data. */
+    PKT_MDC 	      = 19, /* Manipulation detection code packet. */
+  };
+
+
+static inline unsigned long
+buf32_to_ulong (const void *buffer)
+{
+  const unsigned char *p = buffer;
+
+  return (((unsigned long)p[0] << 24) | (p[1] << 16) | (p[2] << 8) | p[3]);
+}
+
+
+/* Parse the next openpgp packet.  This function assumes a valid
+ * OpenPGP packet at the address pointed to by BUFPTR which has a
+ * maximum length as stored at BUFLEN.  Return the header information
+ * of that packet and advance the pointer stored at BUFPTR to the next
+ * packet; also adjust the length stored at BUFLEN to match the
+ * remaining bytes. If there are no more packets, store NULL at
+ * BUFPTR.  Return an non-zero error code on failure or the following
+ * data on success:
+ *
+ *  R_PKTTYPE = The packet type.
+ *  R_NTOTAL  = The total number of bytes of this packet
+ *
+ * If GPG_ERR_TRUNCATED is returned, a packet type is anyway stored at
+ * R_PKTTYPE but R_NOTAL won't have a usable value,
+ */
+static gpg_error_t
+next_openpgp_packet (unsigned char const **bufptr, size_t *buflen,
+                     int *r_pkttype, size_t *r_ntotal)
+{
+  const unsigned char *buf = *bufptr;
+  size_t len = *buflen;
+  int c, ctb, pkttype;
+  unsigned long pktlen;
+
+  if (!len)
+    return gpg_error (GPG_ERR_NO_DATA);
+
+  ctb = *buf++; len--;
+  if ( !(ctb & 0x80) )
+    return gpg_error (GPG_ERR_INV_PACKET); /* Invalid CTB. */
+
+  if ((ctb & 0x40))  /* New style (OpenPGP) CTB.  */
+    {
+      pkttype = (ctb & 0x3f);
+      if (!len)
+        return gpg_error (GPG_ERR_INV_PACKET); /* No 1st length byte. */
+      c = *buf++; len--;
+      if ( c < 192 )
+        pktlen = c;
+      else if ( c < 224 )
+        {
+          pktlen = (c - 192) * 256;
+          if (!len)
+            return gpg_error (GPG_ERR_INV_PACKET); /* No 2nd length byte. */
+          c = *buf++; len--;
+          pktlen += c + 192;
+        }
+      else if (c == 255)
+        {
+          if (len < 4)
+            return gpg_error (GPG_ERR_INV_PACKET); /* No length bytes. */
+          pktlen = buf32_to_ulong (buf);
+          buf += 4;
+          len -= 4;
+      }
+      else /* Partial length encoding is not allowed for key packets. */
+        return gpg_error (GPG_ERR_UNEXPECTED);
+    }
+  else /* Old style CTB.  */
+    {
+      int lenbytes;
+
+      pktlen = 0;
+      pkttype = (ctb>>2)&0xf;
+      lenbytes = ((ctb&3)==3)? 0 : (1<<(ctb & 3));
+      if (!lenbytes) /* Not allowed in key packets.  */
+        return gpg_error (GPG_ERR_UNEXPECTED);
+      if (len < lenbytes)
+        return gpg_error (GPG_ERR_INV_PACKET); /* Not enough length bytes.  */
+      for (; lenbytes; lenbytes--)
+        {
+          pktlen <<= 8;
+          pktlen |= *buf++; len--;
+	}
+    }
+
+  /* Do some basic sanity check.  */
+  switch (pkttype)
+    {
+    case PKT_PUBKEY_ENC:
+    case PKT_SIGNATURE:
+    case PKT_SYMKEY_ENC:
+    case PKT_ONEPASS_SIG:
+    case PKT_SECRET_KEY:
+    case PKT_PUBLIC_KEY:
+    case PKT_SECRET_SUBKEY:
+    case PKT_COMPRESSED:
+    case PKT_ENCRYPTED:
+    case PKT_MARKER:
+    case PKT_PLAINTEXT:
+    case PKT_RING_TRUST:
+    case PKT_USER_ID:
+    case PKT_PUBLIC_SUBKEY:
+    case PKT_OLD_COMMENT:
+    case PKT_ATTRIBUTE:
+    case PKT_ENCRYPTED_MDC:
+    case PKT_MDC:
+      break; /* Okay these are allowed packets. */
+    default:
+      return gpg_error (GPG_ERR_UNEXPECTED);
+    }
+
+  if (pktlen > len)
+    {
+      /* Packet length header too long.  This is possible because we
+       * may have only a truncated image.  */
+      *r_pkttype = pkttype;
+      *r_ntotal = 0;
+      *bufptr = NULL;
+      return gpg_error (GPG_ERR_TRUNCATED);
+    }
+
+  *r_pkttype = pkttype;
+  *r_ntotal = (buf - *bufptr) + pktlen;
+
+  *bufptr = buf + pktlen;
+  *buflen = len - pktlen;
+  if (!*buflen)
+    *bufptr = NULL;
+
+  return 0;
+}
+
+
+/* Detection of PGP binary data.  This function parses an OpenPGP
+ * message.  This parser is robust enough to work on a truncated
+ * version.  Returns a GPGME_DATA_TYPE_.  */
+static gpgme_data_type_t
+pgp_binary_detection (const void *image_arg, size_t imagelen)
+{
+  gpg_error_t err = 0;
+  const unsigned char *image = image_arg;
+  size_t n;
+  int pkttype;
+  int anypacket = 0;
+  int allsignatures = 0;
+
+  while (!err && image)
+    {
+      err = next_openpgp_packet (&image, &imagelen, &pkttype, &n);
+      if (gpg_err_code (err) == GPG_ERR_TRUNCATED)
+        ;
+      else if (err)
+        break;
+
+      if (pkttype == PKT_SIGNATURE)
+        {
+          if (!anypacket)
+            allsignatures = 1;
+        }
+      else
+        allsignatures = 0;
+      anypacket = 1;
+
+      switch (pkttype)
+        {
+        case PKT_SIGNATURE:
+          break;  /* We decide later.  */
+
+        case PKT_PLAINTEXT:
+          /* Old style signature format: {sig}+,plaintext */
+          if (allsignatures)
+            return GPGME_DATA_TYPE_PGP_SIGNED;
+          break;
+
+        case PKT_ONEPASS_SIG:
+          return GPGME_DATA_TYPE_PGP_SIGNED;
+
+        case PKT_SECRET_KEY:
+        case PKT_PUBLIC_KEY:
+          return GPGME_DATA_TYPE_PGP_KEY;
+
+        case PKT_SECRET_SUBKEY:
+        case PKT_PUBLIC_SUBKEY:
+          return GPGME_DATA_TYPE_PGP_OTHER;
+        case PKT_PUBKEY_ENC:
+        case PKT_SYMKEY_ENC:
+          return GPGME_DATA_TYPE_PGP_ENCRYPTED;
+
+        case PKT_MARKER:
+          break;  /* Skip this packet.  */
+
+        default:
+          return GPGME_DATA_TYPE_PGP_OTHER;
+        }
+    }
+
+  if (allsignatures)
+    return  GPGME_DATA_TYPE_PGP_SIGNATURE;
+
+  return GPGME_DATA_TYPE_UNKNOWN;
+}
+
 
 /* Note that DATA may be binary but a final nul is required so that
    string operations will find a terminator.
@@ -167,7 +395,7 @@ basic_detection (const char *data, size_t datalen)
          at all defined and in any case it is uncommon.  Thus we don't
          do any further plausibility checks but stupidly assume no CMS
          armored data will follow.  */
-      return GPGME_DATA_TYPE_UNKNOWN;
+      return pgp_binary_detection (data, datalen);
     }
 
   /* Now check whether there are armor lines.  */

src/gpgme.h.in

@@ -239,8 +239,10 @@ typedef enum
     GPGME_DATA_TYPE_INVALID      = 0,   /* Not detected.  */
     GPGME_DATA_TYPE_UNKNOWN      = 1,
     GPGME_DATA_TYPE_PGP_SIGNED   = 0x10,
+    GPGME_DATA_TYPE_PGP_ENCRYPTED= 0x11,
     GPGME_DATA_TYPE_PGP_OTHER    = 0x12,
     GPGME_DATA_TYPE_PGP_KEY      = 0x13,
+    GPGME_DATA_TYPE_PGP_SIGNATURE= 0x18, /* Detached signature */
     GPGME_DATA_TYPE_CMS_SIGNED   = 0x20,
     GPGME_DATA_TYPE_CMS_ENCRYPTED= 0x21,
     GPGME_DATA_TYPE_CMS_OTHER    = 0x22,