core: New data flags "io-buffer-size" and "sensitive".

* src/data.c (_gpgme_data_release): Free buffers.
(gpgme_data_seek): Adjust from renamed fields.
(gpgme_data_set_flag): Implement new flags.
(_gpgme_data_inbound_handler): Allow the use of a malloced buffer.
(_gpgme_data_outbound_handler): Ditto.
* src/data.h (BUFFER_SIZE): Move out of the struct definition.
(struct gpgme_data): Remove pending filed and introduce inbound and
outbound fields.

* src/conversion.c (_gpgme_wipememory): New.  Taken from GnuPG.
* src/cJSON.c (wipememory): Use this here too.

* tests/run-decrypt.c (main): Add options "--large-buffers" and
"--sensitive".
--

GnuPG-bug-id: 5478
Signed-off-by: Werner Koch <wk@gnupg.org>

doc/gpgme.texi

@@ -2253,6 +2253,22 @@ the data.  If this is set the OpenPGP engine may use this to decide on
 buffer allocation strategies and to provide a total value for its
 progress information.
 
+@item io-buffer-size
+The value is a decimal number with the length of internal buffers to
+used for internal I/O operations.  The value is capped at 1048576 (1
+MiB).  In certain environments large buffers can yield a performance
+boost for callback bases data object, but the details depend a lot on
+the circumstances and the operating system.  This flag may only be set
+once and must be set before any actual I/O happens ion the data
+objects.
+
+@item sensitive
+If the numeric value is not 0 the data object is considered to contain
+sensitive information like passwords or key material.  If this is set
+the internal buffers are securely overwritten with zeroes by
+gpgme_data_release.
+
+
 @end table
 
 This function returns @code{0} on success.

src/cJSON.c

@@ -50,21 +50,13 @@
 
 #include "cJSON.h"
 
+
 /* Only use calloc. */
 #define CALLOC_ONLY 1
 
 /* Maximum recursion depth */
 #define MAX_DEPTH 512
 
-/* To avoid that a compiler optimizes certain memset calls away, these
-   macros may be used instead. */
-#define wipememory2(_ptr,_set,_len) do { \
-        volatile char *_vptr=(volatile char *)(_ptr); \
-        size_t _vlen=(_len); \
-        while(_vlen) { *_vptr=(_set); _vptr++; _vlen--; } \
-    } while(0)
-#define wipememory(_ptr,_len) wipememory2(_ptr,0,_len)
-
 /* We use malloc function wrappers from gpgrt (aka libgpg-error).  */
 #include <gpgrt.h>
 #define xtrycalloc(a,b) gpgrt_calloc ((a), (b))
@@ -77,6 +69,16 @@
 #endif
 
 
+static void
+wipememory (void *ptr, size_t len)
+{
+  /* Prevent compiler from optimizing away the call to memset by accessing
+   * memset through volatile pointer. */
+  static void *(*volatile memset_ptr)(void *, int, size_t) = (void *)memset;
+  memset_ptr (ptr, 0, len);
+}
+
+
 static int
 cJSON_strcasecmp (const char *s1, const char *s2)
 {

src/conversion.c

@@ -43,6 +43,17 @@
 
 
 
+void
+_gpgme_wipememory (void *ptr, size_t len)
+{
+  /* Prevent compiler from optimizing away the call to memset by accessing
+   * memset through volatile pointer. */
+  static void *(*volatile memset_ptr)(void *, int, size_t) = (void *)memset;
+  memset_ptr (ptr, 0, len);
+}
+
+
+
 static char *
 do_strconcat (const char *s1, va_list arg_ptr)
 {

src/data.c

@@ -339,6 +339,21 @@ _gpgme_data_release (gpgme_data_t dh)
   remove_from_property_table (dh, dh->propidx);
   if (dh->file_name)
     free (dh->file_name);
+  if (dh->inbound_buffer)
+    {
+      if (dh->sensitive)
+        _gpgme_wipememory (dh->inbound_buffer, dh->io_buffer_size);
+      free (dh->inbound_buffer);
+    }
+  if (dh->outbound_buffer)
+    {
+      if (dh->sensitive)
+        _gpgme_wipememory (dh->outbound_buffer, dh->io_buffer_size);
+      free (dh->outbound_buffer);
+    }
+  if (dh->sensitive)
+    _gpgme_wipememory (dh->outboundspace, BUFFER_SIZE);
+
   free (dh);
 }
 
@@ -431,11 +446,11 @@ gpgme_data_seek (gpgme_data_t dh, gpgme_off_t offset, int whence)
   /* For relative movement, we must take into account the actual
      position of the read counter.  */
   if (whence == SEEK_CUR)
-    offset -= dh->pending_len;
+    offset -= dh->outbound_pending;
 
   offset = (*dh->cbs->seek) (dh, offset, whence);
   if (offset >= 0)
-    dh->pending_len = 0;
+    dh->outbound_pending = 0;
 
   return TRACE_SYSRES ((int)offset);
 }
@@ -555,6 +570,28 @@ gpgme_data_set_flag (gpgme_data_t dh, const char *name, const char *value)
     {
       dh->size_hint= value? _gpgme_string_to_off (value) : 0;
     }
+  else if (!strcmp (name, "io-buffer-size"))
+    {
+      gpgme_off_t val;
+
+      /* We may set this only once.  */
+      if (dh->io_buffer_size)
+        return gpg_error (GPG_ERR_CONFLICT);
+
+      val = value? _gpgme_string_to_off (value) : 0;
+      if (val > 1024*1024)
+        val = 1024*1024;  /* Cap at 1MiB */
+      else if (val < BUFFER_SIZE)
+        val = 0;          /* We can use the default buffer.  */
+
+      /* Actual allocation happens as needed but we round it to a
+       * multiple of 1k. */
+      dh->io_buffer_size = ((val + 1023)/1024)*1024;
+    }
+  else if (!strcmp (name, "sensitive"))
+    {
+      dh->sensitive = (value && *value)? !!atoi (value) : 0;
+    }
   else
     return gpg_error (GPG_ERR_UNKNOWN_NAME);
 
@@ -569,14 +606,35 @@ gpgme_error_t
 _gpgme_data_inbound_handler (void *opaque, int fd)
 {
   struct io_cb_data *data = (struct io_cb_data *) opaque;
+  gpg_error_t err;
   gpgme_data_t dh = (gpgme_data_t) data->handler_value;
-  char buffer[BUFFER_SIZE];
+  char bufferspace[BUFFER_SIZE];
-  char *bufp = buffer;
+  char *buffer;
+  size_t buffer_size;
+  char *bufp;
   gpgme_ssize_t buflen;
   TRACE_BEG  (DEBUG_CTX, "_gpgme_data_inbound_handler", dh,
 	      "fd=%d", fd);
 
-  buflen = _gpgme_io_read (fd, buffer, BUFFER_SIZE);
+  if (dh->io_buffer_size)
+    {
+      if (!dh->inbound_buffer)
+        {
+          dh->inbound_buffer = malloc (dh->io_buffer_size);
+          if (!dh->inbound_buffer)
+            return TRACE_ERR (gpg_error_from_syserror ());
+        }
+      buffer_size = dh->io_buffer_size;
+      buffer = dh->inbound_buffer;
+    }
+  else
+    {
+      buffer_size = BUFFER_SIZE;
+      buffer = bufferspace;
+    }
+  bufp = buffer;
+
+  buflen = _gpgme_io_read (fd, buffer, buffer_size);
   if (buflen < 0)
     return gpg_error_from_syserror ();
   if (buflen == 0)
@@ -589,12 +647,21 @@ _gpgme_data_inbound_handler (void *opaque, int fd)
     {
       gpgme_ssize_t amt = gpgme_data_write (dh, bufp, buflen);
       if (amt == 0 || (amt < 0 && errno != EINTR))
-	return TRACE_ERR (gpg_error_from_syserror ());
+	{
+          err = gpg_error_from_syserror ();
+          goto leave;
+        }
       bufp += amt;
       buflen -= amt;
     }
   while (buflen > 0);
-  return TRACE_ERR (0);
+  err = 0;
+
+ leave:
+  if (dh->sensitive && buffer == bufferspace)
+    _gpgme_wipememory (bufferspace, BUFFER_SIZE);
+
+  return TRACE_ERR (err);
 }
 
 
@@ -603,13 +670,34 @@ _gpgme_data_outbound_handler (void *opaque, int fd)
 {
   struct io_cb_data *data = (struct io_cb_data *) opaque;
   gpgme_data_t dh = (gpgme_data_t) data->handler_value;
+  char *buffer;
+  size_t buffer_size;
   gpgme_ssize_t nwritten;
   TRACE_BEG  (DEBUG_CTX, "_gpgme_data_outbound_handler", dh,
 	      "fd=%d", fd);
 
-  if (!dh->pending_len)
+  if (dh->io_buffer_size)
     {
-      gpgme_ssize_t amt = gpgme_data_read (dh, dh->pending, BUFFER_SIZE);
+      if (!dh->outbound_buffer)
+        {
+          dh->outbound_buffer = malloc (dh->io_buffer_size);
+          if (!dh->outbound_buffer)
+            return TRACE_ERR (gpg_error_from_syserror ());
+          dh->outbound_pending = 0;
+        }
+      buffer_size = dh->io_buffer_size;
+      buffer = dh->outbound_buffer;
+    }
+  else
+    {
+      buffer_size = BUFFER_SIZE;
+      buffer = dh->outboundspace;
+    }
+
+
+  if (!dh->outbound_pending)
+    {
+      gpgme_ssize_t amt = gpgme_data_read (dh, buffer, buffer_size);
       if (amt < 0)
 	return TRACE_ERR (gpg_error_from_syserror ());
       if (amt == 0)
@@ -617,10 +705,10 @@ _gpgme_data_outbound_handler (void *opaque, int fd)
 	  _gpgme_io_close (fd);
 	  return TRACE_ERR (0);
 	}
-      dh->pending_len = amt;
+      dh->outbound_pending = amt;
     }
 
-  nwritten = _gpgme_io_write (fd, dh->pending, dh->pending_len);
+  nwritten = _gpgme_io_write (fd, buffer, dh->outbound_pending);
   if (nwritten == -1 && errno == EAGAIN)
     return TRACE_ERR (0);
 
@@ -637,9 +725,9 @@ _gpgme_data_outbound_handler (void *opaque, int fd)
   if (nwritten <= 0)
     return TRACE_ERR (gpg_error_from_syserror ());
 
-  if (nwritten < dh->pending_len)
+  if (nwritten < dh->outbound_pending)
-    memmove (dh->pending, dh->pending + nwritten, dh->pending_len - nwritten);
+    memmove (buffer, buffer + nwritten, dh->outbound_pending - nwritten);
-  dh->pending_len -= nwritten;
+  dh->outbound_pending -= nwritten;
   return TRACE_ERR (0);
 }
 

src/data.h

@@ -33,6 +33,22 @@
 
 #include "gpgme.h"
 
+/* Figure out the standard size for internal data buffers.  */
+#ifdef PIPE_BUF
+# define BUFFER_SIZE PIPE_BUF
+#else
+# ifdef _POSIX_PIPE_BUF
+#  define BUFFER_SIZE _POSIX_PIPE_BUF
+# else
+#  ifdef HAVE_W32_SYSTEM
+#   define BUFFER_SIZE 4096
+#  else
+#   define BUFFER_SIZE 512
+#  endif
+# endif
+#endif
+
+
 
 /* Read up to SIZE bytes into buffer BUFFER from the data object with
    the handle DH.  Return the number of characters read, 0 on EOF and
@@ -76,28 +92,33 @@ struct gpgme_data
   gpgme_data_encoding_t encoding;
   unsigned int propidx;  /* Index into the property table.  */
 
-#ifdef PIPE_BUF
-#define BUFFER_SIZE PIPE_BUF
-#else
-#ifdef _POSIX_PIPE_BUF
-#define BUFFER_SIZE _POSIX_PIPE_BUF
-#else
-#ifdef HAVE_W32_SYSTEM
-#define BUFFER_SIZE 4096
-#else
-#define BUFFER_SIZE 512
-#endif
-#endif
-#endif
-  char pending[BUFFER_SIZE];
-  int pending_len;
-
   /* File name of the data object.  */
   char *file_name;
 
   /* Hint on the to be expected total size of the data.  */
   gpgme_off_t size_hint;
 
+  /* If no 0 the size of an allocated inbound or outpund buffers.  The
+   * value is at least BUFFER_SIZE and capped at 1MiB.  */
+  unsigned int io_buffer_size;
+
+  /* If not NULL a malloced buffer used for inbound data used instead
+   * of the handler's static buffer.  Its size is io_buffer_size.  */
+  char *inbound_buffer;
+
+  /* A default memory space for the outbound handler and the number of
+   * actual pending bytes.  If outbound_buffer is not NULL, this is a
+   * malloced buffer used instead of the outboundspace.  Its malloced
+   * size is io_buffer_size. */
+  char outboundspace[BUFFER_SIZE];
+  unsigned int outbound_pending;
+  char *outbound_buffer;
+
+  /* If set sensitive data is conveyed via the internal buffer.  This
+   * flags overwrites the memory of the buffers with zero before they
+   * are released. */
+  unsigned int sensitive:1;
+
   union
   {
     /* For gpgme_data_new_from_fd.  */

src/util.h

@@ -96,6 +96,9 @@ int _gpgme_ttyname_r (int fd, char *buf, size_t buflen);
 
 /*-- conversion.c --*/
 
+/* Make sure to to erase the memory (PTR,LEN).  */
+void _gpgme_wipememory (void *ptr, size_t len);
+
 /* Concatenate the string S1 with all the following strings up to a
    NULL.  Returns a malloced buffer with the new string or NULL on a
    malloc error or if too many arguments are given.  */

tests/run-decrypt.c

@@ -89,6 +89,8 @@ show_usage (int ex)
          "  --no-symkey-cache               disable the use of that cache\n"
          "  --ignore-mdc-error              allow decryption of legacy data\n"
          "  --unwrap         remove only the encryption layer\n"
+         "  --large-buffers  use large I/O buffer\n"
+         "  --sensitive      mark data objects as sensitive\n"
          "  --diagnostics    print diagnostics\n"
          , stderr);
   exit (ex);
@@ -114,6 +116,8 @@ main (int argc, char **argv)
   int no_symkey_cache = 0;
   int ignore_mdc_error = 0;
   int raw_output = 0;
+  int large_buffers = 0;
+  int sensitive = 0;
   int diagnostics = 0;
 
   if (argc)
@@ -185,6 +189,16 @@ main (int argc, char **argv)
           diagnostics = 1;
           argc--; argv++;
         }
+      else if (!strcmp (*argv, "--large-buffers"))
+        {
+          large_buffers = 1;
+          argc--; argv++;
+        }
+      else if (!strcmp (*argv, "--sensitive"))
+        {
+          sensitive = 1;
+          argc--; argv++;
+        }
       else if (!strcmp (*argv, "--unwrap"))
         {
           flags |= GPGME_DECRYPT_UNWRAP;
@@ -288,6 +302,26 @@ main (int argc, char **argv)
                gpgme_strerror (err));
       exit (1);
     }
+  if (large_buffers)
+    {
+      err = gpgme_data_set_flag (out, "io-buffer-size", "1000000");
+      if (err)
+        {
+          fprintf (stderr, PGM ": error setting io-buffer-size (out): %s\n",
+                   gpgme_strerror (err));
+          exit (1);
+        }
+    }
+  if (sensitive)
+    {
+      err = gpgme_data_set_flag (out, "sensitive", "1");
+      if (err)
+        {
+          fprintf (stderr, PGM ": error setting sensitive flag (out): %s\n",
+                   gpgme_strerror (err));
+          exit (1);
+        }
+    }
 
   err = gpgme_op_decrypt_ext (ctx, flags, in, out);
   result = gpgme_op_decrypt_result (ctx);