python: Wrap objects implementing the buffer protocol.

* lang/python/Makefile.am: Add the toplevel source directory to CFLAGS
when compiling the bindings so that we can use private header files.
* lang/python/gpgme.i (gpgme_data_t): Rework the object wrapping.  Do
not create a Python wrapper object, merely a gpgme_data_t object, and
keep references to buffer objects, if any.  If necessary, update the
buffer after the function call.
(pygpgme_wrap_gpgme_data_t): New function.
* lang/python/helpers.c (object_to_gpgme_data_t): Rework object
wrapping.  Also wrap objects implementing the buffer protocol.
* lang/python/helpers.h (object_to_gpgme_data_t): Update prototype.
(pygpgme_wrap_gpgme_data_t): New prototype.
* lang/python/tests/t-idiomatic.py: Demonstrate this.

Signed-off-by: Justus Winter <justus@g10code.com>

lang/python/Makefile.am

@@ -45,7 +45,7 @@ gpgme_wrap.c pyme/pygpgme.py: gpgme.i errors.i gpgme.h copystamp
 	  $<
 
 all-local: gpgme_wrap.c pyme/pygpgme.py copystamp
-	CFLAGS="$(CFLAGS)" \
+	CFLAGS="$(CFLAGS) -I$(top_srcdir)" \
 	  $(PYTHON) setup.py build --verbose
 
 clean-local:

lang/python/gpgme.i

@@ -114,17 +114,19 @@
 }
 
 // Special handling for references to our objects.
-%typemap(in) gpgme_data_t DATAIN (PyObject *wrapper) {
+%typemap(in) gpgme_data_t DATAIN (gpgme_data_t wrapper = NULL,
+                                  PyObject *bytesio = NULL, Py_buffer view) {
   /* If we create a temporary wrapper object, we will store it in
      wrapperN, where N is $argnum.  Here in this fragment, SWIG will
      automatically append $argnum.  */
-  wrapper = NULL;
+  memset(&view, 0, sizeof view);
   if ($input == Py_None)
     $1 = NULL;
   else {
-    PyObject *pypointer = NULL;
+    PyObject *pypointer;
-
+    pypointer = object_to_gpgme_data_t($input, $argnum, &wrapper,
-    if((pypointer=object_to_gpgme_data_t($input, $argnum, &wrapper)) == NULL)
+                                       &bytesio, &view);
+    if (pypointer == NULL)
       return NULL;
 
     /* input = $input, 1 = $1, 1_descriptor = $1_descriptor */
@@ -141,8 +143,79 @@
 }
 
 %typemap(freearg) gpgme_data_t DATAIN {
+  /* See whether we need to update the Python buffer.  */
+  if (resultobj && wrapper$argnum && view$argnum.buf
+      && wrapper$argnum->data.mem.buffer != NULL)
+    {
+      /* The buffer is dirty.  */
+      if (view$argnum.readonly)
+        {
+          Py_XDECREF(resultobj);
+          resultobj = NULL;
+          PyErr_SetString(PyExc_ValueError, "cannot update read-only buffer");
+        }
+
+      /* See if we need to truncate the buffer.  */
+      if (resultobj && view$argnum.len != wrapper$argnum->data.mem.length)
+        {
+          if (bytesio$argnum == NULL)
+            {
+              Py_XDECREF(resultobj);
+              resultobj = NULL;
+              PyErr_SetString(PyExc_ValueError, "cannot resize buffer");
+            }
+          else
+            {
+              PyObject *retval;
+              PyBuffer_Release(&view$argnum);
+              retval = PyObject_CallMethod(bytesio$argnum, "truncate", "l",
+                                           (long)
+                                           wrapper$argnum->data.mem.length);
+              if (retval == NULL)
+                {
+                  Py_XDECREF(resultobj);
+                  resultobj = NULL;
+                }
+              else
+                {
+                  Py_DECREF(retval);
+
+                  retval = PyObject_CallMethod(bytesio$argnum, "getbuffer", NULL);
+                  if (retval == NULL
+                      || PyObject_GetBuffer(retval, &view$argnum,
+                                            PyBUF_SIMPLE|PyBUF_WRITABLE) < 0)
+                    {
+                      Py_XDECREF(resultobj);
+                      resultobj = NULL;
+                    }
+
+                  Py_XDECREF(retval);
+
+                  if (resultobj && view$argnum.len
+                      != wrapper$argnum->data.mem.length)
+                    {
+                      Py_XDECREF(resultobj);
+                      resultobj = NULL;
+                      PyErr_Format(PyExc_ValueError,
+                                   "Expected buffer of length %zu, got %zi",
+                                   wrapper$argnum->data.mem.length,
+                                   view$argnum.len);
+                    }
+                }
+            }
+        }
+
+      if (resultobj)
+        memcpy(view$argnum.buf, wrapper$argnum->data.mem.buffer,
+               wrapper$argnum->data.mem.length);
+    }
+
   /* Free the temporary wrapper, if any.  */
-  Py_XDECREF(wrapper$argnum);
+  if (wrapper$argnum)
+    gpgme_data_release(wrapper$argnum);
+  Py_XDECREF (bytesio$argnum);
+  if (wrapper$argnum && view$argnum.buf)
+    PyBuffer_Release(&view$argnum);
 }
 
 %apply gpgme_data_t DATAIN {gpgme_data_t plain, gpgme_data_t cipher,
@@ -240,7 +313,10 @@
    version for SWIG.  We do, however, want to hide certain fields on
    some structs, which we provide prior to including the version for
    SWIG.  */
- %{ #include <gpgme.h> %}
+%{
+#include <gpgme.h>
+#include "src/data.h"	/* For struct gpgme_data.  */
+%}
 
 /* This is for notations, where we want to hide the length fields, and
    the unused bit field block.  */
@@ -291,5 +367,13 @@ FILE *fdopen(int fildes, const char *mode);
 
 %{
 #include "helpers.h"
+
+/* SWIG support for helpers.c  */
+PyObject *
+pygpgme_wrap_gpgme_data_t(gpgme_data_t data)
+{
+  return SWIG_NewPointerObj(data, SWIGTYPE_p_gpgme_data, 0);
+}
 %}
+
 %include "helpers.h"

lang/python/helpers.c

@@ -206,64 +206,72 @@ object_to_gpgme_t(PyObject *input, const char *objtype, int argnum)
    objects with a fileno method, returning it in WRAPPER.  This object
    must be de-referenced when no longer needed.  */
 PyObject *
-object_to_gpgme_data_t(PyObject *input, int argnum, PyObject **wrapper)
+object_to_gpgme_data_t(PyObject *input, int argnum, gpgme_data_t *wrapper,
+                       PyObject **bytesio, Py_buffer *view)
 {
-  static PyObject *Data = NULL;
+  gpgme_error_t err;
-  PyObject *data = input;
+  PyObject *data;
   PyObject *fd;
-  PyObject *result;
-  *wrapper = NULL;
-
-  if (Data == NULL) {
-    PyObject *core;
-    PyObject *from_list = PyList_New(0);
-    core = PyImport_ImportModuleLevel("core", PyEval_GetGlobals(),
-                                      PyEval_GetLocals(), from_list, 1);
-    Py_XDECREF(from_list);
-    if (core) {
-      Data = PyDict_GetItemString(PyModule_GetDict(core), "Data");
-      Py_XINCREF(Data);
-    }
-    else
-      return NULL;
-  }
 
+  /* See if it is a file-like object with file number.  */
   fd = PyObject_CallMethod(input, "fileno", NULL);
   if (fd) {
-    /* File-like object with file number.  */
+    err = gpgme_data_new_from_fd(wrapper, (int) PyLong_AsLong(fd));
-    PyObject *args = NULL;
-    PyObject *kw = NULL;
-
-    /* We don't need the fd, as we have no constructor accepting file
-       descriptors directly.  */
     Py_DECREF(fd);
+    if (err)
+      return pygpgme_raise_exception (err);
 
-    args = PyTuple_New(0);
+    return pygpgme_wrap_gpgme_data_t(*wrapper);
-    kw = PyDict_New();
-    if (args == NULL || kw == NULL)
-      {
-      fail:
-        Py_XDECREF(args);
-        Py_XDECREF(kw);
-        return NULL;
-      }
-
-    if (PyDict_SetItemString(kw, "file", input) < 0)
-      goto fail;
-
-    *wrapper = PyObject_Call(Data, args, kw);
-    if (*wrapper == NULL)
-      goto fail;
-
-    Py_DECREF(args);
-    Py_DECREF(kw);
-    data = *wrapper;
   }
   else
     PyErr_Clear();
 
-  result = object_to_gpgme_t(data, "gpgme_data_t", argnum);
+  /* No?  Maybe it implements the buffer protocol.  */
-  return result;
+  data = PyObject_CallMethod(input, "getbuffer", NULL);
+  if (data)
+    {
+      /* Save a reference to input, which seems to be a BytesIO
+         object.  */
+      Py_INCREF(input);
+      *bytesio = input;
+    }
+  else
+    {
+      PyErr_Clear();
+
+      /* No, but maybe the user supplied a buffer object?  */
+      data = input;
+    }
+
+  /* Do we have a buffer object?  */
+  if (PyObject_CheckBuffer(data))
+    {
+      if (PyObject_GetBuffer(data, view, PyBUF_SIMPLE) < 0)
+        return NULL;
+
+      if (data != input)
+        Py_DECREF(data);
+
+      assert (view->ndim == 1);
+      assert (view->shape == NULL);
+      assert (view->strides == NULL);
+      assert (view->suboffsets == NULL);
+
+      err = gpgme_data_new_from_mem(wrapper, view->buf, (size_t) view->len, 0);
+      if (err)
+        return pygpgme_raise_exception (err);
+
+      return pygpgme_wrap_gpgme_data_t(*wrapper);
+    }
+
+  /* As last resort we assume it is a wrapped data object.  */
+  if (PyObject_HasAttrString(data, "_getctype"))
+    return object_to_gpgme_t(data, "gpgme_data_t", argnum);
+
+  return PyErr_Format(PyExc_TypeError,
+                      "arg %d: expected pyme.Data, file, or an object "
+                      "implementing the buffer protocol, got %s",
+                      argnum, data->ob_type->tp_name);
 }
 
 

lang/python/helpers.h

@@ -1,4 +1,5 @@
 /*
+# Copyright (C) 2016 g10 Code GmbH
 # Copyright (C) 2004 Igor Belyi <belyi@users.sourceforge.net>
 # Copyright (C) 2002 John Goerzen <jgoerzen@complete.org>
 #
@@ -30,7 +31,8 @@ gpgme_error_t pygpgme_exception2code(void);
 
 PyObject *object_to_gpgme_t(PyObject *input, const char *objtype, int argnum);
 PyObject *object_to_gpgme_data_t(PyObject *input, int argnum,
-				 PyObject **wrapper);
+				 gpgme_data_t *wrapper,
+				 PyObject **bytesio, Py_buffer *view);
 
 void pygpgme_clear_generic_cb(PyObject **cb);
 PyObject *pygpgme_raise_callback_exception(PyObject *self);
@@ -47,3 +49,6 @@ gpgme_error_t pyEditCb(void *opaque, gpgme_status_code_t status,
 gpgme_error_t pygpgme_data_new_from_cbs(gpgme_data_t *r_data,
                                         PyObject *pycbs,
                                         PyObject **freelater);
+
+/* SWIG support for helpers.c  */
+PyObject *pygpgme_wrap_gpgme_data_t(gpgme_data_t data);

lang/python/tests/t-idiomatic.py

@@ -17,6 +17,7 @@
 # You should have received a copy of the GNU Lesser General Public
 # License along with this program; if not, see <http://www.gnu.org/licenses/>.
 
+import io
 import os
 import tempfile
 from pyme import core, constants, errors
@@ -33,14 +34,7 @@ with core.Context() as c, core.Data() as d:
 assert leak_c.wrapped == None
 assert leak_d.wrapped == None
 
-# Demonstrate automatic wrapping of file-like objects with 'fileno'
+def sign_and_verify(source, signed, sink):
-# method.
-with tempfile.TemporaryFile() as source, \
-     tempfile.TemporaryFile() as signed, \
-     tempfile.TemporaryFile() as sink:
-    source.write(b"Hallo Leute\n")
-    source.seek(0, os.SEEK_SET)
-
     with core.Context() as c:
         c.op_sign(source, signed, constants.SIG_MODE_NORMAL)
         signed.seek(0, os.SEEK_SET)
@@ -54,3 +48,28 @@ with tempfile.TemporaryFile() as source, \
 
     sink.seek(0, os.SEEK_SET)
     assert sink.read() == b"Hallo Leute\n"
+
+# Demonstrate automatic wrapping of file-like objects with 'fileno'
+# method.
+with tempfile.TemporaryFile() as source, \
+     tempfile.TemporaryFile() as signed, \
+     tempfile.TemporaryFile() as sink:
+    source.write(b"Hallo Leute\n")
+    source.seek(0, os.SEEK_SET)
+
+    sign_and_verify(source, signed, sink)
+
+# XXX: Python's io.BytesIo.truncate does not work as advertised.
+# http://bugs.python.org/issue27261
+bio = io.BytesIO()
+bio.truncate(1)
+if len(bio.getvalue()) != 1:
+    # This version of Python is affected, preallocate buffer.
+    preallocate = 128*b'\x00'
+else:
+    preallocate = b''
+
+# Demonstrate automatic wrapping of objects implementing the buffer
+# interface, and the use of data objects with the 'with' statement.
+with io.BytesIO(preallocate) as signed, core.Data() as sink:
+    sign_and_verify(b"Hallo Leute\n", signed, sink)