More robust detection of handle and sockets

src/ChangeLog

@@ -1,3 +1,8 @@
+2010-04-16  Werner Koch  <wk@g10code.com>
+
+	* w32-io.c (is_socket): New.
+	(reader, writer): Use it to figure out the API to use.
+
 2010-03-15  Werner Koch  <wk@g10code.com>
 
 	* gpgme.h.in: Add autoconf template to set generated file to

src/w32-io.c

@@ -1,6 +1,6 @@
 /* w32-io.c - W32 API I/O functions.
    Copyright (C) 2000 Werner Koch (dd9jn)
-   Copyright (C) 2001, 2002, 2003, 2004, 2007 g10 Code GmbH
+   Copyright (C) 2001, 2002, 2003, 2004, 2007, 2010 g10 Code GmbH
 
    This file is part of GPGME.
  
@@ -175,16 +175,46 @@ set_synchronize (HANDLE hd)
 }
 
 
+/* Return true if HD refers to a socket.  */
+static int
+is_socket (HANDLE hd)
+{
+  /* We need to figure out whether we are working on a socket or on a
+     handle.  A trivial way would be to check for the return code of
+     recv and see if it is WSAENOTSOCK.  However the recv may block
+     after the server process died and thus the destroy_reader will
+     hang.  Another option is to use getsockopt to test whether it is
+     a socket.  The bug here is that once a socket with a certain
+     values has been opened, closed and later a CreatePipe returned
+     the same value (i.e. handle), getsockopt still believes it is a
+     socket.  What we do now is to use a combination of GetFileType
+     and GetNamedPipeInfo.  The specs say that the latter may be used
+     on anonymous pipes as well.  Note that there are claims that
+     since winsocket version 2 ReadFile may be used on a socket but
+     only if it is supported by the service provider.  Tests on a
+     stock XP using a local TCP socket show that it does not work.  */
+  DWORD dummyflags, dummyoutsize, dummyinsize, dummyinst;
+  if (GetFileType (hd) == FILE_TYPE_PIPE
+      && !GetNamedPipeInfo (hd, &dummyflags, &dummyoutsize,
+                            &dummyinsize, &dummyinst))
+    return 1; /* Function failed; thus we assume it is a socket.  */
+  else
+    return 0; /* Success; this is not a socket.  */
+}
+
+
 static DWORD CALLBACK 
 reader (void *arg)
 {
   struct reader_context_s *ctx = arg;
   int nbytes;
   DWORD nread;
-  int try_recv = 1;
+  int sock;
   TRACE_BEG1 (DEBUG_SYSIO, "gpgme:reader", ctx->file_hd,
 	      "thread=%p", ctx->thread_hd);
 
+  sock = is_socket (ctx->file_hd);
+
   for (;;)
     {
       LOCK (ctx->mutex);
@@ -212,17 +242,15 @@ reader (void *arg)
 	nbytes = READBUF_SIZE - ctx->writepos;
       UNLOCK (ctx->mutex);
       
-      TRACE_LOG1 ("reading %d bytes", nbytes);
+      TRACE_LOG2 ("%s %d bytes", sock? "receiving":"reading", nbytes);
 
-      if (try_recv)
+      if (sock)
         {
           int n;
 
           n = recv (handle_to_socket (ctx->file_hd),
                     ctx->buffer + ctx->writepos, nbytes, 0);
-          if (n < 0 && WSAGetLastError () == WSAENOTSOCK)
+          if (n < 0)
-            try_recv = 0;
-          else if (n < 0)
             {
               ctx->error_code = (int) WSAGetLastError ();
               if (ctx->error_code == ERROR_BROKEN_PIPE)
@@ -237,11 +265,9 @@ reader (void *arg)
                 }
               break;
             }
-          else
           nread = n;
-          
         }
-      if (!try_recv)
+      else
         {
           if (!ReadFile (ctx->file_hd,
                          ctx->buffer + ctx->writepos, nbytes, &nread, NULL))
@@ -540,10 +566,12 @@ writer (void *arg)
 {
   struct writer_context_s *ctx = arg;
   DWORD nwritten;
-  int try_send = 1;
+  int sock;
   TRACE_BEG1 (DEBUG_SYSIO, "gpgme:writer", ctx->file_hd,
 	      "thread=%p", ctx->thread_hd);
 
+  sock = is_socket (ctx->file_hd);
+
   for (;;)
     {
       LOCK (ctx->mutex);
@@ -571,11 +599,12 @@ writer (void *arg)
         }
       UNLOCK (ctx->mutex);
       
-      TRACE_LOG1 ("writing %d bytes", ctx->nbytes);
+      TRACE_LOG2 ("%s %d bytes", sock?"sending":"writing", ctx->nbytes);
+
       /* Note that CTX->nbytes is not zero at this point, because
 	 _gpgme_io_write always writes at least 1 byte before waking
 	 us up, unless CTX->stop_me is true, which we catch above.  */
-      if (try_send)
+      if (sock)
         {
           /* We need to try send first because a socket handle can't
              be used with WriteFile.  */
@@ -583,19 +612,16 @@ writer (void *arg)
           
           n = send (handle_to_socket (ctx->file_hd),
                     ctx->buffer, ctx->nbytes, 0);
-          if (n < 0 && WSAGetLastError () == WSAENOTSOCK)
+          if (n < 0)
-            try_send = 0;
-          else if (n < 0)
             {
               ctx->error_code = (int) WSAGetLastError ();
               ctx->error = 1;
               TRACE_LOG1 ("send error: ec=%d", ctx->error_code);
               break;
             }
-          else
           nwritten = n;
         }
-      if (!try_send)
+      else
         {
           if (!WriteFile (ctx->file_hd, ctx->buffer,
                           ctx->nbytes, &nwritten, NULL))