From 779823c09cba151c3eb9043bc8ce28adb0dd737d Mon Sep 17 00:00:00 2001
From: Werner Koch <wk@gnupg.org>
Date: Fri, 16 Apr 2010 14:08:41 +0000
Subject: [PATCH] More robust detection of handle and sockets

---
 src/ChangeLog |  5 ++++
 src/w32-io.c  | 66 +++++++++++++++++++++++++++++++++++----------------
 2 files changed, 51 insertions(+), 20 deletions(-)

diff --git a/src/ChangeLog b/src/ChangeLog
index 7be7b5f3..d4ef220d 100644
--- a/src/ChangeLog
+++ b/src/ChangeLog
@@ -1,3 +1,8 @@
+2010-04-16  Werner Koch  <wk@g10code.com>
+
+	* w32-io.c (is_socket): New.
+	(reader, writer): Use it to figure out the API to use.
+
 2010-03-15  Werner Koch  <wk@g10code.com>
 
 	* gpgme.h.in: Add autoconf template to set generated file to
diff --git a/src/w32-io.c b/src/w32-io.c
index 762d617f..d05db702 100644
--- a/src/w32-io.c
+++ b/src/w32-io.c
@@ -1,6 +1,6 @@
 /* w32-io.c - W32 API I/O functions.
    Copyright (C) 2000 Werner Koch (dd9jn)
-   Copyright (C) 2001, 2002, 2003, 2004, 2007 g10 Code GmbH
+   Copyright (C) 2001, 2002, 2003, 2004, 2007, 2010 g10 Code GmbH
 
    This file is part of GPGME.
  
@@ -175,16 +175,46 @@ set_synchronize (HANDLE hd)
 }
 
 
+/* Return true if HD refers to a socket.  */
+static int
+is_socket (HANDLE hd)
+{
+  /* We need to figure out whether we are working on a socket or on a
+     handle.  A trivial way would be to check for the return code of
+     recv and see if it is WSAENOTSOCK.  However the recv may block
+     after the server process died and thus the destroy_reader will
+     hang.  Another option is to use getsockopt to test whether it is
+     a socket.  The bug here is that once a socket with a certain
+     values has been opened, closed and later a CreatePipe returned
+     the same value (i.e. handle), getsockopt still believes it is a
+     socket.  What we do now is to use a combination of GetFileType
+     and GetNamedPipeInfo.  The specs say that the latter may be used
+     on anonymous pipes as well.  Note that there are claims that
+     since winsocket version 2 ReadFile may be used on a socket but
+     only if it is supported by the service provider.  Tests on a
+     stock XP using a local TCP socket show that it does not work.  */
+  DWORD dummyflags, dummyoutsize, dummyinsize, dummyinst;
+  if (GetFileType (hd) == FILE_TYPE_PIPE
+      && !GetNamedPipeInfo (hd, &dummyflags, &dummyoutsize,
+                            &dummyinsize, &dummyinst))
+    return 1; /* Function failed; thus we assume it is a socket.  */
+  else
+    return 0; /* Success; this is not a socket.  */
+}
+
+
 static DWORD CALLBACK 
 reader (void *arg)
 {
   struct reader_context_s *ctx = arg;
   int nbytes;
   DWORD nread;
-  int try_recv = 1;
+  int sock;
   TRACE_BEG1 (DEBUG_SYSIO, "gpgme:reader", ctx->file_hd,
 	      "thread=%p", ctx->thread_hd);
 
+  sock = is_socket (ctx->file_hd);
+
   for (;;)
     {
       LOCK (ctx->mutex);
@@ -212,17 +242,15 @@ reader (void *arg)
 	nbytes = READBUF_SIZE - ctx->writepos;
       UNLOCK (ctx->mutex);
       
-      TRACE_LOG1 ("reading %d bytes", nbytes);
+      TRACE_LOG2 ("%s %d bytes", sock? "receiving":"reading", nbytes);
 
-      if (try_recv)
+      if (sock)
         {
           int n;
 
           n = recv (handle_to_socket (ctx->file_hd),
                     ctx->buffer + ctx->writepos, nbytes, 0);
-          if (n < 0 && WSAGetLastError () == WSAENOTSOCK)
-            try_recv = 0;
-          else if (n < 0)
+          if (n < 0)
             {
               ctx->error_code = (int) WSAGetLastError ();
               if (ctx->error_code == ERROR_BROKEN_PIPE)
@@ -237,11 +265,9 @@ reader (void *arg)
                 }
               break;
             }
-          else
-            nread = n;
-          
+          nread = n;
         }
-      if (!try_recv)
+      else
         {
           if (!ReadFile (ctx->file_hd,
                          ctx->buffer + ctx->writepos, nbytes, &nread, NULL))
@@ -540,10 +566,12 @@ writer (void *arg)
 {
   struct writer_context_s *ctx = arg;
   DWORD nwritten;
-  int try_send = 1;
+  int sock;
   TRACE_BEG1 (DEBUG_SYSIO, "gpgme:writer", ctx->file_hd,
 	      "thread=%p", ctx->thread_hd);
 
+  sock = is_socket (ctx->file_hd);
+
   for (;;)
     {
       LOCK (ctx->mutex);
@@ -571,11 +599,12 @@ writer (void *arg)
         }
       UNLOCK (ctx->mutex);
       
-      TRACE_LOG1 ("writing %d bytes", ctx->nbytes);
+      TRACE_LOG2 ("%s %d bytes", sock?"sending":"writing", ctx->nbytes);
+
       /* Note that CTX->nbytes is not zero at this point, because
 	 _gpgme_io_write always writes at least 1 byte before waking
 	 us up, unless CTX->stop_me is true, which we catch above.  */
-      if (try_send)
+      if (sock)
         {
           /* We need to try send first because a socket handle can't
              be used with WriteFile.  */
@@ -583,19 +612,16 @@ writer (void *arg)
           
           n = send (handle_to_socket (ctx->file_hd),
                     ctx->buffer, ctx->nbytes, 0);
-          if (n < 0 && WSAGetLastError () == WSAENOTSOCK)
-            try_send = 0;
-          else if (n < 0)
+          if (n < 0)
             {
               ctx->error_code = (int) WSAGetLastError ();
               ctx->error = 1;
               TRACE_LOG1 ("send error: ec=%d", ctx->error_code);
               break;
             }
-          else
-            nwritten = n;
+          nwritten = n;
         }
-      if (!try_send)
+      else
         {
           if (!WriteFile (ctx->file_hd, ctx->buffer,
                           ctx->nbytes, &nwritten, NULL))