Fix Solaris problems with ttyname_r.

* m4/gnupg-ttyname.m4: New.  Based on ttyname_r from gnulib.
* src/ttyname_r.c (_gpgme_ttyname_r): Rename from ttyname_r.
Implement hacks required for Solaris and possible other non-fully
Posix systems.
* src/util.h: Include unistd.h.  Redefine ttyname_r depending on
REPLACE_TTYNAME_R and put it into the gpgme name space.
--

Unfortunately we cant not use the ttyname_r replacement from gnulib
because we want to keep GPGME LGPLv2+.

4 files changed, 157 insertions, 56 deletions

diff --git a/configure.ac b/configure.ac
index 3ed9c4b7..6e0967c2 100644
--- a/configure.ac
+++ b/configure.ac
@@ -194,7 +194,6 @@ AM_CONDITIONAL(HAVE_DOSISH_SYSTEM, test "$have_dosish_system" = yes)
 if test "$have_w32_system" = yes; then
    AC_DEFINE(HAVE_W32_SYSTEM,1,
              [Defined if we run on any kind of W32 API based system])
-   ACSUBST
 fi
 AM_CONDITIONAL(HAVE_W32_SYSTEM, test "$have_w32_system" = yes)
 
@@ -286,7 +285,7 @@ if test "$ac_cv_func_vasprintf" != yes; then
 fi
 
 # Try to find a thread-safe version of ttyname().
-AC_REPLACE_FUNCS(ttyname_r)
+gnupg_REPLACE_TTYNAME_R
 if test "$ac_cv_func_ttyname_r" != yes; then
   AC_MSG_WARN([
 ***
diff --git a/m4/gnupg-ttyname.m4 b/m4/gnupg-ttyname.m4
index f0ff554d..d9a0e2e0 100644
--- a/m4/gnupg-ttyname.m4
+++ b/m4/gnupg-ttyname.m4
@@ -1,16 +1,35 @@
-# ttyname_r.m4 serial 8
-dnl Copyright (C) 2010-2012 Free Software Foundation, Inc.
-dnl This file is free software; the Free Software Foundation
-dnl gives unlimited permission to copy and/or distribute it,
-dnl with or without modifications, as long as this notice is preserved.
+# gnupg-ttyname.m4
+# Copyright (C) 2010-2012 Free Software Foundation, Inc.
+# This file is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+#
+# This file is based on gnulib/m4/ttyname_r.m4 serial 8.
+#
 
-AC_DEFUN([gl_FUNC_TTYNAME_R],
-[
-  AC_REQUIRE([gl_UNISTD_H_DEFAULTS])
 
-  dnl Persuade Solaris <unistd.h> to provide the POSIX compliant declaration of
-  dnl ttyname_r().
-  AC_REQUIRE([gl_USE_SYSTEM_EXTENSIONS])
+# gnupg_REPLACE_TTYNAME_R
+#
+# This macro is an extended version of AC_REPLACE_FUNCS(ttyname_r).
+# It takes peculiarities in the implementation of ttyname_r in account.
+#
+# The macro HAVE_TTYNAME_R will be defined to 1 if the function
+# exists; it will be defined to 0 if it does not exists or no
+# declaration is available.
+#
+# The macro HAVE_POSIXDECL_TTYNAME_R is defined if ttyname_r conforms
+# to the Posix declaration.
+#
+# The macro HAVE_BROKEN_TTYNAME_R is defined it ttyname_r does not work
+# correctly with the supplied buffer size.  If this is defined the function
+# will also be replaced.
+#
+# The macro REPLACE_TTYNAME_R is defined if ttyname_r is a replacement
+# function.  This macro is useful for the definition of the prototype.
+#
+AC_DEFUN([gnupg_REPLACE_TTYNAME_R],
+[
+  AC_CHECK_HEADERS([unistd.h])
 
   AC_CHECK_DECLS_ONCE([ttyname_r])
   if test $ac_cv_have_decl_ttyname_r = no; then
@@ -20,23 +39,27 @@ AC_DEFUN([gl_FUNC_TTYNAME_R],
   AC_CHECK_FUNCS([ttyname_r])
   if test $ac_cv_func_ttyname_r = no; then
     HAVE_TTYNAME_R=0
+    AC_LIBOBJ([ttyname_r])
+    AC_DEFINE([REPLACE_TTYNAME_R],[1],
+              [Define to 1 if ttyname_r is a replacement function.])
   else
     HAVE_TTYNAME_R=1
-    dnl On MacOS X 10.4 (and Solaris 10 without gl_USE_SYSTEM_EXTENSIONS)
+    dnl On MacOS X 10.4 (and Solaris 10 without __EXTENSIONS__)
     dnl the return type is 'char *', not 'int'.
     AC_CACHE_CHECK([whether ttyname_r is compatible with its POSIX signature],
-      [gl_cv_func_ttyname_r_posix],
+      [gnupg_cv_func_ttyname_r_posix],
       [AC_COMPILE_IFELSE(
          [AC_LANG_PROGRAM(
             [[#include <stddef.h>
               #include <unistd.h>]],
             [[*ttyname_r (0, NULL, 0);]])
          ],
-         [gl_cv_func_ttyname_r_posix=no],
-         [gl_cv_func_ttyname_r_posix=yes])
+         [gnupg_cv_func_ttyname_r_posix=no],
+         [gnupg_cv_func_ttyname_r_posix=yes])
       ])
-    if test $gl_cv_func_ttyname_r_posix = no; then
-      REPLACE_TTYNAME_R=1
+    if test $gnupg_cv_func_ttyname_r_posix = no; then
+      AC_LIBOBJ([ttyname_r])
+      AC_DEFINE([REPLACE_TTYNAME_R],[1])
     else
       AC_DEFINE([HAVE_POSIXDECL_TTYNAME_R], [1],
         [Define if the ttyname_r function has a POSIX compliant declaration.])
@@ -47,18 +70,18 @@ AC_DEFUN([gl_FUNC_TTYNAME_R],
       dnl buffer is large enough.
       AC_REQUIRE([AC_CANONICAL_HOST])
       AC_CACHE_CHECK([whether ttyname_r works with small buffers],
-        [gl_cv_func_ttyname_r_works],
+        [gnupg_cv_func_ttyname_r_works],
         [
           dnl Initial guess, used when cross-compiling or when /dev/tty cannot
           dnl be opened.
 changequote(,)dnl
           case "$host_os" in
                       # Guess no on Solaris.
-            solaris*) gl_cv_func_ttyname_r_works="guessing no" ;;
+            solaris*) gnupg_cv_func_ttyname_r_works="guessing no" ;;
                       # Guess no on OSF/1.
-            osf*)     gl_cv_func_ttyname_r_works="guessing no" ;;
+            osf*)     gnupg_cv_func_ttyname_r_works="guessing no" ;;
                       # Guess yes otherwise.
-            *)        gl_cv_func_ttyname_r_works="guessing yes" ;;
+            *)        gnupg_cv_func_ttyname_r_works="guessing yes" ;;
           esac
 changequote([,])dnl
           AC_RUN_IFELSE(
@@ -81,21 +104,20 @@ main (void)
     result |= 18;
   return result;
 }]])],
-            [gl_cv_func_ttyname_r_works=yes],
+            [gnupg_cv_func_ttyname_r_works=yes],
             [case $? in
-               17 | 18) gl_cv_func_ttyname_r_works=no ;;
+               17 | 18) gnupg_cv_func_ttyname_r_works=no ;;
              esac],
             [:])
         ])
-      case "$gl_cv_func_ttyname_r_works" in
+      case "$gnupg_cv_func_ttyname_r_works" in
         *yes) ;;
-        *) REPLACE_TTYNAME_R=1 ;;
+        *) AC_LIBOBJ([ttyname_r])
+           AC_DEFINE([REPLACE_TTYNAME_R],[1])
+           AC_DEFINE([HAVE_BROKEN_TTYNAME_R], [1],
+                     [Define if ttyname_r is does not work with small buffers])
+           ;;
       esac
     fi
   fi
 ])
-
-# Prerequisites of lib/ttyname_r.c.
-AC_DEFUN([gl_PREREQ_TTYNAME_R], [
-  AC_CHECK_FUNCS([ttyname])
-])
diff --git a/src/ttyname_r.c b/src/ttyname_r.c
index 810c2175..105e0af5 100644
--- a/src/ttyname_r.c
+++ b/src/ttyname_r.c
@@ -1,22 +1,21 @@
 /* ttyname_r.c - A ttyname_r() replacement.
-   Copyright (C) 2003, 2004 g10 Code GmbH
+   Copyright (C) 2003, 2004, 2012 g10 Code GmbH
 
    This file is part of GPGME.
- 
+
    GPGME is free software; you can redistribute it and/or modify it
    under the terms of the GNU Lesser General Public License as
    published by the Free Software Foundation; either version 2.1 of
    the License, or (at your option) any later version.
-   
+
    GPGME is distributed in the hope that it will be useful, but
    WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
    Lesser General Public License for more details.
-   
+
    You should have received a copy of the GNU Lesser General Public
-   License along with this program; if not, write to the Free Software
-   Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
-   02111-1307, USA.  */
+   License along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
 
 #if HAVE_CONFIG_H
 #include <config.h>
@@ -29,29 +28,103 @@
 #endif
 
 
-#ifdef __GNUC__
+#if !HAVE_TTYNAME_R && defined(__GNUC__)
 # warning ttyname is not thread-safe, and ttyname_r is missing
 #endif
 
 int
-ttyname_r (int fd, char *buf, size_t buflen)
+_gpgme_ttyname_r (int fd, char *buf, size_t buflen)
 {
+#if HAVE_TTYNAME_R
+# if HAVE_BROKEN_TTYNAME_R
+   /* Solaris fails if BUFLEN is less than 128. OSF/1 5.1 completely
+      ignores BUFLEN.  We use a large buffer to woraround this.  */
+  {
+    char largebuf[512];
+    size_t namelen;
+    int rc;
+
+#  if HAVE_POSIXDECL_TTYNAME_R
+    if (buflen < sizeof (largebuf))
+      {
+        rc = ttyname_r (fd, largebuf, (int)sizeof (largebuf));
+        if (!rc)
+          {
+            namelen = strlen (largebuf) + 1;
+            if (namelen > buflen)
+              rc = ERANGE;
+            else
+              memcpy (buf, largebuf, namelen);
+          }
+      }
+    else
+      rc = ttyname_r (fd, buf, (int)buflen);
+
+#  else /*!HAVE_POSIXDECL_TTYNAME_R*/
+    char *name;
+
+    if (buflen < sizeof (largebuf))
+      name = ttyname_r (fd, largebuf, (int)sizeof (largebuf));
+    else
+      name = ttyname_r (fd, buf, (int)buflen);
+    rc = name? 0 : (errno? errno : -1);
+    if (!rc && buf != name)
+      {
+        namelen = strlen (name) + 1;
+        if (namelen > buflen)
+          rc = ERANGE;
+        else
+          memmove (buf, name, namelen);
+      }
+#  endif
+
+    return rc;
+  }
+# else /*!HAVE_BROKEN_TTYNAME_R*/
+  {
+    int rc;
+
+#  if HAVE_POSIXDECL_TTYNAME_R
+
+    rc = ttyname_r (fd, buf, buflen);
+
+#  else /*!HAVE_POSIXDECL_TTYNAME_R*/
+    char *name;
+    size_t namelen;
+
+    name = ttyname_r (fd, buf, (int)buflen);
+    rc = name? 0 : (errno? errno : -1);
+    if (!rc && buf != name)
+      {
+        namelen = strlen (name) + 1;
+        if (namelen > buflen)
+          rc = ERANGE;
+        else
+          memmove (buf, name, namelen);
+      }
+#  endif
+
+    return rc;
+  }
+# endif /*!HAVE_BROKEN_TTYNAME_R*/
+#else /*!HAVE_TTYNAME_R*/
   char *tty;
 
-#if HAVE_W32_SYSTEM
+# if HAVE_W32_SYSTEM
   /* We use this default one for now.  AFAICS we only need it to be
      passed to gpg and in turn to pinentry.  Providing a replacement
      is needed because elsewhere we bail out on error.  If we
-     eventually implement a pinentry for Windows it is uinlikely that
+     eventually implement a pinentry for Windows it is inlikely that
      we need a real tty at all.  */
-  tty = "/dev/tty"; 
-#else
+  tty = "/dev/tty";
+# else
   tty = ttyname (fd);
   if (!tty)
-    return errno;
-#endif
-  
+    return errno? errno : -1;
+# endif
+
   strncpy (buf, tty, buflen);
   buf[buflen - 1] = '\0';
   return (strlen (tty) >= buflen) ? ERANGE : 0;
+#endif /*!HAVE_TTYNAME_R*/
 }
diff --git a/src/util.h b/src/util.h
index 6c2cec69..27620aa7 100644
--- a/src/util.h
+++ b/src/util.h
@@ -1,19 +1,19 @@
-/* util.h 
+/* util.h
    Copyright (C) 2000 Werner Koch (dd9jn)
    Copyright (C) 2001, 2002, 2003, 2004, 2005 g10 Code GmbH
 
    This file is part of GPGME.
- 
+
    GPGME is free software; you can redistribute it and/or modify it
    under the terms of the GNU Lesser General Public License as
    published by the Free Software Foundation; either version 2.1 of
    the License, or (at your option) any later version.
-   
+
    GPGME is distributed in the hope that it will be useful, but
    WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
    Lesser General Public License for more details.
-   
+
    You should have received a copy of the GNU Lesser General Public
    License along with this program; if not, write to the Free Software
    Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
@@ -34,6 +34,10 @@
 #ifdef HAVE_SYS_TYPES_H
 # include <sys/types.h>
 #endif
+/* We must see the symbol ttyname_r before a redefinition. */
+#ifdef HAVE_UNISTD_H
+# include <unistd.h>
+#endif
 
 #include "gpgme.h"
 
@@ -78,11 +82,14 @@ int vasprintf (char **result, const char *format, va_list args);
 int asprintf (char **result, const char *format, ...);
 #endif
 
-#ifndef HAVE_TTYNAME_R
-int ttyname_r (int fd, char *buf, size_t buflen);
-#endif
+#if REPLACE_TTYNAME_R
+int _gpgme_ttyname_r (int fd, char *buf, size_t buflen);
+#undef  ttyname_r
+#define ttyname_r(a,b,c) _gpgme_ttyname_r ((a), (b), (c))
 #endif
 
+#endif /*HAVE_CONFIG_H*/
+
 
 /*-- conversion.c --*/
 /* Convert two hexadecimal digits from STR to the value they
@@ -142,7 +149,7 @@ char *_gpgme_w32ce_get_debug_envvar (void);
 #ifndef GPG_ERR_UNFINISHED
 #define GPG_ERR_UNFINISHED 199
 #endif
-#ifndef GPG_ERR_NOT_OPERATIONAL 
+#ifndef GPG_ERR_NOT_OPERATIONAL
 #define GPG_ERR_NOT_OPERATIONAL 176
 #endif
 #ifndef GPG_ERR_MISSING_ISSUER_CERT