On VMS use --batch by default if in batch mode.

This problem was identified and solved by Steven M. Schweda.
Note that the vms specific code is not part of this repository.
See http://antinode.info/dec/sw/gnupg.html for the VMS port.

4 files changed, 31 insertions, 5 deletions

diff --git a/g10/ChangeLog b/g10/ChangeLog
index ecfb3db9f..d812cfd42 100644
--- a/g10/ChangeLog
+++ b/g10/ChangeLog
@@ -1,3 +1,9 @@
+2011-09-12  Werner Koch  <[email protected]>
+
+	* gpg.c [__VMS]: Include vms.h.
+	(main) [__VMS]: Init batch mode according to actual process mode.
+	Suggested by Steven M. Schweda.
+
 2011-08-09  Werner Koch  <[email protected]>
 
 	* keyedit.c (show_key_with_all_names): Remove unused var.
diff --git a/g10/gpg.c b/g10/gpg.c
index 28841c0aa..019079047 100644
--- a/g10/gpg.c
+++ b/g10/gpg.c
@@ -39,6 +39,9 @@
 #ifdef HAVE_W32_SYSTEM
 #include <windows.h>
 #endif
+#ifdef __VMS
+# include "vms.h"
+#endif
 
 #define INCLUDED_BY_MAIN_MODULE 1
 #include "packet.h"
@@ -1872,6 +1875,15 @@ main (int argc, char **argv )
     opt.lock_once = 1;
 #endif /* __riscos__ */
 
+#ifdef __VMS
+    /* On VMS, set the default value of the "--[no-]batch" flag
+     * according to the actual process mode.  The user can override
+     * this with an explicit command-line "--[no-]batch" option.  This
+     * avoids that the process stops while trying to initialize the
+     * tty in batch mode.  */
+    opt.batch = batch_mode_vms();
+#endif
+
     reopen_std();
     trap_unaligned();
     secmem_set_flags( secmem_get_flags() | 2 ); /* suspend warnings */
diff --git a/util/ChangeLog b/util/ChangeLog
index 506e1f359..d4caac385 100644
--- a/util/ChangeLog
+++ b/util/ChangeLog
@@ -1,3 +1,8 @@
+2011-09-12  Werner Koch  <[email protected]>
+
+	* ttyio.c (init_ttyfp): Avoid endless loop due to a failed opening
+	of the tty.  Suggested by Steven M. Schweda.
+
 2011-08-09  Werner Koch  <[email protected]>
 
 	* pka.c (get_pka_info): Turn ANSWER into a union to avoid aliasing
diff --git a/util/ttyio.c b/util/ttyio.c
index 05c0a3796..c8177d755 100644
--- a/util/ttyio.c
+++ b/util/ttyio.c
@@ -185,7 +185,10 @@ init_ttyfp(void)
 #else
     ttyfp = batchmode? stderr : fopen( tty_get_ttyname (), "r+");
     if( !ttyfp ) {
-	log_error("cannot open `%s': %s\n",
+        ttyfp = stderr;  /* Use stderr as fallback for log_error.  */
+        initialized = 1; /* Make sure log_error won't try to init
+                            the tty again.  */
+	log_error("cannot open tty `%s': %s\n",
                   tty_get_ttyname (), strerror(errno) );
 	exit(2);
     }
@@ -258,7 +261,7 @@ tty_printf( const char *fmt, ... )
 
     va_start( arg_ptr, fmt ) ;
 #ifdef _WIN32
-    {   
+    {
         char *buf;
         int n;
 	DWORD nwritten;
@@ -267,7 +270,7 @@ tty_printf( const char *fmt, ... )
 	if (!buf)
           log_bug("xtryvasprintf() failed\n");
         n = strlen (buf);
-        
+
 	if (!WriteConsoleA (con.out, buf, n, &nwritten, NULL))
 	    log_fatal ("WriteConsole failed: %s", w32_strerror (0));
 	if( n != nwritten )
@@ -306,7 +309,7 @@ tty_fprintf (FILE *fp, const char *fmt, ... )
 
     va_start( arg_ptr, fmt ) ;
 #ifdef _WIN32
-    {   
+    {
         char *buf;
         int n;
 	DWORD nwritten;
@@ -315,7 +318,7 @@ tty_fprintf (FILE *fp, const char *fmt, ... )
 	if (!buf)
           log_bug ("xtryvasprintf() failed\n");
         n = strlen (buf);
-        
+
 	if (!WriteConsoleA (con.out, buf, n, &nwritten, NULL))
 	    log_fatal ("WriteConsole failed: %s", w32_strerror (0));
 	if (n != nwritten)