xz: Create separate is_tty() function.

The new is_tty() will report if a file descriptor is a terminal or not.
On POSIX systems, it is a wrapper around isatty(). However, the native
Windows implementation of isatty() will return true for all character
devices, not just terminals. So is_tty() has a special case for Windows
so it can use alternative Windows API functions to determine if a file
descriptor is a terminal.

This fixes a bug with MSVC and MinGW-w64 builds that refused to read from
or write to non-terminal character devices because xz thought it was a
terminal. For instance:

    xz foo -c > /dev/null

would fail because /dev/null was assumed to be a terminal.

src/xz/util.c

@@ -261,10 +261,31 @@ my_snprintf(char **pos, size_t *left, const char *fmt, ...)
 }
 
 
+extern bool
+is_tty(int fd)
+{
+#if defined(_WIN32) && !defined(__CYGWIN__)
+	// There is no need to check if handle == INVALID_HANDLE_VALUE
+	// because it will return false anyway when used in GetConsoleMode().
+	// The resulting HANDLE is owned by the file descriptor.
+	// The HANDLE must not be closed here.
+	intptr_t handle = _get_osfhandle(fd);
+	DWORD mode;
+
+	// GetConsoleMode() is an easy way to tell if the HANDLE is a
+	// console or not. We do not care about the value of mode since we
+	// do not plan to use any further Windows console functions.
+	return GetConsoleMode((HANDLE)handle, &mode);
+#else
+	return isatty(fd);
+#endif
+}
+
+
 extern bool
 is_tty_stdin(void)
 {
-	const bool ret = isatty(STDIN_FILENO);
+	const bool ret = is_tty(STDIN_FILENO);
 
 	if (ret)
 		message_error(_("Compressed data cannot be read from "
@@ -277,7 +298,7 @@ is_tty_stdin(void)
 extern bool
 is_tty_stdout(void)
 {
-	const bool ret = isatty(STDOUT_FILENO);
+	const bool ret = is_tty(STDOUT_FILENO);
 
 	if (ret)
 		message_error(_("Compressed data cannot be written to "

src/xz/util.h

@@ -105,6 +105,20 @@ lzma_attribute((__format__(__printf__, 3, 4)))
 extern void my_snprintf(char **pos, size_t *left, const char *fmt, ...);
 
 
+/// \brief      Test if file descriptor is a terminal
+///
+/// For POSIX systems, this is a simple wrapper around isatty(). However on
+/// Windows, isatty() returns true for all character devices, not just
+/// terminals.
+///
+/// \param      fd    File descriptor to test
+///
+/// \return     bool:
+///             - true if file descriptor is a terminal
+///             - false otherwise
+extern bool is_tty(int fd);
+
+
 /// \brief      Test if stdin is a terminal
 ///
 /// If stdin is a terminal, an error message is printed and exit status set