adb: create unix_isatty() function.
Our Windows code has several different ways it checks whether an FD is
a console or not, some of which aren't exactly correct as they may
treat character devices (e.g. NUL) as consoles.
This CL disallows using the builtin isatty() function and provides
unix_isatty() instead which correctly checks these corner cases.
Change-Id: I6d551c745dae691c7eb3446b585265d62c1e62fa
diff --git a/sysdeps_win32.cpp b/sysdeps_win32.cpp
index 14d1375..d2e6cdb 100644
--- a/sysdeps_win32.cpp
+++ b/sysdeps_win32.cpp
@@ -2499,10 +2499,52 @@
//
// Code organization:
//
+// * _get_console_handle() and unix_isatty() provide console information.
// * stdin_raw_init() and stdin_raw_restore() reconfigure the console.
// * unix_read() detects console windows (as opposed to pipes, files, etc.).
// * _console_read() is the main code of the emulation.
+// Returns a console HANDLE if |fd| is a console, otherwise returns nullptr.
+// If a valid HANDLE is returned and |mode| is not null, |mode| is also filled
+// with the console mode. Requires GENERIC_READ access to the underlying HANDLE.
+static HANDLE _get_console_handle(int fd, DWORD* mode=nullptr) {
+ // First check isatty(); this is very fast and eliminates most non-console
+ // FDs, but returns 1 for both consoles and character devices like NUL.
+#pragma push_macro("isatty")
+#undef isatty
+ if (!isatty(fd)) {
+ return nullptr;
+ }
+#pragma pop_macro("isatty")
+
+ // To differentiate between character devices and consoles we need to get
+ // the underlying HANDLE and use GetConsoleMode(), which is what requires
+ // GENERIC_READ permissions.
+ const intptr_t intptr_handle = _get_osfhandle(fd);
+ if (intptr_handle == -1) {
+ return nullptr;
+ }
+ const HANDLE handle = reinterpret_cast<const HANDLE>(intptr_handle);
+ DWORD temp_mode = 0;
+ if (!GetConsoleMode(handle, mode ? mode : &temp_mode)) {
+ return nullptr;
+ }
+
+ return handle;
+}
+
+// Returns a console handle if |stream| is a console, otherwise returns nullptr.
+static HANDLE _get_console_handle(FILE* const stream) {
+ const int fd = fileno(stream);
+ if (fd < 0) {
+ return nullptr;
+ }
+ return _get_console_handle(fd);
+}
+
+int unix_isatty(int fd) {
+ return _get_console_handle(fd) ? 1 : 0;
+}
// Read an input record from the console; one that should be processed.
static bool _get_interesting_input_record_uncached(const HANDLE console,
@@ -3302,20 +3344,7 @@
void stdin_raw_init(const int fd) {
if (STDIN_FILENO == fd) {
- const HANDLE in = GetStdHandle(STD_INPUT_HANDLE);
- if ((in == INVALID_HANDLE_VALUE) || (in == NULL)) {
- return;
- }
-
- if (GetFileType(in) != FILE_TYPE_CHAR) {
- // stdin might be a file or pipe.
- return;
- }
-
- if (!GetConsoleMode(in, &_old_console_mode)) {
- // If GetConsoleMode() fails, stdin is probably is not a console.
- return;
- }
+ const HANDLE in = _get_console_handle(fd, &_old_console_mode);
// Disable ENABLE_PROCESSED_INPUT so that Ctrl-C is read instead of
// calling the process Ctrl-C routine (configured by
@@ -3366,11 +3395,8 @@
} else {
// On older versions of Windows (definitely 7, definitely not 10),
// ReadConsole() with a size >= 31367 fails, so if |fd| is a console
- // we need to limit the read size. This may also catch devices like NUL,
- // but that is OK as we just want to avoid capping pipes and files which
- // don't need size limiting. This isatty() test is very simple and quick
- // and doesn't call the OS.
- if (isatty(fd) && len > 4096) {
+ // we need to limit the read size.
+ if (len > 4096 && unix_isatty(fd)) {
len = 4096;
}
// Just call into C Runtime which can read from pipes/files and which
@@ -3725,40 +3751,6 @@
return _wchmod(widen(path).c_str(), mode);
}
-// Internal function to get a Win32 console HANDLE from a C Runtime FILE*.
-static HANDLE _get_console_handle(FILE* const stream) {
- // Get a C Runtime file descriptor number from the FILE* structure.
- const int fd = fileno(stream);
- if (fd < 0) {
- return NULL;
- }
-
- // If it is not a "character device", it is probably a file and not a
- // console. Do this check early because it is probably cheap. Still do more
- // checks after this since there are devices that pass this test, but are
- // not a console, such as NUL, the Windows /dev/null equivalent (I think).
- if (!isatty(fd)) {
- return NULL;
- }
-
- // Given a C Runtime file descriptor number, get the underlying OS
- // file handle.
- const intptr_t osfh = _get_osfhandle(fd);
- if (osfh == -1) {
- return NULL;
- }
-
- const HANDLE h = reinterpret_cast<const HANDLE>(osfh);
-
- DWORD old_mode = 0;
- if (!GetConsoleMode(h, &old_mode)) {
- return NULL;
- }
-
- // If GetConsoleMode() was successful, assume this is a console.
- return h;
-}
-
// Internal helper function to write UTF-8 bytes to a console. Returns -1
// on error.
static int _console_write_utf8(const char* buf, size_t size, FILE* stream,