From 8751a2d330a4fdb764dcfd3d97c4b057375f85ec Mon Sep 17 00:00:00 2001 From: Connor McLaughlin Date: Tue, 3 May 2022 14:06:04 +1000 Subject: [PATCH] Qt: Make win32 console less garbage --- pcsx2-qt/QtHost.cpp | 162 +++++++++++++++++++++++++++++++------------- pcsx2-qt/QtHost.h | 3 + 2 files changed, 119 insertions(+), 46 deletions(-) diff --git a/pcsx2-qt/QtHost.cpp b/pcsx2-qt/QtHost.cpp index f1cdeaf426..23c2815aa4 100644 --- a/pcsx2-qt/QtHost.cpp +++ b/pcsx2-qt/QtHost.cpp @@ -486,7 +486,11 @@ void QtHost::HookSignals() static bool s_debugger_attached = false; static bool s_console_handle_set = false; -static HANDLE s_console_handle = nullptr; +static bool s_console_allocated = false; +static HANDLE s_console_handle = INVALID_HANDLE_VALUE; +static HANDLE s_old_console_stdin = NULL; +static HANDLE s_old_console_stdout = NULL; +static HANDLE s_old_console_stderr = NULL; static void __concall ConsoleWinQt_SetTitle(const wxString& title) { @@ -498,31 +502,33 @@ static void __concall ConsoleWinQt_DoSetColor(ConsoleColors color) if (!s_console_handle) return; - static constexpr WORD colors[ConsoleColors_Count] = { - FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_BLUE, // default - FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_BLUE, // black - FOREGROUND_GREEN, // green - FOREGROUND_RED, // red - FOREGROUND_BLUE, // blue - FOREGROUND_RED | FOREGROUND_BLUE, // magenta - FOREGROUND_RED | FOREGROUND_GREEN, // orange - FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_BLUE, // gray - FOREGROUND_BLUE | FOREGROUND_INTENSITY, // cyan - FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_INTENSITY, // yellow - FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_BLUE | FOREGROUND_INTENSITY, // white - FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_BLUE, // strong black - FOREGROUND_RED | FOREGROUND_INTENSITY, // strong red - FOREGROUND_GREEN | FOREGROUND_INTENSITY, // strong green - FOREGROUND_BLUE | FOREGROUND_INTENSITY, // strong blue - FOREGROUND_RED | FOREGROUND_BLUE | FOREGROUND_INTENSITY, // strong magenta - FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_INTENSITY, // strong orange - FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_BLUE, // strong gray - FOREGROUND_BLUE | FOREGROUND_INTENSITY, // strong cyan - FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_INTENSITY, // strong yellow - FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_BLUE | FOREGROUND_INTENSITY, // strong white + static constexpr wchar_t colors[][ConsoleColors_Count] = { + L"\033[0m", // default + L"\033[30m\033[1m", // black + L"\033[32m", // green + L"\033[31m", // red + L"\033[34m", // blue + L"\033[35m", // magenta + L"\033[35m", // orange (FIXME) + L"\033[37m", // gray + L"\033[36m", // cyan + L"\033[33m", // yellow + L"\033[37m", // white + L"\033[30m\033[1m", // strong black + L"\033[31m\033[1m", // strong red + L"\033[32m\033[1m", // strong green + L"\033[34m\033[1m", // strong blue + L"\033[35m\033[1m", // strong magenta + L"\033[35m\033[1m", // strong orange (FIXME) + L"\033[37m\033[1m", // strong gray + L"\033[36m\033[1m", // strong cyan + L"\033[33m\033[1m", // strong yellow + L"\033[37m\033[1m", // strong white }; - SetConsoleTextAttribute(s_console_handle, colors[static_cast(color)]); + const wchar_t* colortext = colors[static_cast(color)]; + DWORD written; + WriteConsoleW(s_console_handle, colortext, std::wcslen(colortext), &written, nullptr); } static void __concall ConsoleWinQt_Newline() @@ -586,41 +592,61 @@ static BOOL WINAPI ConsoleCtrlHandler(DWORD dwCtrlType) return TRUE; } -#endif - -void QtHost::UpdateLogging() +static bool EnableVirtualTerminalProcessing(HANDLE hConsole) { - // TODO: Make this an actual option. - const bool system_console_enabled = QtHost::GetBaseBoolSettingValue("Logging", "EnableSystemConsole", false); + if (hConsole == INVALID_HANDLE_VALUE) + return false; - const bool any_logging_sinks = system_console_enabled; - DevConWriterEnabled = any_logging_sinks && QtHost::GetBaseBoolSettingValue("Logging", "EnableVerbose", false); - SysConsole.eeConsole.Enabled = any_logging_sinks && QtHost::GetBaseBoolSettingValue("Logging", "EnableEEConsole", true); - SysConsole.iopConsole.Enabled = any_logging_sinks && QtHost::GetBaseBoolSettingValue("Logging", "EnableIOPConsole", true); + DWORD old_mode; + if (!GetConsoleMode(hConsole, &old_mode)) + return false; -#ifdef _WIN32 - if (system_console_enabled) + // already enabled? + if (old_mode & ENABLE_VIRTUAL_TERMINAL_PROCESSING) + return true; + + return SetConsoleMode(hConsole, old_mode | ENABLE_VIRTUAL_TERMINAL_PROCESSING); +} + +static void SetSystemConsoleEnabled(bool enabled) +{ + if (enabled) { s_debugger_attached = IsDebuggerPresent(); if (!s_console_handle_set) { + s_old_console_stdin = GetStdHandle(STD_INPUT_HANDLE); + s_old_console_stdout = GetStdHandle(STD_OUTPUT_HANDLE); + s_old_console_stderr = GetStdHandle(STD_ERROR_HANDLE); + bool handle_valid = (GetConsoleWindow() != NULL); if (!handle_valid) { - AllocConsole(); - handle_valid = (freopen("CONIN$", "r", stdin) != nullptr); - handle_valid = (freopen("CONOUT$", "w", stdout) != nullptr); - handle_valid = (freopen("CONOUT$", "w", stderr) != nullptr); + s_console_allocated = AttachConsole(ATTACH_PARENT_PROCESS) || AllocConsole(); + handle_valid = (GetConsoleWindow() != NULL); } if (handle_valid) { s_console_handle = GetStdHandle(STD_OUTPUT_HANDLE); - s_console_handle_set = true; + if (s_console_handle != INVALID_HANDLE_VALUE) + { + s_console_handle_set = true; + + // This gets us unix-style coloured output. + EnableVirtualTerminalProcessing(GetStdHandle(STD_OUTPUT_HANDLE)); + EnableVirtualTerminalProcessing(GetStdHandle(STD_ERROR_HANDLE)); + + // Redirect stdout/stderr. + std::FILE* fp; + freopen_s(&fp, "CONIN$", "r", stdin); + freopen_s(&fp, "CONOUT$", "w", stdout); + freopen_s(&fp, "CONOUT$", "w", stderr); + } } } - if (!s_console_handle && !s_debugger_attached) + if (!s_console_handle_set && !s_debugger_attached) { Console_SetActiveHandler(ConsoleWriter_Null); SetConsoleCtrlHandler(ConsoleCtrlHandler, FALSE); @@ -635,17 +661,61 @@ void QtHost::UpdateLogging() { Console_SetActiveHandler(ConsoleWriter_Null); SetConsoleCtrlHandler(ConsoleCtrlHandler, FALSE); + + if (s_console_handle_set) + { + s_console_handle_set = false; + + // redirect stdout/stderr back to null. + std::FILE* fp; + freopen_s(&fp, "NUL:", "w", stderr); + freopen_s(&fp, "NUL:", "w", stdout); + freopen_s(&fp, "NUL:", "w", stdin); + + // release console and restore state + SetStdHandle(STD_INPUT_HANDLE, s_old_console_stdin); + SetStdHandle(STD_OUTPUT_HANDLE, s_old_console_stdout); + SetStdHandle(STD_ERROR_HANDLE, s_old_console_stderr); + s_old_console_stdin = NULL; + s_old_console_stdout = NULL; + s_old_console_stderr = NULL; + if (s_console_allocated) + { + s_console_allocated = false; + FreeConsole(); + } + } } +} + #else - if (system_console_enabled) - { + +// Unix doesn't need any special handling for console. +static void SetSystemConsoleEnabled(bool enabled) +{ + if (enabled) Console_SetActiveHandler(ConsoleWriter_Stdout); - } else - { Console_SetActiveHandler(ConsoleWriter_Null); - } +} + #endif + +void QtHost::InitializeEarlyConsole() +{ + SetSystemConsoleEnabled(true); +} + +void QtHost::UpdateLogging() +{ + const bool system_console_enabled = QtHost::GetBaseBoolSettingValue("Logging", "EnableSystemConsole", false); + + const bool any_logging_sinks = system_console_enabled; + DevConWriterEnabled = any_logging_sinks && QtHost::GetBaseBoolSettingValue("Logging", "EnableVerbose", false); + SysConsole.eeConsole.Enabled = any_logging_sinks && QtHost::GetBaseBoolSettingValue("Logging", "EnableEEConsole", true); + SysConsole.iopConsole.Enabled = any_logging_sinks && QtHost::GetBaseBoolSettingValue("Logging", "EnableIOPConsole", true); + + SetSystemConsoleEnabled(system_console_enabled); } #include diff --git a/pcsx2-qt/QtHost.h b/pcsx2-qt/QtHost.h index a84269f612..5688eaf755 100644 --- a/pcsx2-qt/QtHost.h +++ b/pcsx2-qt/QtHost.h @@ -42,6 +42,9 @@ namespace QtHost void UpdateFolders(); void UpdateLogging(); + /// Initializes early console logging (for printing command line arguments). + void InitializeEarlyConsole(); + /// Executes a function on the UI thread. void RunOnUIThread(const std::function& func, bool block = false);