From 2e5be7b2d0178fa466ec55648f94c59664acf520 Mon Sep 17 00:00:00 2001
From: Joni Savolainen <joni@savolainen.io>
Date: Mon, 15 Mar 2021 11:22:05 +0200
Subject: [PATCH] Create a debug console on Windows

---
 binding/binding-mri.cpp     | 50 +++++++++++++++++++++++++++++++++++++
 src/main.cpp                | 15 +++++++++++
 src/util/win-consoleutils.h | 34 +++++++++++++++++++++++++
 3 files changed, 99 insertions(+)
 create mode 100644 src/util/win-consoleutils.h

diff --git a/binding/binding-mri.cpp b/binding/binding-mri.cpp
index a98cca4f..50f00ab5 100644
--- a/binding/binding-mri.cpp
+++ b/binding/binding-mri.cpp
@@ -47,6 +47,10 @@ extern "C" {
 #endif
 }
 
+#ifdef __WINDOWS__
+#include <fcntl.h>
+#endif
+
 #include <assert.h>
 #include <string>
 #include <zlib.h>
@@ -70,6 +74,7 @@ extern const char module_rpg3[];
 static void mriBindingExecute();
 static void mriBindingTerminate();
 static void mriBindingReset();
+static void configureWindowsStreams();
 
 ScriptBinding scriptBindingImpl = {mriBindingExecute, mriBindingTerminate,
     mriBindingReset};
@@ -219,6 +224,10 @@ static void mriBindingInit() {
         rb_gv_set("TEST", debug);
     
     rb_gv_set("BTEST", rb_bool_new(shState->config().editor.battleTest));
+
+    // Set $stdout and its ilk accordingly on Windows
+    if (shState->config().editor.debug)
+        configureWindowsStreams();
     
     // Load zlib, if it's present. Requires --with-static-linked-ext or zlib.so.
     // It's okay if it fails, normally it wouldn't be defined anyway.
@@ -746,6 +755,41 @@ static void runRMXPScripts(BacktraceData &btData) {
 }
 #endif
 
+// Attempts to set $stdout and $stdin accordingly on Windows. Only
+// called when debug mode is on, since that's when the console
+// should be active.
+static void configureWindowsStreams() {
+#ifdef __WINDOWS__
+    #define HANDLE_VALID(handle) handle && handle != INVALID_HANDLE_VALUE
+
+    const HANDLE outputHandle = GetStdHandle(STD_OUTPUT_HANDLE);
+
+    // Configure $stdout
+    if (HANDLE_VALID(outputHandle)) {
+        const int stdoutFD = _open_osfhandle((intptr_t)outputHandle, _O_TEXT);
+
+        VALUE winStdout = rb_funcall(rb_cIO, rb_intern("new"), 2,
+            INT2NUM(stdoutFD), rb_str_new_cstr("w"));
+
+        rb_gv_set("stdout", winStdout);
+    }
+
+    const HANDLE inputHandle = GetStdHandle(STD_INPUT_HANDLE);
+
+    // Configure $stdin
+    if (HANDLE_VALID(inputHandle)) {
+        const int stdinFD = _open_osfhandle((intptr_t)inputHandle, _O_TEXT);
+
+        VALUE winStdin = rb_funcall(rb_cIO, rb_intern("new"), 2,
+            INT2NUM(stdinFD), rb_str_new_cstr("r"));
+
+        rb_gv_set("stdin", winStdin);
+    }
+
+    #undef HANDLE_VALID
+#endif // #ifdef __WINDOWS__
+}
+
 static void showExc(VALUE exc, const BacktraceData &btData) {
     VALUE bt = rb_funcall2(exc, rb_intern("backtrace"), 0, NULL);
     VALUE msg = rb_funcall2(exc, rb_intern("message"), 0, NULL);
@@ -821,9 +865,15 @@ static void mriBindingExecute() {
     /* Normally only a ruby executable would do a sysinit,
      * but not doing it will lead to crashes due to closed
      * stdio streams on some platforms (eg. Windows) */
+#ifdef __WINDOWS__
+    if (!conf.editor.debug) {
+#endif
     int argc = 0;
     char **argv = 0;
     ruby_sysinit(&argc, &argv);
+#ifdef __WINDOWS__
+    }
+#endif
     
     RUBY_INIT_STACK;
     ruby_init();
diff --git a/src/main.cpp b/src/main.cpp
index c47dcd12..98974db2 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -48,6 +48,7 @@
 #if defined(__WINDOWS__)
 #include "resource.h"
 #include <Winsock2.h>
+#include "util/win-consoleutils.h"
 #endif
 
 #ifdef MKXPZ_STEAM
@@ -282,6 +283,20 @@ int main(int argc, char *argv[]) {
       showInitError(
           std::string(buf)); // Not an error worth ending the program over
     }
+
+    // Create a debug console in debug mode
+    if (conf.editor.debug) {
+      HANDLE winConsoleHandle;
+
+      if (setupWindowsConsole(winConsoleHandle)) {
+        reopenWindowsStreams();
+      } else {
+        char buf[200];
+        snprintf(buf, sizeof(buf), "Error allocating console: %lu",
+                GetLastError());
+        showInitError(std::string(buf));
+      }
+    }
 #endif
 
     SDL_Window *win;
diff --git a/src/util/win-consoleutils.h b/src/util/win-consoleutils.h
new file mode 100644
index 00000000..365fdd56
--- /dev/null
+++ b/src/util/win-consoleutils.h
@@ -0,0 +1,34 @@
+#ifndef WIN_CONSOLEUTIL_H
+#define WIN_CONSOLEUTIL_H
+
+#include <iostream>
+#include "windows.h"
+
+// Attempts to allocate a console and fetch the output handle.
+// Returns whether the operation was successful.
+// Extended error information can be received via GetLastError().
+bool setupWindowsConsole(HANDLE &handle)
+{
+    if (!AllocConsole())
+        return false;
+        
+    handle = GetStdHandle(STD_OUTPUT_HANDLE);
+
+    return (handle != NULL && handle != INVALID_HANDLE_VALUE);
+}
+
+// Reopens the file streams. This should be done after successfully
+// setting up the console.
+void reopenWindowsStreams()
+{
+    FILE* fDummy;
+    freopen_s(&fDummy, "CONOUT$", "w", stdout);
+    freopen_s(&fDummy, "CONOUT$", "w", stderr);
+    freopen_s(&fDummy, "CONIN$", "r", stdin);
+    std::cout.clear();
+    std::clog.clear();
+    std::cerr.clear();
+    std::cin.clear();
+}
+
+#endif
\ No newline at end of file