Implement stackful coroutine-based executor for libretro builds

This executor has the advantage of being able to work correctly when
there are Ruby stack frames underneath C/C++ stack frames in the stack.

Still need to implement handling Ruby fibers.

.github/workflows/autobuild.yml

@@ -488,7 +488,7 @@ jobs:
           PATH="$HOMEBREW_PREFIX/opt/gpatch/libexec/gnubin:$PATH" meson setup build --buildtype release -Db_lto=true -Dretro=true -Dretro_phase1_path=retro/build/retro-phase1
           cd build
           ninja -v
-          strip libretro-mkxp-z.dylib
+          strip -x libretro-mkxp-z.dylib
           mv libretro-mkxp-z.dylib ${{ runner.temp }}/retro-phase2
 
       - uses: actions/upload-artifact@v4

meson.build

@@ -55,8 +55,8 @@ if get_option('retro') == true
         'ENABLE_LIB_ONLY': true,
     })
 
-    lzma_options = cmake.subproject_options()
-    lzma_options.add_cmake_defines({
+    liblzma_options = cmake.subproject_options()
+    liblzma_options.add_cmake_defines({
         'CMAKE_POSITION_INDEPENDENT_CODE': true,
         'BUILD_SHARED_LIBS': false,
         'ENABLE_NLS': false,
@@ -113,9 +113,19 @@ if get_option('retro') == true
         'retro-' + meson.project_name(),
         dependencies: [
             cmake.subproject('boost_asio', options: boost_options).dependency('boost_asio'),
+            cmake.subproject('boost_mp11', options: boost_options).dependency('boost_mp11'),
+            cmake.subproject('boost_describe', options: boost_options).dependency('boost_describe'),
+            cmake.subproject('boost_config', options: boost_options).dependency('boost_config'),
+            cmake.subproject('boost_assert', options: boost_options).dependency('boost_assert'),
+            cmake.subproject('boost_static_assert', options: boost_options).dependency('boost_static_assert'),
+            cmake.subproject('boost_throw_exception', options: boost_options).dependency('boost_throw_exception'),
+            cmake.subproject('boost_core', options: boost_options).dependency('boost_core'),
+            cmake.subproject('boost_container_hash', options: boost_options).dependency('boost_container_hash'),
+            cmake.subproject('boost_type_index', options: boost_options).dependency('boost_type_index'),
+            cmake.subproject('boost_any', options: boost_options).dependency('boost_any'),
             cmake.subproject('zlib', options: zlib_options).dependency('zlibstatic'),
             cmake.subproject('bzip2', options: bzip2_options).dependency('bz2_static'),
-            cmake.subproject('liblzma', options: lzma_options).dependency('liblzma'),
+            cmake.subproject('liblzma', options: liblzma_options).dependency('liblzma'),
             cmake.subproject('zstd', options: zstd_options).dependency('libzstd_static'),
             cmake.subproject('libzip', options: libzip_options).dependency('zip'),
         ],

retro/sandbox-bindgen.rb

@@ -151,11 +151,14 @@ HEADER_START = <<~HEREDOC
 
   #ifndef MKXP_SANDBOX_BINDGEN_H
   #define MKXP_SANDBOX_BINDGEN_H
-  
+
   #include <cstdint>
   #include <cstring>
   #include <memory>
+  #include <vector>
+  #include <boost/any.hpp>
   #include <boost/asio/coroutine.hpp>
+  #include <boost/asio/yield.hpp>
   #{MODULE_INCLUDE}
   #include "src/sandbox/types.h"
 
@@ -168,20 +171,61 @@ HEADER_START = <<~HEREDOC
   typedef wasm_size_t VALUE;
   typedef wasm_size_t ID;
 
-  struct SandboxBind {
-      private:
-      wasm_ptr_t next_func_ptr;
-      std::shared_ptr<struct w2c_#{MODULE_NAME}> instance;
-      wasm_ptr_t sbindgen_malloc(wasm_ptr_t);
-      wasm_ptr_t sbindgen_create_func_ptr();
+  namespace mkxp_sandbox {
+      struct bindings {
+          private:
+          wasm_ptr_t next_func_ptr;
+          std::shared_ptr<struct w2c_#{MODULE_NAME}> instance;
+          size_t depth;
+          std::vector<boost::any> stack;
+          wasm_ptr_t sbindgen_malloc(wasm_ptr_t);
+          wasm_ptr_t sbindgen_create_func_ptr();
 
-      public:
-      SandboxBind(std::shared_ptr<struct w2c_#{MODULE_NAME}>);
+          public:
+          bindings(std::shared_ptr<struct w2c_#{MODULE_NAME}>);
+
+          template <typename T> struct stack_frame {
+              friend struct bindings;
+
+              private:
+              struct bindings &bindings;
+              T &inner;
+              static inline T &init(struct bindings &bindings) {
+                  if (bindings.depth == bindings.stack.size()) {
+                      bindings.stack.push_back(T(bindings));
+                  } else if (bindings.depth > bindings.stack.size()) {
+                      throw SandboxTrapException();
+                  }
+                  try {
+                      return boost::any_cast<T &>(bindings.stack[bindings.depth++]);
+                  } catch (boost::bad_any_cast &) {
+                      throw SandboxTrapException();
+                  }
+              }
+              stack_frame(struct bindings &b) : bindings(b), inner(init(b)) {}
+
+              public:
+              ~stack_frame() {
+                  if (inner.is_complete()) {
+                      bindings.stack.pop_back();
+                  }
+                  --bindings.depth;
+              }
+              inline T &operator()() {
+                  return inner;
+              }
+          };
+
+          template <typename T> struct stack_frame<T> bind() {
+              return (struct stack_frame<T>)(*this);
+          }
 
 HEREDOC
 
 HEADER_END = <<~HEREDOC
 
+  }
+
   #endif // MKXP_SANDBOX_BINDGEN_H
 HEREDOC
 
@@ -210,7 +254,6 @@ PRELUDE = <<~HEREDOC
   // Autogenerated by sandbox-bindgen.rb. Don't manually modify this file - modify sandbox-bindgen.rb instead!
 
   #include <cstdarg>
-  #include <boost/asio/yield.hpp>
   #include "mkxp-sandbox-bindgen.h"
 
   #if WABT_BIG_ENDIAN
@@ -223,10 +266,13 @@ PRELUDE = <<~HEREDOC
   #define SERIALIZE_PTR(value) SERIALIZE_#{MEMORY64 ? '64' : '32'}(value)
 
 
-  SandboxBind::SandboxBind(std::shared_ptr<struct w2c_#{MODULE_NAME}> m) : next_func_ptr(-1), instance(m) {}
+  using namespace mkxp_sandbox;
 
 
-  wasm_ptr_t SandboxBind::sbindgen_malloc(wasm_size_t size) {
+  bindings::bindings(std::shared_ptr<struct w2c_#{MODULE_NAME}> m) : next_func_ptr(-1), instance(m), depth(0) {}
+
+
+  wasm_ptr_t bindings::sbindgen_malloc(wasm_size_t size) {
       wasm_ptr_t buf = w2c_#{MODULE_NAME}_#{MALLOC_FUNC}(instance.get(), size);
 
       // Verify that the entire allocated buffer is in valid memory
@@ -239,7 +285,7 @@ PRELUDE = <<~HEREDOC
   }
 
 
-  wasm_ptr_t SandboxBind::sbindgen_create_func_ptr() {
+  wasm_ptr_t bindings::sbindgen_create_func_ptr() {
       if (next_func_ptr == (wasm_ptr_t)-1) {
           next_func_ptr = instance->w2c_T0.size;
       }
@@ -426,21 +472,19 @@ File.readlines('tags', chomp: true).each do |line|
 
   coroutine_vars.append("#{coroutine_ret} r") if handler[:primitive] != :void
 
-  coroutine_args = ['SandboxBind &bind']
-  coroutine_args.append((0...args.length).map do |i|
+  coroutine_args = (0...args.length).map do |i|
     args[i] == '...' ? '...'
       : !ARG_HANDLERS[args[i]][:formatter].nil? ? ARG_HANDLERS[args[i]][:formatter].call("a#{i}")
       : !ARG_HANDLERS[args[i]][:keep] ? "#{VAR_TYPE_TABLE[ARG_HANDLERS[args[i]][:primitive]]} a#{i}"
       : "#{args[i]} a#{i}"
-  end)
+  end
 
-  declaration_args = ['SandboxBind &']
-  declaration_args.append((0...args.length).map do |i|
+  declaration_args = (0...args.length).map do |i|
     args[i] == '...' ? '...'
       : !ARG_HANDLERS[args[i]][:formatter].nil? ? ARG_HANDLERS[args[i]][:formatter].call('')
       : !ARG_HANDLERS[args[i]][:keep] ? "#{VAR_TYPE_TABLE[ARG_HANDLERS[args[i]][:primitive]]}"
       : "#{args[i]}"
-  end)
+  end
 
   coroutine_inner = <<~HEREDOC
     #{handler[:primitive] == :void ? '' : 'r = '}w2c_#{MODULE_NAME}_#{func_name}(#{(['bind.instance.get()'] + (0...args.length).map { |i| args[i] == '...' || transformed_args.include?(i) ? "f#{i}" : "a#{i}" }).join(', ')});
@@ -451,6 +495,7 @@ File.readlines('tags', chomp: true).each do |line|
   coroutine_finalizer = (0...buffers.length).map { |i| "w2c_#{MODULE_NAME}_#{FREE_FUNC}(bind.instance.get(), #{buffers[buffers.length - 1 - i]});" }
 
   coroutine_definition = <<~HEREDOC
+    #{func_name}::#{func_name}(bindings &bind) : bind(bind) {}
     #{coroutine_ret} #{func_name}::operator()(#{coroutine_args.join(', ')}) {#{coroutine_vars.empty? ? '' : (coroutine_vars.map { |var| "\n    #{var} = 0;" }.join + "\n")}
         reenter (this) {
     #{coroutine_initializer.empty? ? '' : (coroutine_initializer.split("\n").map { |line| "        #{line}" }.join("\n") + "\n\n")}        for (;;) {
@@ -462,8 +507,13 @@ File.readlines('tags', chomp: true).each do |line|
 
   coroutine_declaration = <<~HEREDOC
     struct #{func_name} : boost::asio::coroutine {
+        friend struct bindings;
+        friend struct bindings::stack_frame<struct #{func_name}>;
         #{coroutine_ret} operator()(#{declaration_args.join(', ')});
-    #{fields.empty? ? '' : ("    private:\n" + fields.map { |field| "    #{field};\n" }.join)}};
+        private:
+        #{func_name}(bindings &bind);
+        bindings &bind;
+    #{fields.empty? ? '' : fields.map { |field| "    #{field};\n" }.join}};
   HEREDOC
 
   func_names.append(func_name)
@@ -474,11 +524,11 @@ end
 File.open('mkxp-sandbox-bindgen.h', 'w') do |file|
   file.write(HEADER_START)
   for func_name in func_names
-    file.write("    friend struct #{func_name};\n")
+    file.write("        friend struct #{func_name};\n")
   end
-  file.write("};\n")
+  file.write("    };\n")
   for declaration in declarations
-    file.write("\n" + declaration)
+    file.write("\n" + declaration.split("\n").map { |line| "    #{line}" }.join("\n").rstrip)
   end
   file.write(HEADER_END)
 end
@@ -486,6 +536,6 @@ File.open('mkxp-sandbox-bindgen.cpp', 'w') do |file|
   file.write(PRELUDE)
   for coroutine in coroutines
     file.write("\n\n")
-    file.write(coroutine)
+    file.write(coroutine.rstrip)
   end
 end

src/core.cpp

@@ -24,24 +24,30 @@
 #include <cstdarg>
 #include <cstring>
 #include <memory>
-#include <boost/asio/coroutine.hpp>
-#include <boost/asio/yield.hpp>
-#include "core.h"
 #include "sandbox/sandbox.h"
+#include "core.h"
 
-#define AWAIT(coroutine, ...) do { coroutine(__VA_ARGS__); if (coroutine.is_complete()) break; yield; } while (1)
+#define SANDBOX_AWAIT(coroutine, ...) \
+    do { \
+        { \
+            auto frame = sandbox->bindings.bind<struct coroutine>(); \
+            frame()(__VA_ARGS__); \
+            if (frame().is_complete()) break; \
+        } \
+        yield; \
+    } while (1)
 
 using namespace mkxp_retro;
 
 static void fallback_log(enum retro_log_level level, const char *fmt, ...) {
-    va_list va;
+    std::va_list va;
     va_start(va, fmt);
-    vfprintf(stderr, fmt, va);
+    std::vfprintf(stderr, fmt, va);
     va_end(va);
 }
 
 static uint32_t *frame_buf;
-static std::unique_ptr<Sandbox> sandbox;
+static std::unique_ptr<struct mkxp_sandbox::sandbox> sandbox;
 static const char *game_path = NULL;
 
 static VALUE my_cpp_func(w2c_ruby *ruby, int32_t argc, wasm_ptr_t argv, VALUE self) {
@@ -50,24 +56,17 @@ static VALUE my_cpp_func(w2c_ruby *ruby, int32_t argc, wasm_ptr_t argv, VALUE se
 }
 
 static bool init_sandbox() {
-    struct runtime : boost::asio::coroutine {
-        struct rb_eval_string eval;
-        struct rb_define_global_function define;
-
+    struct main : boost::asio::coroutine {
         void operator()() {
             reenter (this) {
-                AWAIT(eval, sandbox->bind, "puts 'Hello, World!'");
+                SANDBOX_AWAIT(mkxp_sandbox::rb_eval_string, "puts 'Hello, World!'");
 
-                eval = rb_eval_string();
-                AWAIT(eval, sandbox->bind, "require 'zlib'; p Zlib::Deflate::deflate('hello')");
+                SANDBOX_AWAIT(mkxp_sandbox::rb_eval_string, "require 'zlib'; p Zlib::Deflate::deflate('hello')");
 
-                AWAIT(define, sandbox->bind, "my_cpp_func", (VALUE (*)(void *, ANYARGS))my_cpp_func, -1);
+                SANDBOX_AWAIT(mkxp_sandbox::rb_define_global_function, "my_cpp_func", (VALUE (*)(void *, ANYARGS))my_cpp_func, -1);
+                SANDBOX_AWAIT(mkxp_sandbox::rb_eval_string, "my_cpp_func(1, nil, 3, 'this is a string', :symbol, 2)");
 
-                eval = rb_eval_string();
-                AWAIT(eval, sandbox->bind, "my_cpp_func(1, nil, 3, 'this is a string', :symbol, 2)");
-
-                eval = rb_eval_string();
-                AWAIT(eval, sandbox->bind, "p Dir.glob '/mkxp-retro-game/*'");
+                SANDBOX_AWAIT(mkxp_sandbox::rb_eval_string, "p Dir.glob '/mkxp-retro-game/*'");
             }
         }
     };
@@ -75,12 +74,8 @@ static bool init_sandbox() {
     sandbox.reset();
 
     try {
-        sandbox.reset(new Sandbox(game_path));
-
-        struct runtime runtime;
-
-        // TODO: Replace this loop with a stackful executor, otherwise you won't be able to call into the Ruby API from inside of a C/C++ function that is itself called from inside of Ruby.
-        do runtime(); while (w2c_ruby_mkxp_sandbox_yield(&sandbox->module_instance()));
+        sandbox.reset(new struct mkxp_sandbox::sandbox(game_path));
+        sandbox->run<struct main>();
     } catch (SandboxException) {
         log_printf(RETRO_LOG_ERROR, "Failed to initialize Ruby\n");
         sandbox.reset();

src/sandbox/sandbox.cpp

@@ -38,12 +38,14 @@
 #define WASM_MEM(address) ((void *)&ruby->w2c_memory.data[address])
 #define AWAIT(statement) do statement; while (w2c_ruby_mkxp_sandbox_yield(RB))
 
+using namespace mkxp_sandbox;
+
 // This function is imported by wasm-rt-impl.c from wasm2c
 extern "C" void mkxp_sandbox_trap_handler(wasm_rt_trap_t code) {
     throw SandboxTrapException();
 }
 
-usize Sandbox::sandbox_malloc(usize size) {
+usize sandbox::sandbox_malloc(usize size) {
     usize buf = w2c_ruby_mkxp_sandbox_malloc(RB, size);
 
     // Verify that the returned pointer is non-null and the entire allocated buffer is in valid memory
@@ -55,11 +57,11 @@ usize Sandbox::sandbox_malloc(usize size) {
     return buf;
 }
 
-void Sandbox::sandbox_free(usize ptr) {
+void sandbox::sandbox_free(usize ptr) {
     w2c_ruby_mkxp_sandbox_free(RB, ptr);
 }
 
-Sandbox::Sandbox(const char *game_path) : ruby(new struct w2c_ruby), wasi(new wasi_t(ruby, game_path)), bind(ruby) {
+sandbox::sandbox(const char *game_path) : ruby(new struct w2c_ruby), wasi(new wasi_t(ruby, game_path)), bindings(ruby) {
     try {
         // Initialize the sandbox
         wasm_rt_init();
@@ -138,14 +140,10 @@ Sandbox::Sandbox(const char *game_path) : ruby(new struct w2c_ruby), wasi(new wa
     }
 }
 
-Sandbox::~Sandbox() {
+sandbox::~sandbox() {
     try {
         w2c_ruby_mkxp_sandbox_deinit(RB);
     } catch (SandboxTrapException) {}
     wasm2c_ruby_free(RB);
     wasm_rt_free();
 }
-
-w2c_ruby &Sandbox::module_instance() {
-    return *ruby;
-}

src/sandbox/sandbox.h

@@ -26,19 +26,29 @@
 #include <mkxp-sandbox-bindgen.h>
 #include "types.h"
 
-struct Sandbox {
-    private:
-    std::shared_ptr<struct w2c_ruby> ruby;
-    std::unique_ptr<struct w2c_wasi__snapshot__preview1> wasi;
+namespace mkxp_sandbox {
+    struct sandbox {
+        private:
+        std::shared_ptr<struct w2c_ruby> ruby;
+        std::unique_ptr<struct w2c_wasi__snapshot__preview1> wasi;
+        usize sandbox_malloc(usize size);
+        void sandbox_free(usize ptr);
 
-    usize sandbox_malloc(usize size);
-    void sandbox_free(usize ptr);
+        public:
+        struct mkxp_sandbox::bindings bindings;
+        sandbox(const char *game_path);
+        ~sandbox();
 
-    public:
-    SandboxBind bind;
-    Sandbox(const char *game_path);
-    ~Sandbox();
-    struct w2c_ruby &module_instance();
-};
+        // TODO: handle Ruby fibers properly instead of crashing whenever Ruby switches to a different fiber than the main one
+        template <typename T> inline void run() {
+            T coroutine = T();
+            do {
+                coroutine();
+                w2c_ruby_mkxp_sandbox_yield(ruby.get());
+            } while (!coroutine.is_complete());
+
+        }
+    };
+}
 
 #endif // MKXPZ_SANDBOX_H

subprojects/boost_any.wrap

@@ -0,0 +1,4 @@
+[wrap-git]
+url = https://github.com/boostorg/any
+revision = boost-1.87.0
+depth = 1

subprojects/boost_assert.wrap

@@ -0,0 +1,4 @@
+[wrap-git]
+url = https://github.com/boostorg/assert
+revision = boost-1.87.0
+depth = 1

subprojects/boost_config.wrap

@@ -0,0 +1,4 @@
+[wrap-git]
+url = https://github.com/boostorg/config
+revision = boost-1.87.0
+depth = 1

subprojects/boost_container_hash.wrap

@@ -0,0 +1,4 @@
+[wrap-git]
+url = https://github.com/boostorg/container_hash
+revision = boost-1.87.0
+depth = 1

subprojects/boost_core.wrap

@@ -0,0 +1,4 @@
+[wrap-git]
+url = https://github.com/boostorg/core
+revision = boost-1.87.0
+depth = 1

subprojects/boost_describe.wrap

@@ -0,0 +1,4 @@
+[wrap-git]
+url = https://github.com/boostorg/describe
+revision = boost-1.87.0
+depth = 1

subprojects/boost_mp11.wrap

@@ -0,0 +1,4 @@
+[wrap-git]
+url = https://github.com/boostorg/mp11
+revision = boost-1.87.0
+depth = 1

subprojects/boost_static_assert.wrap

@@ -0,0 +1,4 @@
+[wrap-git]
+url = https://github.com/boostorg/static_assert
+revision = boost-1.87.0
+depth = 1

subprojects/boost_throw_exception.wrap

@@ -0,0 +1,4 @@
+[wrap-git]
+url = https://github.com/boostorg/throw_exception
+revision = boost-1.87.0
+depth = 1

subprojects/boost_type_index.wrap

@@ -0,0 +1,4 @@
+[wrap-git]
+url = https://github.com/boostorg/type_index
+revision = boost-1.87.0
+depth = 1