Make sandbox-bindgen allocate varargs buffers on the stack

Not sure why, but this fixes crashes when calling variadic functions in
the Ruby API in libretro builds when Ruby is built without `-DNDEBUG`.
Maybe the previous way of calling varargs functions was undefined
behaviour somehow.

binding-sandbox/binding-base.h

@@ -51,8 +51,11 @@
 // LLVM uses a stack alignment of 16 on WebAssembly targets
 #define WASMSTACKALIGN 16
 
+// Rounds a number up to the nearest multiple of the WebAssembly stack alignment
+#define CEIL_WASMSTACKALIGN(x) (((x) + (size_t)(WASMSTACKALIGN - 1)) & ~(size_t)(WASMSTACKALIGN - 1))
+
 // Same as `sizeof(T)`, but rounds the result up to the nearest multiple of the WebAssembly stack alignment
-#define SIZEOF_WASMSTACKALIGN(T) ((sizeof(T) + (size_t)(WASMSTACKALIGN - 1)) & ~(size_t)(WASMSTACKALIGN - 1))
+#define SIZEOF_WASMSTACKALIGN(T) CEIL_WASMSTACKALIGN(sizeof(T))
 
 namespace mkxp_sandbox {
     struct binding_base {

libretro/sandbox-bindgen.rb

@@ -511,42 +511,59 @@ File.readlines('tags', chomp: true).each do |line|
 
   coroutine_vars = []
 
+  fields = (0...args.length).filter_map do |i|
+    transformed_args.include?(i) && "wasm_ptr_t f#{i}"
+  end
+
   # If this is a varargs function, manually generate bindings for getting the varargs based on the function name
   if !args.empty? && args[-1] == '...'
     case func_name
     when 'rb_funcall'
       coroutine_initializer += <<~HEREDOC
-        f#{args.length - 1} = bind.sandbox_malloc(a#{args.length - 2} * sizeof(VALUE));
+        fp = w2c_ruby_rb_wasm_get_stack_pointer(&bind.instance());
-        if (f#{args.length - 1} == 0) throw SandboxOutOfMemoryException();
+        sp = fp - CEIL_WASMSTACKALIGN(a#{args.length - 2} * sizeof(VALUE));
+        if (sp > fp) {
+            throw SandboxOutOfMemoryException();
+        }
+        w2c_ruby_rb_wasm_set_stack_pointer(&bind.instance(), sp);
         std::va_list a;
         va_start(a, a#{args.length - 2});
         for (long i = 0; i < a#{args.length - 2}; ++i) {
-            ((VALUE *)(*bind + f#{args.length - 1}))[i] = va_arg(a, VALUE);
+            ((VALUE *)(*bind + sp))[i] = va_arg(a, VALUE);
         }
         va_end(a);
       HEREDOC
       coroutine_initializer += "\n"
-      buffers.append("f#{args.length - 1}")
+      fields.append('wasm_ptr_t fp')
+      fields.append('wasm_ptr_t sp')
+      buffers.append('fp')
+      buffers.append('sp')
     when 'rb_rescue2'
-      coroutine_vars.append('wasm_size_t n')
       coroutine_initializer += <<~HEREDOC
-        std::va_list a, b;
+        {
-        va_start(a, a#{args.length - 2});
+            std::va_list a, b;
-        va_copy(b, a);
+            va_start(a, a#{args.length - 2});
-        n = 0;
+            va_copy(b, a);
-        do ++n; while (va_arg(b, VALUE));
+            wasm_size_t n = 0;
-        va_end(b);
+            do ++n; while (va_arg(b, VALUE));
-        f#{args.length - 1} = bind.sandbox_malloc(n * sizeof(VALUE));
+            va_end(b);
-        if (f#{args.length - 1} == 0) {
+            fp = w2c_ruby_rb_wasm_get_stack_pointer(&bind.instance());
+            sp = fp - CEIL_WASMSTACKALIGN(n * sizeof(VALUE));
+            if (sp > fp) {
+                throw SandboxOutOfMemoryException();
+            }
+            w2c_ruby_rb_wasm_set_stack_pointer(&bind.instance(), sp);
+            for (wasm_size_t i = 0; i < n; ++i) {
+                ((VALUE *)(*bind + sp))[i] = va_arg(a, VALUE);
+            }
             va_end(a);
-            throw SandboxOutOfMemoryException();
-        }
-        for (wasm_size_t i = 0; i < n; ++i) {
-            ((VALUE *)(*bind + f#{args.length - 1}))[i] = va_arg(a, VALUE);
         }
       HEREDOC
       coroutine_initializer += "\n"
-      buffers.append("f#{args.length - 1}")
+      fields.append('wasm_ptr_t fp')
+      fields.append('wasm_ptr_t sp')
+      buffers.append('fp')
+      buffers.append('sp')
     else
       next
     end
@@ -554,10 +571,6 @@ File.readlines('tags', chomp: true).each do |line|
 
   handler = RET_HANDLERS[ret]
 
-  fields = (0...args.length).filter_map do |i|
-    (args[i] == '...' || transformed_args.include?(i)) && "wasm_ptr_t f#{i}"
-  end
-
   coroutine_ret = !RET_HANDLERS[ret][:keep] ? VAR_TYPE_TABLE[RET_HANDLERS[ret][:primitive]] : ret;
 
   coroutine_vars.append("#{coroutine_ret} r") if handler[:primitive] != :void
@@ -577,14 +590,16 @@ File.readlines('tags', chomp: true).each do |line|
   end
 
   coroutine_inner = <<~HEREDOC
-    #{handler[:primitive] == :void ? '' : 'r = '}w2c_#{MODULE_NAME}_#{func_name}(#{(['&bind.instance()'] + (0...args.length).map { |i| args[i] == '...' || transformed_args.include?(i) ? "f#{i}" : args[i] == 'VALUE' ? "SERIALIZE_VALUE(a#{i})" : args[i] == 'const rb_data_type_t *' ? "a#{i}.get()" : "a#{i}" }).join(', ')});
+    #{handler[:primitive] == :void ? '' : 'r = '}w2c_#{MODULE_NAME}_#{func_name}(#{(['&bind.instance()'] + (0...args.length).map { |i| args[i] == '...' ? 'sp' : transformed_args.include?(i) ? "f#{i}" : args[i] == 'VALUE' ? "SERIALIZE_VALUE(a#{i})" : args[i] == 'const rb_data_type_t *' ? "a#{i}.get()" : "a#{i}" }).join(', ')});
     if (w2c_#{MODULE_NAME}_asyncify_get_state(&bind.instance()) != 1) break;
     BOOST_ASIO_CORO_YIELD;
   HEREDOC
 
   coroutine_destructor = buffers.empty? ? '' : <<~HEREDOC
     #{func_name}::~#{func_name}() {
-    #{(0...buffers.length).map { |i| "    if (#{buffers[buffers.length - 1 - i]} != 0) bind.sandbox_free(#{buffers[buffers.length - 1 - i]});" }.join("\n")}
+    #{(0...buffers.length)
+      .filter { |i| buffers[buffers.length - 1 - i] != 'sp' }
+      .map { |i| "    if (#{buffers[buffers.length - 1 - i]} != 0) #{buffers[buffers.length - 1 - i] == 'fp' ? "w2c_ruby_rb_wasm_set_stack_pointer(&bind.instance(), #{buffers[buffers.length - 1 - i]})" : "bind.sandbox_free(#{buffers[buffers.length - 1 - i]})"};" }.join("\n")}
     }
   HEREDOC
 

src/core.cpp

@@ -145,7 +145,7 @@ static VALUE rescue(VALUE arg, VALUE exception) {
 SANDBOX_COROUTINE(main,
     void operator()() {
         BOOST_ASIO_CORO_REENTER (this) {
-            SANDBOX_AWAIT(rb_rescue2, func, 0, rescue, 0, sb()->rb_eException(), 0);
+            SANDBOX_AWAIT(rb_rescue2, func, SANDBOX_NIL, rescue, SANDBOX_NIL, sb()->rb_eException(), 0);
         }
     }
 )