Implement loading scripts from Scripts.rxdata for libretro

binding-sandbox/binding-sandbox.h

@@ -0,0 +1,247 @@
+/*
+** binding-sandbox.h
+**
+** This file is part of mkxp.
+**
+** Copyright (C) 2013 - 2021 Amaryllis Kulla <ancurio@mapleshrine.eu>
+**
+** mkxp is free software: you can redistribute it and/or modify
+** it under the terms of the GNU General Public License as published by
+** the Free Software Foundation, either version 2 of the License, or
+** (at your option) any later version.
+**
+** mkxp is distributed in the hope that it will be useful,
+** but WITHOUT ANY WARRANTY; without even the implied warranty of
+** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+** GNU General Public License for more details.
+**
+** You should have received a copy of the GNU General Public License
+** along with mkxp.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef MKXPZ_BINDING_SANDBOX_H
+#define MKXPZ_BINDING_SANDBOX_H
+
+#include <zlib.h>
+#include "binding-sandbox/core.h"
+#include "sandbox.h"
+
+namespace mkxp_sandbox {
+    // Gets the length of a Ruby object.
+    struct length : boost::asio::coroutine {
+        inline length(struct mkxp_sandbox::bindings &bind) {}
+
+        ID id;
+        VALUE length_value;
+        wasm_size_t result;
+
+        wasm_size_t operator()(VALUE ary) {
+            reenter (this) {
+                SANDBOX_AWAIT_AND_SET(id, mkxp_sandbox::rb_intern, "length");
+                SANDBOX_AWAIT_AND_SET(length_value, mkxp_sandbox::rb_funcall, ary, id, 0);
+                SANDBOX_AWAIT_AND_SET(result, mkxp_sandbox::rb_num2ulong, length_value);
+            }
+
+            return result;
+        }
+    };
+
+    // Prints the backtrace of a Ruby exception to the log.
+    struct log_backtrace : boost::asio::coroutine {
+        inline log_backtrace(struct mkxp_sandbox::bindings &bind) {}
+
+        ID id;
+        VALUE backtrace;
+        wasm_ptr_t backtrace_str;
+
+        void operator()(VALUE exception) {
+            reenter (this) {
+                SANDBOX_AWAIT(mkxp_sandbox::rb_p, exception);
+                SANDBOX_AWAIT_AND_SET(id, mkxp_sandbox::rb_intern, "backtrace");
+                SANDBOX_AWAIT_AND_SET(backtrace, mkxp_sandbox::rb_funcall, exception, id, 0);
+                SANDBOX_AWAIT_AND_SET(id, mkxp_sandbox::rb_intern, "join");
+                SANDBOX_AWAIT_AND_SET(backtrace_str, mkxp_sandbox::rb_str_new_cstr, "\n\t");
+                SANDBOX_AWAIT_AND_SET(backtrace, mkxp_sandbox::rb_funcall, backtrace, id, 1, backtrace_str);
+                SANDBOX_AWAIT_AND_SET(backtrace_str, mkxp_sandbox::rb_string_value_cstr, &backtrace);
+                mkxp_retro::log_printf(RETRO_LOG_ERROR, "%s\n", *mkxp_sandbox::sandbox->bindings + backtrace_str);
+            }
+        }
+    };
+
+    // Evaluates a script, returning the exception if it encountered an exception or `SANDBOX_UNDEF` otherwise.
+    struct eval_script : boost::asio::coroutine {
+        private:
+        static VALUE func(void *_, VALUE arg) {
+            struct coro : boost::asio::coroutine {
+                inline coro(struct mkxp_sandbox::bindings &bind) {}
+
+                VALUE string;
+                VALUE filename;
+                ID id;
+
+                void operator()(VALUE arg) {
+                    reenter (this) {
+                        SANDBOX_AWAIT_AND_SET(string, mkxp_sandbox::rb_ary_entry, arg, 0);
+                        SANDBOX_AWAIT_AND_SET(filename, mkxp_sandbox::rb_ary_entry, arg, 1);
+                        SANDBOX_AWAIT_AND_SET(id, mkxp_sandbox::rb_intern, "eval");
+                        SANDBOX_AWAIT(mkxp_sandbox::rb_funcall, SANDBOX_NIL, id, 3, string, SANDBOX_NIL, filename);
+                    }
+                }
+            };
+
+            mkxp_sandbox::sandbox->bindings.bind<struct coro>()()(arg);
+            return SANDBOX_UNDEF;
+        }
+
+        static VALUE rescue(void *_, VALUE arg, VALUE exception) {
+            struct coro : boost::asio::coroutine {
+                inline coro(struct mkxp_sandbox::bindings &bind) {}
+
+                VALUE operator()(VALUE exception) {
+                    return exception;
+                }
+            };
+
+            return mkxp_sandbox::sandbox->bindings.bind<struct coro>()()(exception);
+        }
+
+        public:
+        inline eval_script(struct mkxp_sandbox::bindings &bind) {}
+
+        VALUE value;
+
+        VALUE operator()(VALUE string, VALUE filename) {
+            reenter (this) {
+                SANDBOX_AWAIT_AND_SET(value, mkxp_sandbox::rb_class_new_instance, 0, NULL, mkxp_sandbox::sandbox->bindings.rb_cArray());
+                SANDBOX_AWAIT(mkxp_sandbox::rb_ary_push, value, string);
+                SANDBOX_AWAIT(mkxp_sandbox::rb_ary_push, value, filename);
+                SANDBOX_AWAIT_AND_SET(value, mkxp_sandbox::rb_rescue, func, value, rescue, SANDBOX_NIL);
+            }
+
+            return value;
+        }
+
+    };
+
+    // Runs the game scripts.
+    struct run_rmxp_scripts : boost::asio::coroutine {
+        inline run_rmxp_scripts(struct mkxp_sandbox::bindings &bind) {}
+
+        VALUE value;
+        VALUE scripts;
+        wasm_size_t script_count;
+        unsigned char *decode_buffer;
+        unsigned long decode_buffer_len;
+        wasm_size_t i;
+        VALUE script;
+        wasm_ptr_t script_name;
+        wasm_ptr_t script_string;
+        wasm_size_t script_string_len;
+        wasm_size_t len;
+        VALUE script_string_value;
+        VALUE script_filename_value;
+
+        void operator()() {
+            reenter (this) {
+                decode_buffer = NULL;
+
+                // Unmarshal the script array
+                SANDBOX_AWAIT_AND_SET(value, mkxp_sandbox::rb_file_open, "/mkxp-retro-game/Data/Scripts.rxdata", "rb"); // TODO: handle VX/VXA games
+                SANDBOX_AWAIT_AND_SET(scripts, mkxp_sandbox::rb_marshal_load, value);
+                SANDBOX_AWAIT(mkxp_sandbox::rb_io_close, value);
+
+                // Assign it to the "$RGSS_SCRIPTS" global variable
+                SANDBOX_AWAIT(mkxp_sandbox::rb_gv_set, "$RGSS_SCRIPTS", scripts);
+
+                SANDBOX_AWAIT_AND_SET(script_count, mkxp_sandbox::length, scripts);
+                decode_buffer_len = 0x1000;
+                if ((decode_buffer = (unsigned char *)std::malloc(decode_buffer_len)) == NULL) {
+                    throw SandboxOutOfMemoryException();
+                }
+
+                for (i = 0; i < script_count; ++i) {
+                    // Skip this script object if it's not an array
+                    SANDBOX_AWAIT_AND_SET(script, mkxp_sandbox::rb_ary_entry, scripts, i);
+                    SANDBOX_AWAIT_AND_SET(value, mkxp_sandbox::rb_obj_is_kind_of, script, mkxp_sandbox::sandbox->bindings.rb_cArray());
+                    if (value != SANDBOX_TRUE) {
+                        continue;
+                    }
+
+                    // Get the name of the script and the zlib-compressed script contents
+                    SANDBOX_AWAIT_AND_SET(value, mkxp_sandbox::rb_ary_entry, script, 1);
+                    SANDBOX_AWAIT_AND_SET(script_name, mkxp_sandbox::rb_string_value_cstr, &value);
+                    SANDBOX_AWAIT_AND_SET(value, mkxp_sandbox::rb_ary_entry, script, 2);
+                    SANDBOX_AWAIT_AND_SET(script_string_len, mkxp_sandbox::length, value);
+                    SANDBOX_AWAIT_AND_SET(script_string, mkxp_sandbox::rb_string_value_ptr, &value);
+
+                    // Decompress the script contents
+                    {
+                        int zlib_result = Z_OK;
+                        unsigned long buffer_len = decode_buffer_len - 1;
+                        while (true) {
+                            zlib_result = uncompress(
+                                decode_buffer,
+                                &buffer_len,
+                                (unsigned char *)(*mkxp_sandbox::sandbox->bindings + script_string),
+                                script_string_len
+                            );
+                            decode_buffer[buffer_len] = 0;
+
+                            if (zlib_result != Z_BUF_ERROR) {
+                                break;
+                            }
+
+                            unsigned long new_decode_buffer_len;
+                            if (__builtin_add_overflow(decode_buffer_len, decode_buffer_len, &new_decode_buffer_len)) {
+                                throw SandboxOutOfMemoryException();
+                            }
+                            decode_buffer_len = new_decode_buffer_len;
+                            unsigned char *new_decode_buffer = (unsigned char *)std::realloc(decode_buffer, decode_buffer_len);
+                            if (new_decode_buffer == NULL) {
+                                throw SandboxOutOfMemoryException();
+                            }
+                            decode_buffer = new_decode_buffer;
+                            buffer_len = decode_buffer_len - 1;
+                        }
+
+                        if (zlib_result != Z_OK) {
+                            mkxp_retro::log_printf(RETRO_LOG_ERROR, "Error decoding script %zu: '%s'\n", i, (const char *)(*mkxp_sandbox::sandbox->bindings + script_name));
+                            break;
+                        }
+                    }
+
+                    SANDBOX_AWAIT_AND_SET(value, mkxp_sandbox::rb_utf8_str_new_cstr, (const char *)decode_buffer);
+                    SANDBOX_AWAIT(mkxp_sandbox::rb_ary_store, script, 3, value);
+                }
+
+                std::free(decode_buffer);
+                decode_buffer = NULL;
+
+                // TODO: preload scripts
+
+                while (true) {
+                    for (i = 0; i < script_count; ++i) {
+                        SANDBOX_AWAIT_AND_SET(script_filename_value, mkxp_sandbox::rb_ary_entry, script, 1);
+                        SANDBOX_AWAIT_AND_SET(script_string_value, mkxp_sandbox::rb_ary_entry, script, 3);
+                        SANDBOX_AWAIT_AND_SET(value, mkxp_sandbox::eval_script, script_string_value, script_filename_value);
+                        if (value != SANDBOX_UNDEF) {
+                            SANDBOX_AWAIT(mkxp_sandbox::log_backtrace, value);
+                            break;
+                        }
+                    }
+                    if (value != SANDBOX_UNDEF) {
+                        break;
+                    }
+                }
+            }
+        }
+
+        ~run_rmxp_scripts() {
+            if (decode_buffer != NULL) {
+                std::free(decode_buffer);
+            }
+        }
+    };
+}
+
+#endif // MKXPZ_BINDING_SANDBOX_H

binding-sandbox/sandbox.h

@@ -26,6 +26,29 @@
 #include <mkxp-sandbox-bindgen.h>
 #include "types.h"
 
+#define SANDBOX_AWAIT(coroutine, ...) \
+    do { \
+        { \
+            mkxp_sandbox::bindings::stack_frame<struct coroutine> frame = mkxp_sandbox::sandbox->bindings.bind<struct coroutine>(); \
+            frame()(__VA_ARGS__); \
+            if (frame().is_complete()) break; \
+        } \
+        yield; \
+    } while (1)
+
+#define SANDBOX_AWAIT_AND_SET(variable, coroutine, ...) \
+    do { \
+        { \
+            mkxp_sandbox::bindings::stack_frame<struct coroutine> frame = mkxp_sandbox::sandbox->bindings.bind<struct coroutine>(); \
+            auto ret = frame()(__VA_ARGS__); \
+            if (frame().is_complete()) { \
+                variable = ret; \
+                break; \
+            } \
+        } \
+        yield; \
+    } while (1)
+
 namespace mkxp_sandbox {
     struct sandbox {
         private:
@@ -48,6 +71,8 @@ namespace mkxp_sandbox {
             }
         }
     };
+
+    extern std::unique_ptr<struct mkxp_sandbox::sandbox> sandbox;
 }
 
 #endif // MKXPZ_SANDBOX_H

retro/Makefile

@@ -96,7 +96,7 @@ $(LIBDIR)/mkxp-retro-dist.zip: $(LIBDIR)/mkxp-retro-dist/bin/ruby $(P7ZIP)
 	rm $(LIBDIR)/_mkxp-retro-dist/lib/libruby-static.a
 	rm -r $(LIBDIR)/_mkxp-retro-dist/include
 	rm -r $(LIBDIR)/_mkxp-retro-dist/share
-	rm -r $(LIBDIR)/_mkxp-retro-dist/lib/ruby/gems/3.3.0/cache/*
+	rm -r $(LIBDIR)/_mkxp-retro-dist/lib/ruby/gems/$(RUBY_VERSION).0/cache/*
 	rm -f $(LIBDIR)/mkxp-retro-dist.zip
 	cd $(LIBDIR)/_mkxp-retro-dist && $(P7ZIP) a -bb3 -mm=zstd -mx=19 $(LIBDIR)/mkxp-retro-dist.zip *
 	rm -r $(LIBDIR)/_mkxp-retro-dist
@@ -110,10 +110,12 @@ $(OUTDIR)/mkxp-sandbox-bindgen.cpp $(OUTDIR)/mkxp-sandbox-bindgen.h &: sandbox-b
 	mv $(LIBDIR)/mkxp-sandbox-bindgen.h $(OUTDIR)
 	mv $(LIBDIR)/mkxp-sandbox-bindgen.cpp $(OUTDIR)
 
-$(LIBDIR)/tags: $(DOWNLOADS)/crossruby/.ext/include/$(TARGET)/ruby/config.h
+$(LIBDIR)/tags: $(LIBDIR)/tags.c
+	$(CTAGS) -R --fields=kS --kinds-c=epx -o $(LIBDIR)/tags $(LIBDIR)/tags.c
+
+$(LIBDIR)/tags.c: $(DOWNLOADS)/crossruby/.ext/include/$(TARGET)/ruby/config.h
 	mkdir -p $(LIBDIR)
 	echo '#include <ruby.h>' | $(WASI_CC) -E -I$(DOWNLOADS)/crossruby/include -I$(DOWNLOADS)/crossruby/.ext/include/$(TARGET) -o $(LIBDIR)/tags.c -
-	$(CTAGS) -R --fields=kS --kinds-c=epx -o $(LIBDIR)/tags $(LIBDIR)/tags.c
 
 $(DOWNLOADS)/crossruby/Makefile $(DOWNLOADS)/crossruby/.ext/include/$(TARGET)/ruby/config.h &: $(DOWNLOADS)/crossruby/configure $(RUBY) $(LIBDIR)/usr/local/lib/libyaml.a $(LIBDIR)/usr/local/lib/libz.a $(LIBDIR)/usr/local/lib/libssl.a
 	cd $(DOWNLOADS)/crossruby && ./configure \

retro/README.md

@@ -16,10 +16,14 @@ This will produce the directory "retro/build/retro-phase1".
 
 # Phase 2
 
-This phase produces the actual core file. Go to the root directory of this repository and run:
+This phase produces the actual core file. You need to have [Meson](https://mesonbuild.com), [Ninja](https://ninja-build.org), [CMake](https://cmake.org), [Git](https://git-scm.com), a C compiler and a C++ compiler. No software libraries are required other than the system libraries.
+
+Go to the root directory of this repository and run:
 
 ```
 meson setup build -Dretro=true -Dretro_phase1_path=path/to/retro-phase1
 cd build
 ninja
 ```
+
+If you have a program named `patch` in your PATH, it has to be GNU patch. If your `patch` program isn't GNU patch (e.g. macOS comes with its own incompatible version of `patch` that will break this build system), either install GNU patch and then temporarily add it to your PATH for the duration of the `meson setup` command, or temporarily remove your `patch` program from the PATH for the duration of the `meson setup` command: the build system will fall back to using `git apply` if `patch` is not found, which will work fine.

retro/sandbox-bindgen.rb

@@ -242,11 +242,9 @@ HEADER_START = <<~HEREDOC
                       throw SandboxTrapException();
                   }
 
-                  try {
-                      T &inner = boost::any_cast<T &>(fiber.stack[fiber.stack_ptr]);
-                      ++fiber.stack_ptr;
-                      return inner;
-                  } catch (boost::bad_any_cast &) {
+                  if (fiber.stack[fiber.stack_ptr].type() == typeid(T &)) {
+                      return boost::any_cast<T &>(fiber.stack[fiber.stack_ptr++]);
+                  } else {
                       fiber.stack.resize(fiber.stack_ptr++);
                       fiber.stack.push_back(T(bind));
                       return boost::any_cast<T &>(fiber.stack.back());

src/core.cpp

@@ -23,30 +23,10 @@
 #include <cstdlib>
 #include <cstdarg>
 #include <cstring>
-#include <memory>
 #include "../binding-sandbox/sandbox.h"
+#include "../binding-sandbox/binding-sandbox.h"
 #include "../binding-sandbox/core.h"
 
-#define SANDBOX_AWAIT(coroutine, ...) \
-    do { \
-        { \
-            auto frame = sandbox->bindings.bind<struct coroutine>(); \
-            frame()(__VA_ARGS__); \
-            if (frame().is_complete()) break; \
-        } \
-        yield; \
-    } while (1)
-
-#define SANDBOX_AWAIT_AND_SET(variable, coroutine, ...) \
-    do { \
-        { \
-            auto frame = sandbox->bindings.bind<struct coroutine>(); \
-            variable = frame()(__VA_ARGS__); \
-            if (frame().is_complete()) break; \
-        } \
-        yield; \
-    } while (1)
-
 using namespace mkxp_retro;
 
 static void fallback_log(enum retro_log_level level, const char *fmt, ...) {
@@ -57,12 +37,12 @@ static void fallback_log(enum retro_log_level level, const char *fmt, ...) {
 }
 
 static uint32_t *frame_buf;
-static std::unique_ptr<struct mkxp_sandbox::sandbox> sandbox;
+std::unique_ptr<struct mkxp_sandbox::sandbox> mkxp_sandbox::sandbox;
 static const char *game_path = NULL;
 
 static VALUE my_cpp_func(void *_, VALUE self, VALUE args) {
-    struct co : boost::asio::coroutine {
-        inline co(struct mkxp_sandbox::bindings &bind) {}
+    struct coro : boost::asio::coroutine {
+        inline coro(struct mkxp_sandbox::bindings &bind) {}
 
         void operator()(VALUE args) {
             reenter (this) {
@@ -72,14 +52,14 @@ static VALUE my_cpp_func(void *_, VALUE self, VALUE args) {
         }
     };
 
-    sandbox->bindings.bind<struct co>()()(args);
+    mkxp_sandbox::sandbox->bindings.bind<struct coro>()()(args);
 
     return self;
 }
 
 static VALUE func(void *_, VALUE arg) {
-    struct co : boost::asio::coroutine {
-        inline co(struct mkxp_sandbox::bindings &bind) {}
+    struct coro : boost::asio::coroutine {
+        inline coro(struct mkxp_sandbox::bindings &bind) {}
 
         void operator()() {
             reenter (this) {
@@ -92,6 +72,8 @@ static VALUE func(void *_, VALUE arg) {
 
                 SANDBOX_AWAIT(mkxp_sandbox::rb_eval_string, "p Dir.glob '/mkxp-retro-game/*'");
 
+                SANDBOX_AWAIT(mkxp_sandbox::run_rmxp_scripts);
+
                 SANDBOX_AWAIT(mkxp_sandbox::rb_eval_string, "throw 'Throw an error on purpose to see if we can catch it'");
 
                 SANDBOX_AWAIT(mkxp_sandbox::rb_eval_string, "puts 'Unreachable code'");
@@ -99,24 +81,24 @@ static VALUE func(void *_, VALUE arg) {
         }
     };
 
-    sandbox->bindings.bind<struct co>()()();
+    mkxp_sandbox::sandbox->bindings.bind<struct coro>()()();
 
     return arg;
 }
 
-static VALUE rescue(void *_, VALUE arg, VALUE error) {
-    struct co : boost::asio::coroutine {
-        inline co(struct mkxp_sandbox::bindings &bind) {}
+static VALUE rescue(void *_, VALUE arg, VALUE exception) {
+    struct coro : boost::asio::coroutine {
+        inline coro(struct mkxp_sandbox::bindings &bind) {}
 
-        void operator()(VALUE error) {
+        void operator()(VALUE exception) {
             reenter (this) {
                 SANDBOX_AWAIT(mkxp_sandbox::rb_eval_string, "puts 'Entered rescue()'");
-                SANDBOX_AWAIT(mkxp_sandbox::rb_p, error);
+                SANDBOX_AWAIT(mkxp_sandbox::rb_p, exception);
             }
         }
     };
 
-    sandbox->bindings.bind<struct co>()()(error);
+    mkxp_sandbox::sandbox->bindings.bind<struct coro>()()(exception);
 
     return arg;
 }
@@ -130,14 +112,14 @@ static bool init_sandbox() {
         }
     };
 
-    sandbox.reset();
+    mkxp_sandbox::sandbox.reset();
 
     try {
-        sandbox.reset(new struct mkxp_sandbox::sandbox(game_path));
-        sandbox->run<struct main>();
+        mkxp_sandbox::sandbox.reset(new struct mkxp_sandbox::sandbox(game_path));
+        mkxp_sandbox::sandbox->run<struct main>();
     } catch (SandboxException) {
         log_printf(RETRO_LOG_ERROR, "Failed to initialize Ruby\n");
-        sandbox.reset();
+        mkxp_sandbox::sandbox.reset();
         return false;
     }
 
@@ -277,7 +259,7 @@ extern "C" RETRO_API bool retro_load_game_special(unsigned int type, const struc
 }
 
 extern "C" RETRO_API void retro_unload_game() {
-    sandbox.reset();
+    mkxp_sandbox::sandbox.reset();
 }
 
 extern "C" RETRO_API unsigned int retro_get_region() {