Async environment for mods to do concurrent tasks (#11131)

builtin/async/game.lua

@@ -0,0 +1,46 @@
+core.log("info", "Initializing asynchronous environment (game)")
+
+local function pack2(...)
+	return {n=select('#', ...), ...}
+end
+
+-- Entrypoint to run async jobs, called by C++
+function core.job_processor(func, params)
+	local retval = pack2(func(unpack(params, 1, params.n)))
+
+	return retval
+end
+
+-- Import a bunch of individual files from builtin/game/
+local gamepath = core.get_builtin_path() .. "game" .. DIR_DELIM
+
+dofile(gamepath .. "constants.lua")
+dofile(gamepath .. "item_s.lua")
+dofile(gamepath .. "misc_s.lua")
+dofile(gamepath .. "features.lua")
+dofile(gamepath .. "voxelarea.lua")
+
+-- Transfer of globals
+do
+	assert(core.transferred_globals)
+	local all = core.deserialize(core.transferred_globals, true)
+	core.transferred_globals = nil
+
+	-- reassemble other tables
+	all.registered_nodes = {}
+	all.registered_craftitems = {}
+	all.registered_tools = {}
+	for k, v in pairs(all.registered_items) do
+		if v.type == "node" then
+			all.registered_nodes[k] = v
+		elseif v.type == "craftitem" then
+			all.registered_craftitems[k] = v
+		elseif v.type == "tool" then
+			all.registered_tools[k] = v
+		end
+	end
+
+	for k, v in pairs(all) do
+		core[k] = v
+	end
+end

builtin/async/mainmenu.lua

@@ -1,5 +1,4 @@
-
-core.log("info", "Initializing Asynchronous environment")
+core.log("info", "Initializing asynchronous environment")
 
 function core.job_processor(func, serialized_param)
 	local param = core.deserialize(serialized_param)
@@ -8,4 +7,3 @@ function core.job_processor(func, serialized_param)
 
 	return retval or core.serialize(nil)
 end
-

builtin/game/async.lua

@@ -0,0 +1,22 @@
+
+core.async_jobs = {}
+
+function core.async_event_handler(jobid, retval)
+	local callback = core.async_jobs[jobid]
+	assert(type(callback) == "function")
+	callback(unpack(retval, 1, retval.n))
+	core.async_jobs[jobid] = nil
+end
+
+function core.handle_async(func, callback, ...)
+	assert(type(func) == "function" and type(callback) == "function",
+		"Invalid minetest.handle_async invocation")
+	local args = {n = select("#", ...), ...}
+	local mod_origin = core.get_last_run_mod()
+
+	local jobid = core.do_async_callback(func, args, mod_origin)
+	core.async_jobs[jobid] = callback
+
+	return true
+end
+

builtin/game/init.lua

@@ -34,5 +34,6 @@ dofile(gamepath .. "voxelarea.lua")
 dofile(gamepath .. "forceloading.lua")
 dofile(gamepath .. "statbars.lua")
 dofile(gamepath .. "knockback.lua")
+dofile(gamepath .. "async.lua")
 
 profiler = nil

builtin/game/misc.lua

@@ -235,3 +235,32 @@ end
 
 -- Used for callback handling with dynamic_add_media
 core.dynamic_media_callbacks = {}
+
+
+-- Transfer of certain globals into async environment
+-- see builtin/async/game.lua for the other side
+
+local function copy_filtering(t, seen)
+	if type(t) == "userdata" or type(t) == "function" then
+		return true -- don't use nil so presence can still be detected
+	elseif type(t) ~= "table" then
+		return t
+	end
+	local n = {}
+	seen = seen or {}
+	seen[t] = n
+	for k, v in pairs(t) do
+		local k_ = seen[k] or copy_filtering(k, seen)
+		local v_ = seen[v] or copy_filtering(v, seen)
+		n[k_] = v_
+	end
+	return n
+end
+
+function core.get_globals_to_transfer()
+	local all = {
+		registered_items = copy_filtering(core.registered_items),
+		registered_aliases = core.registered_aliases,
+	}
+	return core.serialize(all)
+end

builtin/init.lua

@@ -56,8 +56,10 @@ elseif INIT == "mainmenu" then
 	if not custom_loaded then
 		dofile(core.get_mainmenu_path() .. DIR_DELIM .. "init.lua")
 	end
-elseif INIT == "async" then
-	dofile(asyncpath .. "init.lua")
+elseif INIT == "async"  then
+	dofile(asyncpath .. "mainmenu.lua")
+elseif INIT == "async_game" then
+	dofile(asyncpath .. "game.lua")
 elseif INIT == "client" then
 	dofile(clientpath .. "init.lua")
 else

doc/lua_api.txt

@@ -5767,6 +5767,68 @@ Timing
 * `job:cancel()`
     * Cancels the job function from being called
 
+Async environment
+-----------------
+
+The engine allows you to submit jobs to be ran in an isolated environment
+concurrently with normal server operation.
+A job consists of a function to be ran in the async environment, any amount of
+arguments (will be serialized) and a callback that will be called with the return
+value of the job function once it is finished.
+
+The async environment does *not* have access to the map, entities, players or any
+globals defined in the 'usual' environment. Consequently, functions like
+`minetest.get_node()` or `minetest.get_player_by_name()` simply do not exist in it.
+
+Arguments and return values passed through this can contain certain userdata
+objects that will be seamlessly copied (not shared) to the async environment.
+This allows you easy interoperability for delegating work to jobs.
+
+* `minetest.handle_async(func, callback, ...)`:
+    * Queue the function `func` to be ran in an async environment.
+      Note that there are multiple persistent workers and any of them may
+      end up running a given job. The engine will scale the amount of
+      worker threads automatically.
+    * When `func` returns the callback is called (in the normal environment)
+      with all of the return values as arguments.
+    * Optional: Variable number of arguments that are passed to `func`
+* `minetest.register_async_dofile(path)`:
+    * Register a path to a Lua file to be imported when an async environment
+      is initialized. You can use this to preload code which you can then call
+      later using `minetest.handle_async()`.
+
+### List of APIs available in an async environment
+
+Classes:
+* `ItemStack`
+* `PerlinNoise`
+* `PerlinNoiseMap`
+* `PseudoRandom`
+* `PcgRandom`
+* `SecureRandom`
+* `VoxelArea`
+* `VoxelManip`
+    * only if transferred into environment; can't read/write to map
+* `Settings`
+
+Class instances that can be transferred between environments:
+* `ItemStack`
+* `PerlinNoise`
+* `PerlinNoiseMap`
+* `VoxelManip`
+
+Functions:
+* Standalone helpers such as logging, filesystem, encoding,
+  hashing or compression APIs
+* `minetest.request_insecure_environment` (same restrictions apply)
+
+Variables:
+* `minetest.settings`
+* `minetest.registered_items`, `registered_nodes`, `registered_tools`,
+  `registered_craftitems` and `registered_aliases`
+    * with all functions and userdata values replaced by `true`, calling any
+      callbacks here is obviously not possible
+
 Server
 ------
 

games/devtest/mods/unittests/async_env.lua

@@ -0,0 +1,149 @@
+-- helper
+
+core.register_async_dofile(core.get_modpath(core.get_current_modname()) ..
+	DIR_DELIM .. "inside_async_env.lua")
+
+local function deepequal(a, b)
+	if type(a) == "function" then
+		return type(b) == "function"
+	elseif type(a) ~= "table" then
+		return a == b
+	elseif type(b) ~= "table" then
+		return false
+	end
+	for k, v in pairs(a) do
+		if not deepequal(v, b[k]) then
+			return false
+		end
+	end
+	for k, v in pairs(b) do
+		if not deepequal(a[k], v) then
+			return false
+		end
+	end
+	return true
+end
+
+-- Object Passing / Serialization
+
+local test_object = {
+	name = "stairs:stair_glass",
+	type = "node",
+	groups = {oddly_breakable_by_hand = 3, cracky = 3, stair = 1},
+	description = "Glass Stair",
+	sounds = {
+		dig = {name = "default_glass_footstep", gain = 0.5},
+		footstep = {name = "default_glass_footstep", gain = 0.3},
+		dug = {name = "default_break_glass", gain = 1}
+	},
+	node_box = {
+		fixed = {
+			{-0.5, -0.5, -0.5, 0.5, 0, 0.5},
+			{-0.5, 0, 0, 0.5, 0.5, 0.5}
+		},
+		type = "fixed"
+	},
+	tiles = {
+		{name = "stairs_glass_split.png", backface_culling = true},
+		{name = "default_glass.png", backface_culling = true},
+		{name = "stairs_glass_stairside.png^[transformFX", backface_culling = true}
+	},
+	on_place = function(itemstack, placer)
+		return core.is_player(placer)
+	end,
+	sunlight_propagates = true,
+	is_ground_content = false,
+	light_source = 0,
+}
+
+local function test_object_passing()
+	local tmp = core.serialize_roundtrip(test_object)
+	assert(deepequal(test_object, tmp))
+
+	-- Circular key, should error
+	tmp = {"foo", "bar"}
+	tmp[tmp] = true
+	assert(not pcall(core.serialize_roundtrip, tmp))
+
+	-- Circular value, should error
+	tmp = {"foo"}
+	tmp[2] = tmp
+	assert(not pcall(core.serialize_roundtrip, tmp))
+end
+unittests.register("test_object_passing", test_object_passing)
+
+local function test_userdata_passing(_, pos)
+	-- basic userdata passing
+	local obj = table.copy(test_object.tiles[1])
+	obj.test = ItemStack("default:cobble 99")
+	local tmp = core.serialize_roundtrip(obj)
+	assert(type(tmp.test) == "userdata")
+	assert(obj.test:to_string() == tmp.test:to_string())
+
+	-- object can't be passed, should error
+	obj = core.raycast(pos, pos)
+	assert(not pcall(core.serialize_roundtrip, obj))
+
+	-- VManip
+	local vm = core.get_voxel_manip(pos, pos)
+	local expect = vm:get_node_at(pos)
+	local vm2 = core.serialize_roundtrip(vm)
+	assert(deepequal(vm2:get_node_at(pos), expect))
+end
+unittests.register("test_userdata_passing", test_userdata_passing, {map=true})
+
+-- Asynchronous jobs
+
+local function test_handle_async(cb)
+	-- Basic test including mod name tracking and unittests.async_test()
+	-- which is defined inside_async_env.lua
+	local func = function(x)
+		return core.get_last_run_mod(), _VERSION, unittests[x]()
+	end
+	local expect = {core.get_last_run_mod(), _VERSION, true}
+
+	core.handle_async(func, function(...)
+		if not deepequal(expect, {...}) then
+			cb("Values did not equal")
+		end
+		if core.get_last_run_mod() ~= expect[1] then
+			cb("Mod name not tracked correctly")
+		end
+
+		-- Test passing of nil arguments and return values
+		core.handle_async(function(a, b)
+			return a, b
+		end, function(a, b)
+			if b ~= 123 then
+				cb("Argument went missing")
+			end
+			cb()
+		end, nil, 123)
+	end, "async_test")
+end
+unittests.register("test_handle_async", test_handle_async, {async=true})
+
+local function test_userdata_passing2(cb, _, pos)
+	-- VManip: check transfer into other env
+	local vm = core.get_voxel_manip(pos, pos)
+	local expect = vm:get_node_at(pos)
+
+	core.handle_async(function(vm_, pos_)
+		return vm_:get_node_at(pos_)
+	end, function(ret)
+		if not deepequal(expect, ret) then
+			cb("Node data mismatch (one-way)")
+		end
+
+		-- VManip: test a roundtrip
+		core.handle_async(function(vm_)
+			return vm_
+		end, function(vm2)
+			if not deepequal(expect, vm2:get_node_at(pos)) then
+				cb("Node data mismatch (roundtrip)")
+			end
+			cb()
+		end, vm)
+	end, vm, pos)
+end
+unittests.register("test_userdata_passing2", test_userdata_passing2, {map=true, async=true})

games/devtest/mods/unittests/init.lua

@@ -175,6 +175,7 @@ dofile(modpath .. "/misc.lua")
 dofile(modpath .. "/player.lua")
 dofile(modpath .. "/crafting.lua")
 dofile(modpath .. "/itemdescription.lua")
+dofile(modpath .. "/async_env.lua")
 
 --------------
 

games/devtest/mods/unittests/inside_async_env.lua

@@ -0,0 +1,15 @@
+unittests = {}
+
+core.log("info", "Hello World")
+
+function unittests.async_test()
+	assert(core == minetest)
+	-- stuff that should not be here
+	assert(not core.get_player_by_name)
+	assert(not core.set_node)
+	assert(not core.object_refs)
+	-- stuff that should be here
+	assert(ItemStack)
+	assert(core.registered_items[""])
+	return true
+end

src/map.cpp

@@ -1896,6 +1896,7 @@ MMVManip::MMVManip(Map *map):
 		VoxelManipulator(),
 		m_map(map)
 {
+	assert(map);
 }
 
 void MMVManip::initialEmerge(v3s16 blockpos_min, v3s16 blockpos_max,
@@ -1903,6 +1904,8 @@ void MMVManip::initialEmerge(v3s16 blockpos_min, v3s16 blockpos_max,
 {
 	TimeTaker timer1("initialEmerge", &emerge_time);
 
+	assert(m_map);
+
 	// Units of these are MapBlocks
 	v3s16 p_min = blockpos_min;
 	v3s16 p_max = blockpos_max;
@@ -1986,6 +1989,7 @@ void MMVManip::blitBackAll(std::map<v3s16, MapBlock*> *modified_blocks,
 {
 	if(m_area.getExtent() == v3s16(0,0,0))
 		return;
+	assert(m_map);
 
 	/*
 		Copy data of all blocks
@@ -2006,4 +2010,33 @@ void MMVManip::blitBackAll(std::map<v3s16, MapBlock*> *modified_blocks,
 	}
 }
 
+MMVManip *MMVManip::clone() const
+{
+	MMVManip *ret = new MMVManip();
+
+	const s32 size = m_area.getVolume();
+	ret->m_area = m_area;
+	if (m_data) {
+		ret->m_data = new MapNode[size];
+		memcpy(ret->m_data, m_data, size * sizeof(MapNode));
+	}
+	if (m_flags) {
+		ret->m_flags = new u8[size];
+		memcpy(ret->m_flags, m_flags, size * sizeof(u8));
+	}
+
+	ret->m_is_dirty = m_is_dirty;
+	// Even if the copy is disconnected from a map object keep the information
+	// needed to write it back to one
+	ret->m_loaded_blocks = m_loaded_blocks;
+
+	return ret;
+}
+
+void MMVManip::reparent(Map *map)
+{
+	assert(map && !m_map);
+	m_map = map;
+}
+
 //END

src/map.h

@@ -446,10 +446,25 @@ public:
 	void blitBackAll(std::map<v3s16, MapBlock*> * modified_blocks,
 		bool overwrite_generated = true);
 
+	/*
+		Creates a copy of this VManip including contents, the copy will not be
+		associated with a Map.
+	*/
+	MMVManip *clone() const;
+
+	// Reassociates a copied VManip to a map
+	void reparent(Map *map);
+
+	// Is it impossible to call initialEmerge / blitBackAll?
+	inline bool isOrphan() const { return !m_map; }
+
 	bool m_is_dirty = false;
 
 protected:
-	Map *m_map;
+	MMVManip() {};
+
+	// may be null
+	Map *m_map = nullptr;
 	/*
 		key = blockpos
 		value = flags describing the block

src/script/common/CMakeLists.txt

@@ -3,6 +3,7 @@ set(common_SCRIPT_COMMON_SRCS
 	${CMAKE_CURRENT_SOURCE_DIR}/c_converter.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/c_types.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/c_internal.cpp
+	${CMAKE_CURRENT_SOURCE_DIR}/c_packer.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/helper.cpp
 	PARENT_SCOPE)
 

src/script/common/c_internal.cpp

@@ -166,3 +166,17 @@ void log_deprecated(lua_State *L, std::string message, int stack_depth)
 		infostream << script_get_backtrace(L) << std::endl;
 }
 
+void call_string_dump(lua_State *L, int idx)
+{
+	// Retrieve string.dump from insecure env to avoid it being tampered with
+	lua_rawgeti(L, LUA_REGISTRYINDEX, CUSTOM_RIDX_GLOBALS_BACKUP);
+	if (!lua_isnil(L, -1))
+		lua_getfield(L, -1, "string");
+	else
+		lua_getglobal(L, "string");
+	lua_getfield(L, -1, "dump");
+	lua_remove(L, -2); // remove _G
+	lua_remove(L, -2); // remove 'string' table
+	lua_pushvalue(L, idx);
+	lua_call(L, 1, 1);
+}

src/script/common/c_internal.h

@@ -56,6 +56,7 @@ extern "C" {
 #define CUSTOM_RIDX_BACKTRACE           (CUSTOM_RIDX_BASE + 3)
 #define CUSTOM_RIDX_HTTP_API_LUA        (CUSTOM_RIDX_BASE + 4)
 #define CUSTOM_RIDX_VECTOR_METATABLE    (CUSTOM_RIDX_BASE + 5)
+#define CUSTOM_RIDX_METATABLE_MAP       (CUSTOM_RIDX_BASE + 6)
 
 
 // Determine if CUSTOM_RIDX_SCRIPTAPI will hold a light or full userdata
@@ -139,3 +140,7 @@ DeprecatedHandlingMode get_deprecated_handling_mode();
  * @param stack_depth How far on the stack to the first user function (ie: not builtin or core)
  */
 void log_deprecated(lua_State *L, std::string message, int stack_depth = 1);
+
+// Safely call string.dump on a function value
+// (does not pop, leaves one value on stack)
+void call_string_dump(lua_State *L, int idx);

src/script/common/c_packer.cpp

@@ -0,0 +1,583 @@
+/*
+Minetest
+Copyright (C) 2022 sfan5 <sfan5@live.de>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU Lesser General Public License as published by
+the Free Software Foundation; either version 2.1 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+*/
+
+#include <cstdio>
+#include <cstring>
+#include <cmath>
+#include <cassert>
+#include <unordered_set>
+#include <unordered_map>
+#include "c_packer.h"
+#include "c_internal.h"
+#include "log.h"
+#include "debug.h"
+#include "threading/mutex_auto_lock.h"
+
+extern "C" {
+#include <lauxlib.h>
+}
+
+//
+// Helpers
+//
+
+// convert negative index to absolute position on Lua stack
+static inline int absidx(lua_State *L, int idx)
+{
+	assert(idx < 0);
+	return lua_gettop(L) + idx + 1;
+}
+
+// does the type put anything into PackedInstr::sdata?
+static inline bool uses_sdata(int type)
+{
+	switch (type) {
+		case LUA_TSTRING:
+		case LUA_TFUNCTION:
+		case LUA_TUSERDATA:
+			return true;
+		default:
+			return false;
+	}
+}
+
+// does the type put anything into PackedInstr::<union>?
+static inline bool uses_union(int type)
+{
+	switch (type) {
+		case LUA_TNIL:
+		case LUA_TSTRING:
+		case LUA_TFUNCTION:
+			return false;
+		default:
+			return true;
+	}
+}
+
+static inline bool can_set_into(int ktype, int vtype)
+{
+	switch (ktype) {
+		case LUA_TNUMBER:
+			return !uses_union(vtype);
+		case LUA_TSTRING:
+			return !uses_sdata(vtype);
+		default:
+			return false;
+	}
+}
+
+// is the key suitable for use with set_into?
+static inline bool suitable_key(lua_State *L, int idx)
+{
+	if (lua_type(L, idx) == LUA_TSTRING) {
+		// strings may not have a NULL byte (-> lua_setfield)
+		size_t len;
+		const char *str = lua_tolstring(L, idx, &len);
+		return strlen(str) == len;
+	} else {
+		assert(lua_type(L, idx) == LUA_TNUMBER);
+		// numbers must fit into an s32 and be integers (-> lua_rawseti)
+		lua_Number n = lua_tonumber(L, idx);
+		return std::floor(n) == n && n >= S32_MIN && n <= S32_MAX;
+	}
+}
+
+namespace {
+	// checks if you left any values on the stack, for debugging
+	class StackChecker {
+		lua_State *L;
+		int top;
+	public:
+		StackChecker(lua_State *L) : L(L), top(lua_gettop(L)) {}
+		~StackChecker() {
+			assert(lua_gettop(L) >= top);
+			if (lua_gettop(L) > top) {
+				rawstream << "Lua stack not cleaned up: "
+					<< lua_gettop(L) << " != " << top
+					<< " (false-positive if exception thrown)" << std::endl;
+			}
+		}
+	};
+
+	// Since an std::vector may reallocate, this is the only safe way to keep
+	// a reference to a particular element.
+	template <typename T>
+	class VectorRef {
+		std::vector<T> *vec;
+		size_t idx;
+		VectorRef(std::vector<T> *vec, size_t idx) : vec(vec), idx(idx) {}
+	public:
+		static VectorRef<T> front(std::vector<T> &vec) {
+			return VectorRef(&vec, 0);
+		}
+		static VectorRef<T> back(std::vector<T> &vec) {
+			return VectorRef(&vec, vec.size() - 1);
+		}
+		T &operator*() { return (*vec)[idx]; }
+		T *operator->() { return &(*vec)[idx]; }
+	};
+
+	struct Packer {
+		PackInFunc fin;
+		PackOutFunc fout;
+	};
+
+	typedef std::pair<std::string, Packer> PackerTuple;
+}
+
+static inline auto emplace(PackedValue &pv, s16 type)
+{
+	pv.i.emplace_back();
+	auto ref = VectorRef<PackedInstr>::back(pv.i);
+	ref->type = type;
+	// Initialize fields that may be left untouched
+	if (type == LUA_TTABLE) {
+		ref->uidata1 = 0;
+		ref->uidata2 = 0;
+	} else if (type == LUA_TUSERDATA) {
+		ref->ptrdata = nullptr;
+	} else if (type == INSTR_POP) {
+		ref->sidata2 = 0;
+	}
+	return ref;
+}
+
+//
+// Management of registered packers
+//
+
+static std::unordered_map<std::string, Packer> g_packers;
+static std::mutex g_packers_lock;
+
+void script_register_packer(lua_State *L, const char *regname,
+	PackInFunc fin, PackOutFunc fout)
+{
+	// Store away callbacks
+	{
+		MutexAutoLock autolock(g_packers_lock);
+		auto it = g_packers.find(regname);
+		if (it == g_packers.end()) {
+			auto &ref = g_packers[regname];
+			ref.fin = fin;
+			ref.fout = fout;
+		} else {
+			FATAL_ERROR_IF(it->second.fin != fin || it->second.fout != fout,
+				"Packer registered twice with mismatching callbacks");
+		}
+	}
+
+	// Save metatable so we can identify instances later
+	lua_rawgeti(L, LUA_REGISTRYINDEX, CUSTOM_RIDX_METATABLE_MAP);
+	if (lua_isnil(L, -1)) {
+		lua_newtable(L);
+		lua_pushvalue(L, -1);
+		lua_rawseti(L, LUA_REGISTRYINDEX, CUSTOM_RIDX_METATABLE_MAP);
+	}
+
+	luaL_getmetatable(L, regname);
+	FATAL_ERROR_IF(lua_isnil(L, -1), "No metatable registered with that name");
+
+	// CUSTOM_RIDX_METATABLE_MAP contains { [metatable] = "regname", ... }
+	// check first
+	lua_pushstring(L, regname);
+	lua_rawget(L, -3);
+	if (!lua_isnil(L, -1)) {
+		FATAL_ERROR_IF(lua_topointer(L, -1) != lua_topointer(L, -2),
+				"Packer registered twice with inconsistent metatable");
+	}
+	lua_pop(L, 1);
+	// then set
+	lua_pushstring(L, regname);
+	lua_rawset(L, -3);
+
+	lua_pop(L, 1);
+}
+
+static bool find_packer(const char *regname, PackerTuple &out)
+{
+	MutexAutoLock autolock(g_packers_lock);
+	auto it = g_packers.find(regname);
+	if (it == g_packers.end())
+		return false;
+	// copy data for thread safety
+	out.first = it->first;
+	out.second = it->second;
+	return true;
+}
+
+static bool find_packer(lua_State *L, int idx, PackerTuple &out)
+{
+#ifndef NDEBUG
+	StackChecker checker(L);
+#endif
+
+	// retrieve metatable of the object
+	if (lua_getmetatable(L, idx) != 1)
+		return false;
+
+	// use our global table to map it to the registry name
+	lua_rawgeti(L, LUA_REGISTRYINDEX, CUSTOM_RIDX_METATABLE_MAP);
+	assert(lua_istable(L, -1));
+	lua_pushvalue(L, -2);
+	lua_rawget(L, -2);
+	if (lua_isnil(L, -1)) {
+		lua_pop(L, 3);
+		return false;
+	}
+
+	// load the associated data
+	bool found = find_packer(lua_tostring(L, -1), out);
+	FATAL_ERROR_IF(!found, "Inconsistent internal state");
+	lua_pop(L, 3);
+	return true;
+}
+
+//
+// Packing implementation
+//
+
+// recursively goes through the structure and ensures there are no circular references
+static void pack_validate(lua_State *L, int idx, std::unordered_set<const void*> &seen)
+{
+#ifndef NDEBUG
+	StackChecker checker(L);
+	assert(idx > 0);
+#endif
+
+	if (lua_type(L, idx) != LUA_TTABLE)
+		return;
+
+	const void *ptr = lua_topointer(L, idx);
+	assert(ptr);
+
+	if (seen.find(ptr) != seen.end())
+		throw LuaError("Circular references cannot be packed (yet)");
+	seen.insert(ptr);
+
+	lua_checkstack(L, 5);
+	lua_pushnil(L);
+	while (lua_next(L, idx) != 0) {
+		// key at -2, value at -1
+		pack_validate(L, absidx(L, -2), seen);
+		pack_validate(L, absidx(L, -1), seen);
+
+		lua_pop(L, 1);
+	}
+
+	seen.erase(ptr);
+}
+
+static VectorRef<PackedInstr> pack_inner(lua_State *L, int idx, int vidx, PackedValue &pv)
+{
+#ifndef NDEBUG
+	StackChecker checker(L);
+	assert(idx > 0);
+	assert(vidx > 0);
+#endif
+
+	switch (lua_type(L, idx)) {
+		case LUA_TNONE:
+		case LUA_TNIL:
+			return emplace(pv, LUA_TNIL);
+		case LUA_TBOOLEAN: {
+			auto r = emplace(pv, LUA_TBOOLEAN);
+			r->bdata = lua_toboolean(L, idx);
+			return r;
+		}
+		case LUA_TNUMBER: {
+			auto r = emplace(pv, LUA_TNUMBER);
+			r->ndata = lua_tonumber(L, idx);
+			return r;
+		}
+		case LUA_TSTRING: {
+			auto r = emplace(pv, LUA_TSTRING);
+			size_t len;
+			const char *str = lua_tolstring(L, idx, &len);
+			assert(str);
+			r->sdata.assign(str, len);
+			return r;
+		}
+		case LUA_TTABLE:
+			break; // execution continues
+		case LUA_TFUNCTION: {
+			auto r = emplace(pv, LUA_TFUNCTION);
+			call_string_dump(L, idx);
+			size_t len;
+			const char *str = lua_tolstring(L, -1, &len);
+			assert(str);
+			r->sdata.assign(str, len);
+			lua_pop(L, 1);
+			return r;
+		}
+		case LUA_TUSERDATA: {
+			PackerTuple ser;
+			if (!find_packer(L, idx, ser))
+				throw LuaError("Cannot serialize unsupported userdata");
+			pv.contains_userdata = true;
+			auto r = emplace(pv, LUA_TUSERDATA);
+			r->sdata = ser.first;
+			r->ptrdata = ser.second.fin(L, idx);
+			return r;
+		}
+		default: {
+			std::string err = "Cannot serialize type ";
+			err += lua_typename(L, lua_type(L, idx));
+			throw LuaError(err);
+		}
+	}
+
+	// LUA_TTABLE
+	lua_checkstack(L, 5);
+
+	auto rtable = emplace(pv, LUA_TTABLE);
+	const int vi_table = vidx++;
+
+	lua_pushnil(L);
+	while (lua_next(L, idx) != 0) {
+		// key at -2, value at -1
+		const int ktype = lua_type(L, -2), vtype = lua_type(L, -1);
+		if (ktype == LUA_TNUMBER)
+			rtable->uidata1++; // narr
+		else
+			rtable->uidata2++; // nrec
+
+		// check if we can use a shortcut
+		if (can_set_into(ktype, vtype) && suitable_key(L, -2)) {
+			// push only the value
+			auto rval = pack_inner(L, absidx(L, -1), vidx, pv);
+			rval->pop = vtype != LUA_TTABLE;
+			// and where to put it:
+			rval->set_into = vi_table;
+			if (ktype == LUA_TSTRING)
+				rval->sdata = lua_tostring(L, -2);
+			else
+				rval->sidata1 = lua_tointeger(L, -2);
+			// pop tables after the fact
+			if (!rval->pop) {
+				auto ri1 = emplace(pv, INSTR_POP);
+				ri1->sidata1 = vidx;
+			}
+		} else {
+			// push the key and value
+			pack_inner(L, absidx(L, -2), vidx, pv);
+			vidx++;
+			pack_inner(L, absidx(L, -1), vidx, pv);
+			vidx++;
+			// push an instruction to set them
+			auto ri1 = emplace(pv, INSTR_SETTABLE);
+			ri1->set_into = vi_table;
+			ri1->sidata1 = vidx - 2;
+			ri1->sidata2 = vidx - 1;
+			ri1->pop = true;
+			vidx -= 2;
+		}
+
+		lua_pop(L, 1);
+	}
+
+	assert(vidx == vi_table + 1);
+	return rtable;
+}
+
+PackedValue *script_pack(lua_State *L, int idx)
+{
+	if (idx < 0)
+		idx = absidx(L, idx);
+
+	std::unordered_set<const void*> seen;
+	pack_validate(L, idx, seen);
+	assert(seen.size() == 0);
+
+	// Actual serialization
+	PackedValue pv;
+	pack_inner(L, idx, 1, pv);
+
+	return new PackedValue(std::move(pv));
+}
+
+//
+// Unpacking implementation
+//
+
+void script_unpack(lua_State *L, PackedValue *pv)
+{
+	const int top = lua_gettop(L);
+	int ctr = 0;
+
+	for (auto &i : pv->i) {
+		// If leaving values on stack make sure there's space (every 5th iteration)
+		if (!i.pop && (ctr++) >= 5) {
+			lua_checkstack(L, 5);
+			ctr = 0;
+		}
+
+		/* Instructions */
+		switch (i.type) {
+			case INSTR_SETTABLE:
+				lua_pushvalue(L, top + i.sidata1); // key
+				lua_pushvalue(L, top + i.sidata2); // value
+				lua_rawset(L, top + i.set_into);
+				if (i.pop) {
+					if (i.sidata1 != i.sidata2) {
+						// removing moves indices so pop higher index first
+						lua_remove(L, top + std::max(i.sidata1, i.sidata2));
+						lua_remove(L, top + std::min(i.sidata1, i.sidata2));
+					} else {
+						lua_remove(L, top + i.sidata1);
+					}
+				}
+				continue;
+			case INSTR_POP:
+				lua_remove(L, top + i.sidata1);
+				if (i.sidata2 > 0)
+					lua_remove(L, top + i.sidata2);
+				continue;
+			default:
+				break;
+		}
+
+		/* Lua types */
+		switch (i.type) {
+			case LUA_TNIL:
+				lua_pushnil(L);
+				break;
+			case LUA_TBOOLEAN:
+				lua_pushboolean(L, i.bdata);
+				break;
+			case LUA_TNUMBER:
+				lua_pushnumber(L, i.ndata);
+				break;
+			case LUA_TSTRING:
+				lua_pushlstring(L, i.sdata.data(), i.sdata.size());
+				break;
+			case LUA_TTABLE:
+				lua_createtable(L, i.uidata1, i.uidata2);
+				break;
+			case LUA_TFUNCTION:
+				luaL_loadbuffer(L, i.sdata.data(), i.sdata.size(), nullptr);
+				break;
+			case LUA_TUSERDATA: {
+				PackerTuple ser;
+				sanity_check(find_packer(i.sdata.c_str(), ser));
+				ser.second.fout(L, i.ptrdata);
+				i.ptrdata = nullptr; // ownership taken by callback
+				break;
+			}
+			default:
+				assert(0);
+				break;
+		}
+
+		if (i.set_into) {
+			if (!i.pop)
+				lua_pushvalue(L, -1);
+			if (uses_sdata(i.type))
+				lua_rawseti(L, top + i.set_into, i.sidata1);
+			else
+				lua_setfield(L, top + i.set_into, i.sdata.c_str());
+		} else {
+			if (i.pop)
+				lua_pop(L, 1);
+		}
+	}
+
+	// as part of the unpacking process we take ownership of all userdata
+	pv->contains_userdata = false;
+	// leave exactly one value on the stack
+	lua_settop(L, top+1);
+}
+
+//
+// PackedValue
+//
+
+PackedValue::~PackedValue()
+{
+	if (!contains_userdata)
+		return;
+	for (auto &i : this->i) {
+		if (i.type == LUA_TUSERDATA && i.ptrdata) {
+			PackerTuple ser;
+			if (find_packer(i.sdata.c_str(), ser)) {
+				// tell it to deallocate object
+				ser.second.fout(nullptr, i.ptrdata);
+			} else {
+				assert(false);
+			}
+		}
+	}
+}
+
+//
+// script_dump_packed
+//
+
+#ifndef NDEBUG
+void script_dump_packed(const PackedValue *val)
+{
+	printf("instruction stream: [\n");
+	for (const auto &i : val->i) {
+		printf("\t(");
+		switch (i.type) {
+			case INSTR_SETTABLE:
+				printf("SETTABLE(%d, %d)", i.sidata1, i.sidata2);
+				break;
+			case INSTR_POP:
+				printf(i.sidata2 ? "POP(%d, %d)" : "POP(%d)", i.sidata1, i.sidata2);
+				break;
+			case LUA_TNIL:
+				printf("nil");
+				break;
+			case LUA_TBOOLEAN:
+				printf(i.bdata ? "true" : "false");
+				break;
+			case LUA_TNUMBER:
+				printf("%f", i.ndata);
+				break;
+			case LUA_TSTRING:
+				printf("\"%s\"", i.sdata.c_str());
+				break;
+			case LUA_TTABLE:
+				printf("table(%d, %d)", i.uidata1, i.uidata2);
+				break;
+			case LUA_TFUNCTION:
+				printf("function(%d byte)", i.sdata.size());
+				break;
+			case LUA_TUSERDATA:
+				printf("userdata %s %p", i.sdata.c_str(), i.ptrdata);
+				break;
+			default:
+				printf("!!UNKNOWN!!");
+				break;
+		}
+		if (i.set_into) {
+			if (i.type >= 0 && uses_sdata(i.type))
+				printf(", k=%d, into=%d", i.sidata1, i.set_into);
+			else if (i.type >= 0)
+				printf(", k=\"%s\", into=%d", i.sdata.c_str(), i.set_into);
+			else
+				printf(", into=%d", i.set_into);
+		}
+		if (i.pop)
+			printf(", pop");
+		printf(")\n");
+	}
+	printf("]\n");
+}
+#endif

src/script/common/c_packer.h

@@ -0,0 +1,123 @@
+/*
+Minetest
+Copyright (C) 2022 sfan5 <sfan5@live.de>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU Lesser General Public License as published by
+the Free Software Foundation; either version 2.1 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+*/
+
+#pragma once
+
+#include <string>
+#include <vector>
+#include "irrlichttypes.h"
+#include "util/basic_macros.h"
+
+extern "C" {
+#include <lua.h>
+}
+
+/*
+	This file defines an in-memory representation of Lua objects including
+	support for functions and userdata. It it used to move data between Lua
+	states and cannot be used for persistence or network transfer.
+*/
+
+#define INSTR_SETTABLE (-10)
+#define INSTR_POP      (-11)
+
+/**
+ * Represents a single instruction that pushes a new value or works with existing ones.
+ */
+struct PackedInstr
+{
+	s16 type; // LUA_T* or INSTR_*
+	u16 set_into; // set into table on stack
+	bool pop; // remove from stack?
+	union {
+		bool bdata; // boolean: value
+		lua_Number ndata; // number: value
+		struct {
+			u16 uidata1, uidata2; // table: narr, nrec
+		};
+		struct {
+			/*
+				SETTABLE: key index, value index
+				POP: indices to remove
+				otherwise w/ set_into: numeric key, -
+			*/
+			s32 sidata1, sidata2;
+		};
+		void *ptrdata; // userdata: implementation defined
+	};
+	/*
+		- string: value
+		- function: buffer
+		- w/ set_into: string key (no null bytes!)
+		- userdata: name in registry
+	*/
+	std::string sdata;
+
+	PackedInstr() : type(0), set_into(0), pop(false) {}
+};
+
+/**
+ * A packed value can be a primitive like a string or number but also a table
+ * including all of its contents. It is made up of a linear stream of
+ * 'instructions' that build the final value when executed.
+ */
+struct PackedValue
+{
+	std::vector<PackedInstr> i;
+	// Indicates whether there are any userdata pointers that need to be deallocated
+	bool contains_userdata = false;
+
+	PackedValue() = default;
+	~PackedValue();
+
+	DISABLE_CLASS_COPY(PackedValue)
+
+	ALLOW_CLASS_MOVE(PackedValue)
+};
+
+/*
+ * Packing callback: Turns a Lua value at given index into a void*
+ */
+typedef void *(*PackInFunc)(lua_State *L, int idx);
+/*
+ * Unpacking callback: Turns a void* back into the Lua value (left on top of stack)
+ *
+ * Note that this function must take ownership of the pointer, so make sure
+ * to free or keep the memory.
+ * `L` can be nullptr to indicate that data should just be discarded.
+ */
+typedef void (*PackOutFunc)(lua_State *L, void *ptr);
+/*
+ * Register a packable type with the name of its metatable.
+ *
+ * Even though the callbacks are global this must be called for every Lua state
+ * that supports objects of this type.
+ * This function is thread-safe.
+ */
+void script_register_packer(lua_State *L, const char *regname,
+		PackInFunc fin, PackOutFunc fout);
+
+// Pack a Lua value
+PackedValue *script_pack(lua_State *L, int idx);
+// Unpack a Lua value (left on top of stack)
+// Note that this may modify the PackedValue, you can't reuse it!
+void script_unpack(lua_State *L, PackedValue *val);
+
+// Dump contents of PackedValue to stdout for debugging
+void script_dump_packed(const PackedValue *val);

src/script/cpp_api/s_async.cpp

@@ -21,9 +21,9 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include <cstdlib>
 
 extern "C" {
-#include "lua.h"
-#include "lauxlib.h"
-#include "lualib.h"
+#include <lua.h>
+#include <lauxlib.h>
+#include <lualib.h>
 }
 
 #include "server.h"
@@ -32,6 +32,7 @@ extern "C" {
 #include "filesys.h"
 #include "porting.h"
 #include "common/c_internal.h"
+#include "common/c_packer.h"
 #include "lua_api/l_base.h"
 
 /******************************************************************************/
@@ -76,19 +77,34 @@ void AsyncEngine::initialize(unsigned int numEngines)
 {
 	initDone = true;
 
-	for (unsigned int i = 0; i < numEngines; i++) {
-		AsyncWorkerThread *toAdd = new AsyncWorkerThread(this,
-			std::string("AsyncWorker-") + itos(i));
-		workerThreads.push_back(toAdd);
-		toAdd->start();
+	if (numEngines == 0) {
+		// Leave one core for the main thread and one for whatever else
+		autoscaleMaxWorkers = Thread::getNumberOfProcessors();
+		if (autoscaleMaxWorkers >= 2)
+			autoscaleMaxWorkers -= 2;
+		infostream << "AsyncEngine: using at most " << autoscaleMaxWorkers
+			<< " threads with automatic scaling" << std::endl;
+
+		addWorkerThread();
+	} else {
+		for (unsigned int i = 0; i < numEngines; i++)
+			addWorkerThread();
 	}
 }
 
+void AsyncEngine::addWorkerThread()
+{
+	AsyncWorkerThread *toAdd = new AsyncWorkerThread(this,
+		std::string("AsyncWorker-") + itos(workerThreads.size()));
+	workerThreads.push_back(toAdd);
+	toAdd->start();
+}
+
 /******************************************************************************/
 u32 AsyncEngine::queueAsyncJob(std::string &&func, std::string &&params,
 		const std::string &mod_origin)
 {
-	jobQueueMutex.lock();
+	MutexAutoLock autolock(jobQueueMutex);
 	u32 jobId = jobIdCounter++;
 
 	jobQueue.emplace_back();
@@ -99,7 +115,23 @@ u32 AsyncEngine::queueAsyncJob(std::string &&func, std::string &&params,
 	to_add.mod_origin = mod_origin;
 
 	jobQueueCounter.post();
-	jobQueueMutex.unlock();
+	return jobId;
+}
+
+u32 AsyncEngine::queueAsyncJob(std::string &&func, PackedValue *params,
+		const std::string &mod_origin)
+{
+	MutexAutoLock autolock(jobQueueMutex);
+	u32 jobId = jobIdCounter++;
+
+	jobQueue.emplace_back();
+	auto &to_add = jobQueue.back();
+	to_add.id = jobId;
+	to_add.function = std::move(func);
+	to_add.params_ext.reset(params);
+	to_add.mod_origin = mod_origin;
+
+	jobQueueCounter.post();
 	return jobId;
 }
 
@@ -131,6 +163,12 @@ void AsyncEngine::putJobResult(LuaJobInfo &&result)
 
 /******************************************************************************/
 void AsyncEngine::step(lua_State *L)
+{
+	stepJobResults(L);
+	stepAutoscale();
+}
+
+void AsyncEngine::stepJobResults(lua_State *L)
 {
 	int error_handler = PUSH_ERROR_HANDLER(L);
 	lua_getglobal(L, "core");
@@ -148,7 +186,10 @@ void AsyncEngine::step(lua_State *L)
 		luaL_checktype(L, -1, LUA_TFUNCTION);
 
 		lua_pushinteger(L, j.id);
-		lua_pushlstring(L, j.result.data(), j.result.size());
+		if (j.result_ext)
+			script_unpack(L, j.result_ext.get());
+		else
+			lua_pushlstring(L, j.result.data(), j.result.size());
 
 		// Call handler
 		const char *origin = j.mod_origin.empty() ? nullptr : j.mod_origin.c_str();
@@ -161,12 +202,71 @@ void AsyncEngine::step(lua_State *L)
 	lua_pop(L, 2); // Pop core and error handler
 }
 
+void AsyncEngine::stepAutoscale()
+{
+	if (workerThreads.size() >= autoscaleMaxWorkers)
+		return;
+
+	MutexAutoLock autolock(jobQueueMutex);
+
+	// 2) If the timer elapsed, check again
+	if (autoscaleTimer && porting::getTimeMs() >= autoscaleTimer) {
+		autoscaleTimer = 0;
+		// Determine overlap with previous snapshot
+		unsigned int n = 0;
+		for (const auto &it : jobQueue)
+			n += autoscaleSeenJobs.count(it.id);
+		autoscaleSeenJobs.clear();
+		infostream << "AsyncEngine: " << n << " jobs were still waiting after 1s" << std::endl;
+		// Start this many new threads
+		while (workerThreads.size() < autoscaleMaxWorkers && n > 0) {
+			addWorkerThread();
+			n--;
+		}
+		return;
+	}
+
+	// 1) Check if there's anything in the queue
+	if (!autoscaleTimer && !jobQueue.empty()) {
+		// Take a snapshot of all jobs we have seen
+		for (const auto &it : jobQueue)
+			autoscaleSeenJobs.emplace(it.id);
+		// and set a timer for 1 second
+		autoscaleTimer = porting::getTimeMs() + 1000;
+	}
+}
+
 /******************************************************************************/
-void AsyncEngine::prepareEnvironment(lua_State* L, int top)
+bool AsyncEngine::prepareEnvironment(lua_State* L, int top)
 {
 	for (StateInitializer &stateInitializer : stateInitializers) {
 		stateInitializer(L, top);
 	}
+
+	auto *script = ModApiBase::getScriptApiBase(L);
+	try {
+		script->loadMod(Server::getBuiltinLuaPath() + DIR_DELIM + "init.lua",
+			BUILTIN_MOD_NAME);
+	} catch (const ModError &e) {
+		errorstream << "Execution of async base environment failed: "
+			<< e.what() << std::endl;
+		FATAL_ERROR("Execution of async base environment failed");
+	}
+
+	// Load per mod stuff
+	if (server) {
+		const auto &list = server->m_async_init_files;
+		try {
+			for (auto &it : list)
+				script->loadMod(it.second, it.first);
+		} catch (const ModError &e) {
+			errorstream << "Failed to load mod script inside async environment." << std::endl;
+			server->setAsyncFatalError(e.what());
+			return false;
+		}
+	}
+
+	return true;
 }
 
 /******************************************************************************/
@@ -178,15 +278,25 @@ AsyncWorkerThread::AsyncWorkerThread(AsyncEngine* jobDispatcher,
 {
 	lua_State *L = getStack();
 
+	if (jobDispatcher->server) {
+		setGameDef(jobDispatcher->server);
+
+		if (g_settings->getBool("secure.enable_security"))
+			initializeSecurity();
+	}
+
 	// Prepare job lua environment
 	lua_getglobal(L, "core");
 	int top = lua_gettop(L);
 
 	// Push builtin initialization type
-	lua_pushstring(L, "async");
+	lua_pushstring(L, jobDispatcher->server ? "async_game" : "async");
 	lua_setglobal(L, "INIT");
 
-	jobDispatcher->prepareEnvironment(L, top);
+	if (!jobDispatcher->prepareEnvironment(L, top)) {
+		// can't throw from here so we're stuck with this
+		isErrored = true;
+	}
 }
 
 /******************************************************************************/
@@ -198,19 +308,20 @@ AsyncWorkerThread::~AsyncWorkerThread()
 /******************************************************************************/
 void* AsyncWorkerThread::run()
 {
+	if (isErrored)
+		return nullptr;
+
 	lua_State *L = getStack();
 
-	try {
-		loadMod(getServer()->getBuiltinLuaPath() + DIR_DELIM + "init.lua",
-			BUILTIN_MOD_NAME);
-	} catch (const ModError &e) {
-		errorstream << "Execution of async base environment failed: "
-			<< e.what() << std::endl;
-		FATAL_ERROR("Execution of async base environment failed");
-	}
-
 	int error_handler = PUSH_ERROR_HANDLER(L);
 
+	auto report_error = [this] (const ModError &e) {
+		if (jobDispatcher->server)
+			jobDispatcher->server->setAsyncFatalError(e.what());
+		else
+			errorstream << e.what() << std::endl;
+	};
+
 	lua_getglobal(L, "core");
 	if (lua_isnil(L, -1)) {
 		FATAL_ERROR("Unable to find core within async environment!");
@@ -223,6 +334,8 @@ void* AsyncWorkerThread::run()
 		if (!jobDispatcher->getJob(&j) || stopRequested())
 			continue;
 
+		const bool use_ext = !!j.params_ext;
+
 		lua_getfield(L, -1, "job_processor");
 		if (lua_isnil(L, -1))
 			FATAL_ERROR("Unable to get async job processor!");
@@ -232,7 +345,10 @@ void* AsyncWorkerThread::run()
 			errorstream << "ASYNC WORKER: Unable to deserialize function" << std::endl;
 			lua_pushnil(L);
 		}
-		lua_pushlstring(L, j.params.data(), j.params.size());
+		if (use_ext)
+			script_unpack(L, j.params_ext.get());
+		else
+			lua_pushlstring(L, j.params.data(), j.params.size());
 
 		// Call it
 		setOriginDirect(j.mod_origin.empty() ? nullptr : j.mod_origin.c_str());
@@ -241,19 +357,28 @@ void* AsyncWorkerThread::run()
 			try {
 				scriptError(result, "<async>");
 			} catch (const ModError &e) {
-				errorstream << e.what() << std::endl;
+				report_error(e);
 			}
 		} else {
 			// Fetch result
-			size_t length;
-			const char *retval = lua_tolstring(L, -1, &length);
-			j.result.assign(retval, length);
+			if (use_ext) {
+				try {
+					j.result_ext.reset(script_pack(L, -1));
+				} catch (const ModError &e) {
+					report_error(e);
+					result = LUA_ERRERR;
+				}
+			} else {
+				size_t length;
+				const char *retval = lua_tolstring(L, -1, &length);
+				j.result.assign(retval, length);
+			}
 		}
 
 		lua_pop(L, 1);  // Pop retval
 
 		// Put job result
-		if (!j.result.empty())
+		if (result == 0)
 			jobDispatcher->putJobResult(std::move(j));
 	}
 

src/script/cpp_api/s_async.h

@@ -21,11 +21,15 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 
 #include <vector>
 #include <deque>
+#include <unordered_set>
+#include <memory>
 
+#include <lua.h>
 #include "threading/semaphore.h"
 #include "threading/thread.h"
-#include "lua.h"
+#include "common/c_packer.h"
 #include "cpp_api/s_base.h"
+#include "cpp_api/s_security.h"
 
 // Forward declarations
 class AsyncEngine;
@@ -42,8 +46,12 @@ struct LuaJobInfo
 	std::string function;
 	// Parameter to be passed to function (serialized)
 	std::string params;
+	// Alternative parameters
+	std::unique_ptr<PackedValue> params_ext;
 	// Result of function call (serialized)
 	std::string result;
+	// Alternative result
+	std::unique_ptr<PackedValue> result_ext;
 	// Name of the mod who invoked this call
 	std::string mod_origin;
 	// JobID used to identify a job and match it to callback
@@ -51,7 +59,8 @@ struct LuaJobInfo
 };
 
 // Asynchronous working environment
-class AsyncWorkerThread : public Thread, virtual public ScriptApiBase {
+class AsyncWorkerThread : public Thread,
+	virtual public ScriptApiBase, public ScriptApiSecurity {
 	friend class AsyncEngine;
 public:
 	virtual ~AsyncWorkerThread();
@@ -63,6 +72,7 @@ protected:
 
 private:
 	AsyncEngine *jobDispatcher = nullptr;
+	bool isErrored = false;
 };
 
 // Asynchornous thread and job management
@@ -71,6 +81,7 @@ class AsyncEngine {
 	typedef void (*StateInitializer)(lua_State *L, int top);
 public:
 	AsyncEngine() = default;
+	AsyncEngine(Server *server) : server(server) {};
 	~AsyncEngine();
 
 	/**
@@ -81,7 +92,7 @@ public:
 
 	/**
 	 * Create async engine tasks and lock function registration
-	 * @param numEngines Number of async threads to be started
+	 * @param numEngines Number of worker threads, 0 for automatic scaling
 	 */
 	void initialize(unsigned int numEngines);
 
@@ -94,9 +105,17 @@ public:
 	u32 queueAsyncJob(std::string &&func, std::string &&params,
 			const std::string &mod_origin = "");
 
+	/**
+	 * Queue an async job
+	 * @param func Serialized lua function
+	 * @param params Serialized parameters (takes ownership!)
+	 * @return ID of queued job
+	 */
+	u32 queueAsyncJob(std::string &&func, PackedValue *params,
+			const std::string &mod_origin = "");
+
 	/**
 	 * Engine step to process finished jobs
-	 *   the engine step is one way to pass events back, PushFinishedJobs another
 	 * @param L The Lua stack
 	 */
 	void step(lua_State *L);
@@ -116,19 +135,44 @@ protected:
 	 */
 	void putJobResult(LuaJobInfo &&result);
 
+	/**
+	 * Start an additional worker thread
+	 */
+	void addWorkerThread();
+
+	/**
+	 * Process finished jobs callbacks
+	 */
+	void stepJobResults(lua_State *L);
+
+	/**
+	 * Handle automatic scaling of worker threads
+	 */
+	void stepAutoscale();
+
 	/**
 	 * Initialize environment with current registred functions
 	 *  this function adds all functions registred by registerFunction to the
 	 *  passed lua stack
 	 * @param L Lua stack to initialize
 	 * @param top Stack position
+	 * @return false if a mod error ocurred
 	 */
-	void prepareEnvironment(lua_State* L, int top);
+	bool prepareEnvironment(lua_State* L, int top);
 
 private:
 	// Variable locking the engine against further modification
 	bool initDone = false;
 
+	// Maximum number of worker threads for automatic scaling
+	// 0 if disabled
+	unsigned int autoscaleMaxWorkers = 0;
+	u64 autoscaleTimer = 0;
+	std::unordered_set<u32> autoscaleSeenJobs;
+
+	// Only set for the server async environment (duh)
+	Server *server = nullptr;
+
 	// Internal store for registred state initializers
 	std::vector<StateInitializer> stateInitializers;
 

src/script/lua_api/l_craft.cpp

@@ -525,3 +525,11 @@ void ModApiCraft::Initialize(lua_State *L, int top)
 	API_FCT(register_craft);
 	API_FCT(clear_craft);
 }
+
+void ModApiCraft::InitializeAsync(lua_State *L, int top)
+{
+	// all read-only functions
+	API_FCT(get_all_craft_recipes);
+	API_FCT(get_craft_recipe);
+	API_FCT(get_craft_result);
+}

src/script/lua_api/l_craft.h

@@ -45,4 +45,5 @@ private:
 
 public:
 	static void Initialize(lua_State *L, int top);
+	static void InitializeAsync(lua_State *L, int top);
 };

src/script/lua_api/l_internal.h

@@ -69,7 +69,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 
 // Retrieve Environment pointer as `env` (no map lock)
 #define GET_PLAIN_ENV_PTR_NO_MAP_LOCK            \
-	Environment *env = (Environment *)getEnv(L); \
+	Environment *env = getEnv(L);                \
 	if (env == NULL)                             \
 		return 0
 

src/script/lua_api/l_item.cpp

@@ -22,6 +22,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "lua_api/l_internal.h"
 #include "common/c_converter.h"
 #include "common/c_content.h"
+#include "common/c_packer.h"
 #include "itemdef.h"
 #include "nodedef.h"
 #include "server.h"
@@ -441,6 +442,7 @@ int LuaItemStack::create_object(lua_State *L)
 	lua_setmetatable(L, -2);
 	return 1;
 }
+
 // Not callable from Lua
 int LuaItemStack::create(lua_State *L, const ItemStack &item)
 {
@@ -457,6 +459,20 @@ LuaItemStack *LuaItemStack::checkobject(lua_State *L, int narg)
 	return *(LuaItemStack **)luaL_checkudata(L, narg, className);
 }
 
+void *LuaItemStack::packIn(lua_State *L, int idx)
+{
+	LuaItemStack *o = checkobject(L, idx);
+	return new ItemStack(o->getItem());
+}
+
+void LuaItemStack::packOut(lua_State *L, void *ptr)
+{
+	ItemStack *stack = reinterpret_cast<ItemStack*>(ptr);
+	if (L)
+		create(L, *stack);
+	delete stack;
+}
+
 void LuaItemStack::Register(lua_State *L)
 {
 	lua_newtable(L);
@@ -488,6 +504,8 @@ void LuaItemStack::Register(lua_State *L)
 
 	// Can be created from Lua (ItemStack(itemstack or itemstring or table or nil))
 	lua_register(L, className, create_object);
+
+	script_register_packer(L, className, packIn, packOut);
 }
 
 const char LuaItemStack::className[] = "ItemStack";
@@ -673,3 +691,10 @@ void ModApiItemMod::Initialize(lua_State *L, int top)
 	API_FCT(get_content_id);
 	API_FCT(get_name_from_content_id);
 }
+
+void ModApiItemMod::InitializeAsync(lua_State *L, int top)
+{
+	// all read-only functions
+	API_FCT(get_content_id);
+	API_FCT(get_name_from_content_id);
+}

src/script/lua_api/l_item.h

@@ -141,8 +141,11 @@ public:
 	// Not callable from Lua
 	static int create(lua_State *L, const ItemStack &item);
 	static LuaItemStack* checkobject(lua_State *L, int narg);
-	static void Register(lua_State *L);
 
+	static void *packIn(lua_State *L, int idx);
+	static void packOut(lua_State *L, void *ptr);
+
+	static void Register(lua_State *L);
 };
 
 class ModApiItemMod : public ModApiBase {
@@ -152,6 +155,8 @@ private:
 	static int l_register_alias_raw(lua_State *L);
 	static int l_get_content_id(lua_State *L);
 	static int l_get_name_from_content_id(lua_State *L);
+
 public:
 	static void Initialize(lua_State *L, int top);
+	static void InitializeAsync(lua_State *L, int top);
 };

src/script/lua_api/l_noise.cpp

@@ -21,6 +21,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "lua_api/l_internal.h"
 #include "common/c_converter.h"
 #include "common/c_content.h"
+#include "common/c_packer.h"
 #include "log.h"
 #include "porting.h"
 #include "util/numeric.h"
@@ -101,6 +102,25 @@ LuaPerlinNoise *LuaPerlinNoise::checkobject(lua_State *L, int narg)
 }
 
 
+void *LuaPerlinNoise::packIn(lua_State *L, int idx)
+{
+	LuaPerlinNoise *o = checkobject(L, idx);
+	return new NoiseParams(o->np);
+}
+
+void LuaPerlinNoise::packOut(lua_State *L, void *ptr)
+{
+	NoiseParams *np = reinterpret_cast<NoiseParams*>(ptr);
+	if (L) {
+		LuaPerlinNoise *o = new LuaPerlinNoise(np);
+		*(void **)(lua_newuserdata(L, sizeof(void *))) = o;
+		luaL_getmetatable(L, className);
+		lua_setmetatable(L, -2);
+	}
+	delete np;
+}
+
+
 void LuaPerlinNoise::Register(lua_State *L)
 {
 	lua_newtable(L);
@@ -126,6 +146,8 @@ void LuaPerlinNoise::Register(lua_State *L)
 	lua_pop(L, 1);
 
 	lua_register(L, className, create_object);
+
+	script_register_packer(L, className, packIn, packOut);
 }
 
 
@@ -357,6 +379,35 @@ LuaPerlinNoiseMap *LuaPerlinNoiseMap::checkobject(lua_State *L, int narg)
 }
 
 
+struct NoiseMapParams {
+	NoiseParams np;
+	s32 seed;
+	v3s16 size;
+};
+
+void *LuaPerlinNoiseMap::packIn(lua_State *L, int idx)
+{
+	LuaPerlinNoiseMap *o = checkobject(L, idx);
+	NoiseMapParams *ret = new NoiseMapParams();
+	ret->np = o->noise->np;
+	ret->seed = o->noise->seed;
+	ret->size = v3s16(o->noise->sx, o->noise->sy, o->noise->sz);
+	return ret;
+}
+
+void LuaPerlinNoiseMap::packOut(lua_State *L, void *ptr)
+{
+	NoiseMapParams *p = reinterpret_cast<NoiseMapParams*>(ptr);
+	if (L) {
+		LuaPerlinNoiseMap *o = new LuaPerlinNoiseMap(&p->np, p->seed, p->size);
+		*(void **)(lua_newuserdata(L, sizeof(void *))) = o;
+		luaL_getmetatable(L, className);
+		lua_setmetatable(L, -2);
+	}
+	delete p;
+}
+
+
 void LuaPerlinNoiseMap::Register(lua_State *L)
 {
 	lua_newtable(L);
@@ -382,6 +433,8 @@ void LuaPerlinNoiseMap::Register(lua_State *L)
 	lua_pop(L, 1);
 
 	lua_register(L, className, create_object);
+
+	script_register_packer(L, className, packIn, packOut);
 }
 
 

src/script/lua_api/l_noise.h

@@ -52,6 +52,9 @@ public:
 
 	static LuaPerlinNoise *checkobject(lua_State *L, int narg);
 
+	static void *packIn(lua_State *L, int idx);
+	static void packOut(lua_State *L, void *ptr);
+
 	static void Register(lua_State *L);
 };
 
@@ -91,6 +94,9 @@ public:
 
 	static LuaPerlinNoiseMap *checkobject(lua_State *L, int narg);
 
+	static void *packIn(lua_State *L, int idx);
+	static void packOut(lua_State *L, void *ptr);
+
 	static void Register(lua_State *L);
 };
 

src/script/lua_api/l_server.cpp

@@ -21,6 +21,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "lua_api/l_internal.h"
 #include "common/c_converter.h"
 #include "common/c_content.h"
+#include "common/c_packer.h"
 #include "cpp_api/s_base.h"
 #include "cpp_api/s_security.h"
 #include "scripting_server.h"
@@ -526,6 +527,76 @@ int ModApiServer::l_notify_authentication_modified(lua_State *L)
 	return 0;
 }
 
+// do_async_callback(func, params, mod_origin)
+int ModApiServer::l_do_async_callback(lua_State *L)
+{
+	NO_MAP_LOCK_REQUIRED;
+	ServerScripting *script = getScriptApi<ServerScripting>(L);
+
+	luaL_checktype(L, 1, LUA_TFUNCTION);
+	luaL_checktype(L, 2, LUA_TTABLE);
+	luaL_checktype(L, 3, LUA_TSTRING);
+
+	call_string_dump(L, 1);
+	size_t func_length;
+	const char *serialized_func_raw = lua_tolstring(L, -1, &func_length);
+
+	PackedValue *param = script_pack(L, 2);
+
+	std::string mod_origin = readParam<std::string>(L, 3);
+
+	u32 jobId = script->queueAsync(
+		std::string(serialized_func_raw, func_length),
+		param, mod_origin);
+
+	lua_settop(L, 0);
+	lua_pushinteger(L, jobId);
+	return 1;
+}
+
+// register_async_dofile(path)
+int ModApiServer::l_register_async_dofile(lua_State *L)
+{
+	NO_MAP_LOCK_REQUIRED;
+
+	std::string path = readParam<std::string>(L, 1);
+	CHECK_SECURE_PATH(L, path.c_str(), false);
+
+	// Find currently running mod name (only at init time)
+	lua_rawgeti(L, LUA_REGISTRYINDEX, CUSTOM_RIDX_CURRENT_MOD_NAME);
+	if (!lua_isstring(L, -1))
+		return 0;
+	std::string modname = readParam<std::string>(L, -1);
+
+	getServer(L)->m_async_init_files.emplace_back(modname, path);
+	lua_pushboolean(L, true);
+	return 1;
+}
+
+// serialize_roundtrip(value)
+// Meant for unit testing the packer from Lua
+int ModApiServer::l_serialize_roundtrip(lua_State *L)
+{
+	NO_MAP_LOCK_REQUIRED;
+
+	int top = lua_gettop(L);
+	auto *pv = script_pack(L, 1);
+	if (top != lua_gettop(L))
+		throw LuaError("stack values leaked");
+
+#ifndef NDEBUG
+	script_dump_packed(pv);
+#endif
+
+	top = lua_gettop(L);
+	script_unpack(L, pv);
+	delete pv;
+	if (top + 1 != lua_gettop(L))
+		throw LuaError("stack values leaked");
+
+	return 1;
+}
+
 void ModApiServer::Initialize(lua_State *L, int top)
 {
 	API_FCT(request_shutdown);
@@ -559,4 +630,18 @@ void ModApiServer::Initialize(lua_State *L, int top)
 	API_FCT(remove_player);
 	API_FCT(unban_player_or_ip);
 	API_FCT(notify_authentication_modified);
+
+	API_FCT(do_async_callback);
+	API_FCT(register_async_dofile);
+	API_FCT(serialize_roundtrip);
+}
+
+void ModApiServer::InitializeAsync(lua_State *L, int top)
+{
+	API_FCT(get_worldpath);
+	API_FCT(is_singleplayer);
+
+	API_FCT(get_current_modname);
+	API_FCT(get_modpath);
+	API_FCT(get_modnames);
 }

src/script/lua_api/l_server.h

@@ -106,6 +106,16 @@ private:
 	// notify_authentication_modified(name)
 	static int l_notify_authentication_modified(lua_State *L);
 
+	// do_async_callback(func, params, mod_origin)
+	static int l_do_async_callback(lua_State *L);
+
+	// register_async_dofile(path)
+	static int l_register_async_dofile(lua_State *L);
+
+	// serialize_roundtrip(obj)
+	static int l_serialize_roundtrip(lua_State *L);
+
 public:
 	static void Initialize(lua_State *L, int top);
+	static void InitializeAsync(lua_State *L, int top);
 };

src/script/lua_api/l_util.cpp

@@ -671,6 +671,9 @@ void ModApiUtil::InitializeAsync(lua_State *L, int top)
 	API_FCT(cpdir);
 	API_FCT(mvdir);
 	API_FCT(get_dir_list);
+	API_FCT(safe_file_write);
+
+	API_FCT(request_insecure_environment);
 
 	API_FCT(encode_base64);
 	API_FCT(decode_base64);
@@ -680,6 +683,8 @@ void ModApiUtil::InitializeAsync(lua_State *L, int top)
 	API_FCT(colorspec_to_colorstring);
 	API_FCT(colorspec_to_bytes);
 
+	API_FCT(encode_png);
+
 	API_FCT(get_last_run_mod);
 	API_FCT(set_last_run_mod);
 

src/script/lua_api/l_util.h

@@ -129,6 +129,4 @@ public:
 	static void Initialize(lua_State *L, int top);
 	static void InitializeAsync(lua_State *L, int top);
 	static void InitializeClient(lua_State *L, int top);
-
-	static void InitializeAsync(AsyncEngine &engine);
 };

src/script/lua_api/l_vmanip.cpp

@@ -22,6 +22,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "lua_api/l_internal.h"
 #include "common/c_content.h"
 #include "common/c_converter.h"
+#include "common/c_packer.h"
 #include "emerge.h"
 #include "environment.h"
 #include "map.h"
@@ -45,6 +46,8 @@ int LuaVoxelManip::l_read_from_map(lua_State *L)
 
 	LuaVoxelManip *o = checkobject(L, 1);
 	MMVManip *vm = o->vm;
+	if (vm->isOrphan())
+		return 0;
 
 	v3s16 bp1 = getNodeBlockPos(check_v3s16(L, 2));
 	v3s16 bp2 = getNodeBlockPos(check_v3s16(L, 3));
@@ -429,6 +432,34 @@ LuaVoxelManip *LuaVoxelManip::checkobject(lua_State *L, int narg)
 	return *(LuaVoxelManip **)ud;  // unbox pointer
 }
 
+void *LuaVoxelManip::packIn(lua_State *L, int idx)
+{
+	LuaVoxelManip *o = checkobject(L, idx);
+
+	if (o->is_mapgen_vm)
+		throw LuaError("nope");
+	return o->vm->clone();
+}
+
+void LuaVoxelManip::packOut(lua_State *L, void *ptr)
+{
+	MMVManip *vm = reinterpret_cast<MMVManip*>(ptr);
+	if (!L) {
+		delete vm;
+		return;
+	}
+
+	// Associate vmanip with map if the Lua env has one
+	Environment *env = getEnv(L);
+	if (env)
+		vm->reparent(&(env->getMap()));
+
+	LuaVoxelManip *o = new LuaVoxelManip(vm, false);
+	*(void **)(lua_newuserdata(L, sizeof(void *))) = o;
+	luaL_getmetatable(L, className);
+	lua_setmetatable(L, -2);
+}
+
 void LuaVoxelManip::Register(lua_State *L)
 {
 	lua_newtable(L);
@@ -455,6 +486,8 @@ void LuaVoxelManip::Register(lua_State *L)
 
 	// Can be created from Lua (VoxelManip())
 	lua_register(L, className, create_object);
+
+	script_register_packer(L, className, packIn, packOut);
 }
 
 const char LuaVoxelManip::className[] = "VoxelManip";

src/script/lua_api/l_vmanip.h

@@ -75,5 +75,8 @@ public:
 
 	static LuaVoxelManip *checkobject(lua_State *L, int narg);
 
+	static void *packIn(lua_State *L, int idx);
+	static void packOut(lua_State *L, void *ptr);
+
 	static void Register(lua_State *L);
 };

src/script/scripting_server.cpp

@@ -47,11 +47,12 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "lua_api/l_storage.h"
 
 extern "C" {
-#include "lualib.h"
+#include <lualib.h>
 }
 
 ServerScripting::ServerScripting(Server* server):
-		ScriptApiBase(ScriptingType::Server)
+		ScriptApiBase(ScriptingType::Server),
+		asyncEngine(server)
 {
 	setGameDef(server);
 
@@ -88,6 +89,47 @@ ServerScripting::ServerScripting(Server* server):
 	infostream << "SCRIPTAPI: Initialized game modules" << std::endl;
 }
 
+void ServerScripting::initAsync()
+{
+	// Save globals to transfer
+	{
+		lua_State *L = getStack();
+		lua_getglobal(L, "core");
+		luaL_checktype(L, -1, LUA_TTABLE);
+		lua_getfield(L, -1, "get_globals_to_transfer");
+		lua_call(L, 0, 1);
+		luaL_checktype(L, -1, LUA_TSTRING);
+		getServer()->m_async_globals_data.set(readParam<std::string>(L, -1));
+		lua_pushnil(L);
+		lua_setfield(L, -3, "get_globals_to_transfer"); // unset function too
+		lua_pop(L, 2); // pop 'core', return value
+	}
+
+	infostream << "SCRIPTAPI: Initializing async engine" << std::endl;
+	asyncEngine.registerStateInitializer(InitializeAsync);
+	asyncEngine.registerStateInitializer(ModApiUtil::InitializeAsync);
+	asyncEngine.registerStateInitializer(ModApiCraft::InitializeAsync);
+	asyncEngine.registerStateInitializer(ModApiItemMod::InitializeAsync);
+	asyncEngine.registerStateInitializer(ModApiServer::InitializeAsync);
+	// not added: ModApiMapgen is a minefield for thread safety
+	// not added: ModApiHttp async api can't really work together with our jobs
+	// not added: ModApiStorage is probably not thread safe(?)
+
+	asyncEngine.initialize(0);
+}
+
+void ServerScripting::stepAsync()
+{
+	asyncEngine.step(getStack());
+}
+
+u32 ServerScripting::queueAsync(std::string &&serialized_func,
+	PackedValue *param, const std::string &mod_origin)
+{
+	return asyncEngine.queueAsyncJob(std::move(serialized_func),
+			param, mod_origin);
+}
+
 void ServerScripting::InitializeModApi(lua_State *L, int top)
 {
 	// Register reference classes (userdata)
@@ -125,3 +167,24 @@ void ServerScripting::InitializeModApi(lua_State *L, int top)
 	ModApiStorage::Initialize(L, top);
 	ModApiChannels::Initialize(L, top);
 }
+
+void ServerScripting::InitializeAsync(lua_State *L, int top)
+{
+	// classes
+	LuaItemStack::Register(L);
+	LuaPerlinNoise::Register(L);
+	LuaPerlinNoiseMap::Register(L);
+	LuaPseudoRandom::Register(L);
+	LuaPcgRandom::Register(L);
+	LuaSecureRandom::Register(L);
+	LuaVoxelManip::Register(L);
+	LuaSettings::Register(L);
+
+	// globals data
+	lua_getglobal(L, "core");
+	luaL_checktype(L, -1, LUA_TTABLE);
+	std::string s = ModApiBase::getServer(L)->m_async_globals_data.get();
+	lua_pushlstring(L, s.c_str(), s.size());
+	lua_setfield(L, -2, "transferred_globals");
+	lua_pop(L, 1); // pop 'core'
+}

src/script/scripting_server.h

@@ -27,6 +27,9 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "cpp_api/s_player.h"
 #include "cpp_api/s_server.h"
 #include "cpp_api/s_security.h"
+#include "cpp_api/s_async.h"
+
+struct PackedValue;
 
 /*****************************************************************************/
 /* Scripting <-> Server Game Interface                                       */
@@ -48,6 +51,20 @@ public:
 
 	// use ScriptApiBase::loadMod() to load mods
 
+	// Initialize async engine, call this AFTER loading all mods
+	void initAsync();
+
+	// Global step handler to collect async results
+	void stepAsync();
+
+	// Pass job to async threads
+	u32 queueAsync(std::string &&serialized_func,
+		PackedValue *param, const std::string &mod_origin);
+
 private:
 	void InitializeModApi(lua_State *L, int top);
+
+	static void InitializeAsync(lua_State *L, int top);
+
+	AsyncEngine asyncEngine;
 };

src/server.cpp

@@ -243,6 +243,7 @@ Server::Server(
 	m_clients(m_con),
 	m_admin_chat(iface),
 	m_on_shutdown_errmsg(on_shutdown_errmsg),
+	m_async_globals_data(""),
 	m_modchannel_mgr(new ModChannelMgr())
 {
 	if (m_path_world.empty())
@@ -480,6 +481,9 @@ void Server::init()
 	// Give environment reference to scripting api
 	m_script->initializeEnvironment(m_env);
 
+	// Do this after regular script init is done
+	m_script->initAsync();
+
 	// Register us to receive map edit events
 	servermap->addEventReceiver(this);
 

src/server.h

@@ -292,7 +292,7 @@ public:
 
 	virtual const std::vector<ModSpec> &getMods() const;
 	virtual const ModSpec* getModSpec(const std::string &modname) const;
-	std::string getBuiltinLuaPath();
+	static std::string getBuiltinLuaPath();
 	virtual std::string getWorldPath() const { return m_path_world; }
 
 	inline bool isSingleplayer() const
@@ -385,6 +385,12 @@ public:
 	static bool migrateModStorageDatabase(const GameParams &game_params,
 			const Settings &cmd_args);
 
+	// Lua files registered for init of async env, pair of modname + path
+	std::vector<std::pair<std::string, std::string>> m_async_init_files;
+
+	// Serialized data transferred into async envs at init time
+	MutexedVariable<std::string> m_async_globals_data;
+
 	// Bind address
 	Address m_bind_addr;
 

src/serverenvironment.cpp

@@ -1481,6 +1481,8 @@ void ServerEnvironment::step(float dtime)
 	*/
 	m_script->environment_Step(dtime);
 
+	m_script->stepAsync();
+
 	/*
 		Step active objects
 	*/

src/util/basic_macros.h

@@ -29,13 +29,19 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #define CONTAINS(c, v) (std::find((c).begin(), (c).end(), (v)) != (c).end())
 
 // To disable copy constructors and assignment operations for some class
-// 'Foobar', add the macro DISABLE_CLASS_COPY(Foobar) as a private member.
+// 'Foobar', add the macro DISABLE_CLASS_COPY(Foobar) in the class definition.
 // Note this also disables copying for any classes derived from 'Foobar' as well
 // as classes having a 'Foobar' member.
 #define DISABLE_CLASS_COPY(C)        \
 	C(const C &) = delete;           \
 	C &operator=(const C &) = delete;
 
+// If you have used DISABLE_CLASS_COPY with a class but still want to permit moving
+// use this macro to add the default move constructors back.
+#define ALLOW_CLASS_MOVE(C)      \
+	C(C &&other) = default;      \
+	C &operator=(C &&) = default;
+
 #ifndef _MSC_VER
 	#define UNUSED_ATTRIBUTE __attribute__ ((unused))
 #else