Async environment for mods to do concurrent tasks (#11131)

This commit is contained in:
sfan5 2022-05-02 20:55:04 +02:00
parent 663c936428
commit e7659883cc
38 changed files with 1646 additions and 48 deletions

46
builtin/async/game.lua Normal file

@ -0,0 +1,46 @@
core.log("info", "Initializing asynchronous environment (game)")
local function pack2(...)
return {n=select('#', ...), ...}
end
-- Entrypoint to run async jobs, called by C++
function core.job_processor(func, params)
local retval = pack2(func(unpack(params, 1, params.n)))
return retval
end
-- Import a bunch of individual files from builtin/game/
local gamepath = core.get_builtin_path() .. "game" .. DIR_DELIM
dofile(gamepath .. "constants.lua")
dofile(gamepath .. "item_s.lua")
dofile(gamepath .. "misc_s.lua")
dofile(gamepath .. "features.lua")
dofile(gamepath .. "voxelarea.lua")
-- Transfer of globals
do
assert(core.transferred_globals)
local all = core.deserialize(core.transferred_globals, true)
core.transferred_globals = nil
-- reassemble other tables
all.registered_nodes = {}
all.registered_craftitems = {}
all.registered_tools = {}
for k, v in pairs(all.registered_items) do
if v.type == "node" then
all.registered_nodes[k] = v
elseif v.type == "craftitem" then
all.registered_craftitems[k] = v
elseif v.type == "tool" then
all.registered_tools[k] = v
end
end
for k, v in pairs(all) do
core[k] = v
end
end

@ -1,5 +1,4 @@
core.log("info", "Initializing Asynchronous environment")
core.log("info", "Initializing asynchronous environment")
function core.job_processor(func, serialized_param)
local param = core.deserialize(serialized_param)
@ -8,4 +7,3 @@ function core.job_processor(func, serialized_param)
return retval or core.serialize(nil)
end

22
builtin/game/async.lua Normal file

@ -0,0 +1,22 @@
core.async_jobs = {}
function core.async_event_handler(jobid, retval)
local callback = core.async_jobs[jobid]
assert(type(callback) == "function")
callback(unpack(retval, 1, retval.n))
core.async_jobs[jobid] = nil
end
function core.handle_async(func, callback, ...)
assert(type(func) == "function" and type(callback) == "function",
"Invalid minetest.handle_async invocation")
local args = {n = select("#", ...), ...}
local mod_origin = core.get_last_run_mod()
local jobid = core.do_async_callback(func, args, mod_origin)
core.async_jobs[jobid] = callback
return true
end

@ -34,5 +34,6 @@ dofile(gamepath .. "voxelarea.lua")
dofile(gamepath .. "forceloading.lua")
dofile(gamepath .. "statbars.lua")
dofile(gamepath .. "knockback.lua")
dofile(gamepath .. "async.lua")
profiler = nil

@ -235,3 +235,32 @@ end
-- Used for callback handling with dynamic_add_media
core.dynamic_media_callbacks = {}
-- Transfer of certain globals into async environment
-- see builtin/async/game.lua for the other side
local function copy_filtering(t, seen)
if type(t) == "userdata" or type(t) == "function" then
return true -- don't use nil so presence can still be detected
elseif type(t) ~= "table" then
return t
end
local n = {}
seen = seen or {}
seen[t] = n
for k, v in pairs(t) do
local k_ = seen[k] or copy_filtering(k, seen)
local v_ = seen[v] or copy_filtering(v, seen)
n[k_] = v_
end
return n
end
function core.get_globals_to_transfer()
local all = {
registered_items = copy_filtering(core.registered_items),
registered_aliases = core.registered_aliases,
}
return core.serialize(all)
end

@ -56,8 +56,10 @@ elseif INIT == "mainmenu" then
if not custom_loaded then
dofile(core.get_mainmenu_path() .. DIR_DELIM .. "init.lua")
end
elseif INIT == "async" then
dofile(asyncpath .. "init.lua")
elseif INIT == "async" then
dofile(asyncpath .. "mainmenu.lua")
elseif INIT == "async_game" then
dofile(asyncpath .. "game.lua")
elseif INIT == "client" then
dofile(clientpath .. "init.lua")
else

@ -5767,6 +5767,68 @@ Timing
* `job:cancel()`
* Cancels the job function from being called
Async environment
-----------------
The engine allows you to submit jobs to be ran in an isolated environment
concurrently with normal server operation.
A job consists of a function to be ran in the async environment, any amount of
arguments (will be serialized) and a callback that will be called with the return
value of the job function once it is finished.
The async environment does *not* have access to the map, entities, players or any
globals defined in the 'usual' environment. Consequently, functions like
`minetest.get_node()` or `minetest.get_player_by_name()` simply do not exist in it.
Arguments and return values passed through this can contain certain userdata
objects that will be seamlessly copied (not shared) to the async environment.
This allows you easy interoperability for delegating work to jobs.
* `minetest.handle_async(func, callback, ...)`:
* Queue the function `func` to be ran in an async environment.
Note that there are multiple persistent workers and any of them may
end up running a given job. The engine will scale the amount of
worker threads automatically.
* When `func` returns the callback is called (in the normal environment)
with all of the return values as arguments.
* Optional: Variable number of arguments that are passed to `func`
* `minetest.register_async_dofile(path)`:
* Register a path to a Lua file to be imported when an async environment
is initialized. You can use this to preload code which you can then call
later using `minetest.handle_async()`.
### List of APIs available in an async environment
Classes:
* `ItemStack`
* `PerlinNoise`
* `PerlinNoiseMap`
* `PseudoRandom`
* `PcgRandom`
* `SecureRandom`
* `VoxelArea`
* `VoxelManip`
* only if transferred into environment; can't read/write to map
* `Settings`
Class instances that can be transferred between environments:
* `ItemStack`
* `PerlinNoise`
* `PerlinNoiseMap`
* `VoxelManip`
Functions:
* Standalone helpers such as logging, filesystem, encoding,
hashing or compression APIs
* `minetest.request_insecure_environment` (same restrictions apply)
Variables:
* `minetest.settings`
* `minetest.registered_items`, `registered_nodes`, `registered_tools`,
`registered_craftitems` and `registered_aliases`
* with all functions and userdata values replaced by `true`, calling any
callbacks here is obviously not possible
Server
------

@ -0,0 +1,149 @@
-- helper
core.register_async_dofile(core.get_modpath(core.get_current_modname()) ..
DIR_DELIM .. "inside_async_env.lua")
local function deepequal(a, b)
if type(a) == "function" then
return type(b) == "function"
elseif type(a) ~= "table" then
return a == b
elseif type(b) ~= "table" then
return false
end
for k, v in pairs(a) do
if not deepequal(v, b[k]) then
return false
end
end
for k, v in pairs(b) do
if not deepequal(a[k], v) then
return false
end
end
return true
end
-- Object Passing / Serialization
local test_object = {
name = "stairs:stair_glass",
type = "node",
groups = {oddly_breakable_by_hand = 3, cracky = 3, stair = 1},
description = "Glass Stair",
sounds = {
dig = {name = "default_glass_footstep", gain = 0.5},
footstep = {name = "default_glass_footstep", gain = 0.3},
dug = {name = "default_break_glass", gain = 1}
},
node_box = {
fixed = {
{-0.5, -0.5, -0.5, 0.5, 0, 0.5},
{-0.5, 0, 0, 0.5, 0.5, 0.5}
},
type = "fixed"
},
tiles = {
{name = "stairs_glass_split.png", backface_culling = true},
{name = "default_glass.png", backface_culling = true},
{name = "stairs_glass_stairside.png^[transformFX", backface_culling = true}
},
on_place = function(itemstack, placer)
return core.is_player(placer)
end,
sunlight_propagates = true,
is_ground_content = false,
light_source = 0,
}
local function test_object_passing()
local tmp = core.serialize_roundtrip(test_object)
assert(deepequal(test_object, tmp))
-- Circular key, should error
tmp = {"foo", "bar"}
tmp[tmp] = true
assert(not pcall(core.serialize_roundtrip, tmp))
-- Circular value, should error
tmp = {"foo"}
tmp[2] = tmp
assert(not pcall(core.serialize_roundtrip, tmp))
end
unittests.register("test_object_passing", test_object_passing)
local function test_userdata_passing(_, pos)
-- basic userdata passing
local obj = table.copy(test_object.tiles[1])
obj.test = ItemStack("default:cobble 99")
local tmp = core.serialize_roundtrip(obj)
assert(type(tmp.test) == "userdata")
assert(obj.test:to_string() == tmp.test:to_string())
-- object can't be passed, should error
obj = core.raycast(pos, pos)
assert(not pcall(core.serialize_roundtrip, obj))
-- VManip
local vm = core.get_voxel_manip(pos, pos)
local expect = vm:get_node_at(pos)
local vm2 = core.serialize_roundtrip(vm)
assert(deepequal(vm2:get_node_at(pos), expect))
end
unittests.register("test_userdata_passing", test_userdata_passing, {map=true})
-- Asynchronous jobs
local function test_handle_async(cb)
-- Basic test including mod name tracking and unittests.async_test()
-- which is defined inside_async_env.lua
local func = function(x)
return core.get_last_run_mod(), _VERSION, unittests[x]()
end
local expect = {core.get_last_run_mod(), _VERSION, true}
core.handle_async(func, function(...)
if not deepequal(expect, {...}) then
cb("Values did not equal")
end
if core.get_last_run_mod() ~= expect[1] then
cb("Mod name not tracked correctly")
end
-- Test passing of nil arguments and return values
core.handle_async(function(a, b)
return a, b
end, function(a, b)
if b ~= 123 then
cb("Argument went missing")
end
cb()
end, nil, 123)
end, "async_test")
end
unittests.register("test_handle_async", test_handle_async, {async=true})
local function test_userdata_passing2(cb, _, pos)
-- VManip: check transfer into other env
local vm = core.get_voxel_manip(pos, pos)
local expect = vm:get_node_at(pos)
core.handle_async(function(vm_, pos_)
return vm_:get_node_at(pos_)
end, function(ret)
if not deepequal(expect, ret) then
cb("Node data mismatch (one-way)")
end
-- VManip: test a roundtrip
core.handle_async(function(vm_)
return vm_
end, function(vm2)
if not deepequal(expect, vm2:get_node_at(pos)) then
cb("Node data mismatch (roundtrip)")
end
cb()
end, vm)
end, vm, pos)
end
unittests.register("test_userdata_passing2", test_userdata_passing2, {map=true, async=true})

@ -175,6 +175,7 @@ dofile(modpath .. "/misc.lua")
dofile(modpath .. "/player.lua")
dofile(modpath .. "/crafting.lua")
dofile(modpath .. "/itemdescription.lua")
dofile(modpath .. "/async_env.lua")
--------------

@ -0,0 +1,15 @@
unittests = {}
core.log("info", "Hello World")
function unittests.async_test()
assert(core == minetest)
-- stuff that should not be here
assert(not core.get_player_by_name)
assert(not core.set_node)
assert(not core.object_refs)
-- stuff that should be here
assert(ItemStack)
assert(core.registered_items[""])
return true
end

@ -1896,6 +1896,7 @@ MMVManip::MMVManip(Map *map):
VoxelManipulator(),
m_map(map)
{
assert(map);
}
void MMVManip::initialEmerge(v3s16 blockpos_min, v3s16 blockpos_max,
@ -1903,6 +1904,8 @@ void MMVManip::initialEmerge(v3s16 blockpos_min, v3s16 blockpos_max,
{
TimeTaker timer1("initialEmerge", &emerge_time);
assert(m_map);
// Units of these are MapBlocks
v3s16 p_min = blockpos_min;
v3s16 p_max = blockpos_max;
@ -1986,6 +1989,7 @@ void MMVManip::blitBackAll(std::map<v3s16, MapBlock*> *modified_blocks,
{
if(m_area.getExtent() == v3s16(0,0,0))
return;
assert(m_map);
/*
Copy data of all blocks
@ -2006,4 +2010,33 @@ void MMVManip::blitBackAll(std::map<v3s16, MapBlock*> *modified_blocks,
}
}
MMVManip *MMVManip::clone() const
{
MMVManip *ret = new MMVManip();
const s32 size = m_area.getVolume();
ret->m_area = m_area;
if (m_data) {
ret->m_data = new MapNode[size];
memcpy(ret->m_data, m_data, size * sizeof(MapNode));
}
if (m_flags) {
ret->m_flags = new u8[size];
memcpy(ret->m_flags, m_flags, size * sizeof(u8));
}
ret->m_is_dirty = m_is_dirty;
// Even if the copy is disconnected from a map object keep the information
// needed to write it back to one
ret->m_loaded_blocks = m_loaded_blocks;
return ret;
}
void MMVManip::reparent(Map *map)
{
assert(map && !m_map);
m_map = map;
}
//END

@ -446,10 +446,25 @@ public:
void blitBackAll(std::map<v3s16, MapBlock*> * modified_blocks,
bool overwrite_generated = true);
/*
Creates a copy of this VManip including contents, the copy will not be
associated with a Map.
*/
MMVManip *clone() const;
// Reassociates a copied VManip to a map
void reparent(Map *map);
// Is it impossible to call initialEmerge / blitBackAll?
inline bool isOrphan() const { return !m_map; }
bool m_is_dirty = false;
protected:
Map *m_map;
MMVManip() {};
// may be null
Map *m_map = nullptr;
/*
key = blockpos
value = flags describing the block

@ -3,6 +3,7 @@ set(common_SCRIPT_COMMON_SRCS
${CMAKE_CURRENT_SOURCE_DIR}/c_converter.cpp
${CMAKE_CURRENT_SOURCE_DIR}/c_types.cpp
${CMAKE_CURRENT_SOURCE_DIR}/c_internal.cpp
${CMAKE_CURRENT_SOURCE_DIR}/c_packer.cpp
${CMAKE_CURRENT_SOURCE_DIR}/helper.cpp
PARENT_SCOPE)

@ -166,3 +166,17 @@ void log_deprecated(lua_State *L, std::string message, int stack_depth)
infostream << script_get_backtrace(L) << std::endl;
}
void call_string_dump(lua_State *L, int idx)
{
// Retrieve string.dump from insecure env to avoid it being tampered with
lua_rawgeti(L, LUA_REGISTRYINDEX, CUSTOM_RIDX_GLOBALS_BACKUP);
if (!lua_isnil(L, -1))
lua_getfield(L, -1, "string");
else
lua_getglobal(L, "string");
lua_getfield(L, -1, "dump");
lua_remove(L, -2); // remove _G
lua_remove(L, -2); // remove 'string' table
lua_pushvalue(L, idx);
lua_call(L, 1, 1);
}

@ -56,6 +56,7 @@ extern "C" {
#define CUSTOM_RIDX_BACKTRACE (CUSTOM_RIDX_BASE + 3)
#define CUSTOM_RIDX_HTTP_API_LUA (CUSTOM_RIDX_BASE + 4)
#define CUSTOM_RIDX_VECTOR_METATABLE (CUSTOM_RIDX_BASE + 5)
#define CUSTOM_RIDX_METATABLE_MAP (CUSTOM_RIDX_BASE + 6)
// Determine if CUSTOM_RIDX_SCRIPTAPI will hold a light or full userdata
@ -139,3 +140,7 @@ DeprecatedHandlingMode get_deprecated_handling_mode();
* @param stack_depth How far on the stack to the first user function (ie: not builtin or core)
*/
void log_deprecated(lua_State *L, std::string message, int stack_depth = 1);
// Safely call string.dump on a function value
// (does not pop, leaves one value on stack)
void call_string_dump(lua_State *L, int idx);

@ -0,0 +1,583 @@
/*
Minetest
Copyright (C) 2022 sfan5 <sfan5@live.de>
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation; either version 2.1 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License along
with this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
#include <cstdio>
#include <cstring>
#include <cmath>
#include <cassert>
#include <unordered_set>
#include <unordered_map>
#include "c_packer.h"
#include "c_internal.h"
#include "log.h"
#include "debug.h"
#include "threading/mutex_auto_lock.h"
extern "C" {
#include <lauxlib.h>
}
//
// Helpers
//
// convert negative index to absolute position on Lua stack
static inline int absidx(lua_State *L, int idx)
{
assert(idx < 0);
return lua_gettop(L) + idx + 1;
}
// does the type put anything into PackedInstr::sdata?
static inline bool uses_sdata(int type)
{
switch (type) {
case LUA_TSTRING:
case LUA_TFUNCTION:
case LUA_TUSERDATA:
return true;
default:
return false;
}
}
// does the type put anything into PackedInstr::<union>?
static inline bool uses_union(int type)
{
switch (type) {
case LUA_TNIL:
case LUA_TSTRING:
case LUA_TFUNCTION:
return false;
default:
return true;
}
}
static inline bool can_set_into(int ktype, int vtype)
{
switch (ktype) {
case LUA_TNUMBER:
return !uses_union(vtype);
case LUA_TSTRING:
return !uses_sdata(vtype);
default:
return false;
}
}
// is the key suitable for use with set_into?
static inline bool suitable_key(lua_State *L, int idx)
{
if (lua_type(L, idx) == LUA_TSTRING) {
// strings may not have a NULL byte (-> lua_setfield)
size_t len;
const char *str = lua_tolstring(L, idx, &len);
return strlen(str) == len;
} else {
assert(lua_type(L, idx) == LUA_TNUMBER);
// numbers must fit into an s32 and be integers (-> lua_rawseti)
lua_Number n = lua_tonumber(L, idx);
return std::floor(n) == n && n >= S32_MIN && n <= S32_MAX;
}
}
namespace {
// checks if you left any values on the stack, for debugging
class StackChecker {
lua_State *L;
int top;
public:
StackChecker(lua_State *L) : L(L), top(lua_gettop(L)) {}
~StackChecker() {
assert(lua_gettop(L) >= top);
if (lua_gettop(L) > top) {
rawstream << "Lua stack not cleaned up: "
<< lua_gettop(L) << " != " << top
<< " (false-positive if exception thrown)" << std::endl;
}
}
};
// Since an std::vector may reallocate, this is the only safe way to keep
// a reference to a particular element.
template <typename T>
class VectorRef {
std::vector<T> *vec;
size_t idx;
VectorRef(std::vector<T> *vec, size_t idx) : vec(vec), idx(idx) {}
public:
static VectorRef<T> front(std::vector<T> &vec) {
return VectorRef(&vec, 0);
}
static VectorRef<T> back(std::vector<T> &vec) {
return VectorRef(&vec, vec.size() - 1);
}
T &operator*() { return (*vec)[idx]; }
T *operator->() { return &(*vec)[idx]; }
};
struct Packer {
PackInFunc fin;
PackOutFunc fout;
};
typedef std::pair<std::string, Packer> PackerTuple;
}
static inline auto emplace(PackedValue &pv, s16 type)
{
pv.i.emplace_back();
auto ref = VectorRef<PackedInstr>::back(pv.i);
ref->type = type;
// Initialize fields that may be left untouched
if (type == LUA_TTABLE) {
ref->uidata1 = 0;
ref->uidata2 = 0;
} else if (type == LUA_TUSERDATA) {
ref->ptrdata = nullptr;
} else if (type == INSTR_POP) {
ref->sidata2 = 0;
}
return ref;
}
//
// Management of registered packers
//
static std::unordered_map<std::string, Packer> g_packers;
static std::mutex g_packers_lock;
void script_register_packer(lua_State *L, const char *regname,
PackInFunc fin, PackOutFunc fout)
{
// Store away callbacks
{
MutexAutoLock autolock(g_packers_lock);
auto it = g_packers.find(regname);
if (it == g_packers.end()) {
auto &ref = g_packers[regname];
ref.fin = fin;
ref.fout = fout;
} else {
FATAL_ERROR_IF(it->second.fin != fin || it->second.fout != fout,
"Packer registered twice with mismatching callbacks");
}
}
// Save metatable so we can identify instances later
lua_rawgeti(L, LUA_REGISTRYINDEX, CUSTOM_RIDX_METATABLE_MAP);
if (lua_isnil(L, -1)) {
lua_newtable(L);
lua_pushvalue(L, -1);
lua_rawseti(L, LUA_REGISTRYINDEX, CUSTOM_RIDX_METATABLE_MAP);
}
luaL_getmetatable(L, regname);
FATAL_ERROR_IF(lua_isnil(L, -1), "No metatable registered with that name");
// CUSTOM_RIDX_METATABLE_MAP contains { [metatable] = "regname", ... }
// check first
lua_pushstring(L, regname);
lua_rawget(L, -3);
if (!lua_isnil(L, -1)) {
FATAL_ERROR_IF(lua_topointer(L, -1) != lua_topointer(L, -2),
"Packer registered twice with inconsistent metatable");
}
lua_pop(L, 1);
// then set
lua_pushstring(L, regname);
lua_rawset(L, -3);
lua_pop(L, 1);
}
static bool find_packer(const char *regname, PackerTuple &out)
{
MutexAutoLock autolock(g_packers_lock);
auto it = g_packers.find(regname);
if (it == g_packers.end())
return false;
// copy data for thread safety
out.first = it->first;
out.second = it->second;
return true;
}
static bool find_packer(lua_State *L, int idx, PackerTuple &out)
{
#ifndef NDEBUG
StackChecker checker(L);
#endif
// retrieve metatable of the object
if (lua_getmetatable(L, idx) != 1)
return false;
// use our global table to map it to the registry name
lua_rawgeti(L, LUA_REGISTRYINDEX, CUSTOM_RIDX_METATABLE_MAP);
assert(lua_istable(L, -1));
lua_pushvalue(L, -2);
lua_rawget(L, -2);
if (lua_isnil(L, -1)) {
lua_pop(L, 3);
return false;
}
// load the associated data
bool found = find_packer(lua_tostring(L, -1), out);
FATAL_ERROR_IF(!found, "Inconsistent internal state");
lua_pop(L, 3);
return true;
}
//
// Packing implementation
//
// recursively goes through the structure and ensures there are no circular references
static void pack_validate(lua_State *L, int idx, std::unordered_set<const void*> &seen)
{
#ifndef NDEBUG
StackChecker checker(L);
assert(idx > 0);
#endif
if (lua_type(L, idx) != LUA_TTABLE)
return;
const void *ptr = lua_topointer(L, idx);
assert(ptr);
if (seen.find(ptr) != seen.end())
throw LuaError("Circular references cannot be packed (yet)");
seen.insert(ptr);
lua_checkstack(L, 5);
lua_pushnil(L);
while (lua_next(L, idx) != 0) {
// key at -2, value at -1
pack_validate(L, absidx(L, -2), seen);
pack_validate(L, absidx(L, -1), seen);
lua_pop(L, 1);
}
seen.erase(ptr);
}
static VectorRef<PackedInstr> pack_inner(lua_State *L, int idx, int vidx, PackedValue &pv)
{
#ifndef NDEBUG
StackChecker checker(L);
assert(idx > 0);
assert(vidx > 0);
#endif
switch (lua_type(L, idx)) {
case LUA_TNONE:
case LUA_TNIL:
return emplace(pv, LUA_TNIL);
case LUA_TBOOLEAN: {
auto r = emplace(pv, LUA_TBOOLEAN);
r->bdata = lua_toboolean(L, idx);
return r;
}
case LUA_TNUMBER: {
auto r = emplace(pv, LUA_TNUMBER);
r->ndata = lua_tonumber(L, idx);
return r;
}
case LUA_TSTRING: {
auto r = emplace(pv, LUA_TSTRING);
size_t len;
const char *str = lua_tolstring(L, idx, &len);
assert(str);
r->sdata.assign(str, len);
return r;
}
case LUA_TTABLE:
break; // execution continues
case LUA_TFUNCTION: {
auto r = emplace(pv, LUA_TFUNCTION);
call_string_dump(L, idx);
size_t len;
const char *str = lua_tolstring(L, -1, &len);
assert(str);
r->sdata.assign(str, len);
lua_pop(L, 1);
return r;
}
case LUA_TUSERDATA: {
PackerTuple ser;
if (!find_packer(L, idx, ser))
throw LuaError("Cannot serialize unsupported userdata");
pv.contains_userdata = true;
auto r = emplace(pv, LUA_TUSERDATA);
r->sdata = ser.first;
r->ptrdata = ser.second.fin(L, idx);
return r;
}
default: {
std::string err = "Cannot serialize type ";
err += lua_typename(L, lua_type(L, idx));
throw LuaError(err);
}
}
// LUA_TTABLE
lua_checkstack(L, 5);
auto rtable = emplace(pv, LUA_TTABLE);
const int vi_table = vidx++;
lua_pushnil(L);
while (lua_next(L, idx) != 0) {
// key at -2, value at -1
const int ktype = lua_type(L, -2), vtype = lua_type(L, -1);
if (ktype == LUA_TNUMBER)
rtable->uidata1++; // narr
else
rtable->uidata2++; // nrec
// check if we can use a shortcut
if (can_set_into(ktype, vtype) && suitable_key(L, -2)) {
// push only the value
auto rval = pack_inner(L, absidx(L, -1), vidx, pv);
rval->pop = vtype != LUA_TTABLE;
// and where to put it:
rval->set_into = vi_table;
if (ktype == LUA_TSTRING)
rval->sdata = lua_tostring(L, -2);
else
rval->sidata1 = lua_tointeger(L, -2);
// pop tables after the fact
if (!rval->pop) {
auto ri1 = emplace(pv, INSTR_POP);
ri1->sidata1 = vidx;
}
} else {
// push the key and value
pack_inner(L, absidx(L, -2), vidx, pv);
vidx++;
pack_inner(L, absidx(L, -1), vidx, pv);
vidx++;
// push an instruction to set them
auto ri1 = emplace(pv, INSTR_SETTABLE);
ri1->set_into = vi_table;
ri1->sidata1 = vidx - 2;
ri1->sidata2 = vidx - 1;
ri1->pop = true;
vidx -= 2;
}
lua_pop(L, 1);
}
assert(vidx == vi_table + 1);
return rtable;
}
PackedValue *script_pack(lua_State *L, int idx)
{
if (idx < 0)
idx = absidx(L, idx);
std::unordered_set<const void*> seen;
pack_validate(L, idx, seen);
assert(seen.size() == 0);
// Actual serialization
PackedValue pv;
pack_inner(L, idx, 1, pv);
return new PackedValue(std::move(pv));
}
//
// Unpacking implementation
//
void script_unpack(lua_State *L, PackedValue *pv)
{
const int top = lua_gettop(L);
int ctr = 0;
for (auto &i : pv->i) {
// If leaving values on stack make sure there's space (every 5th iteration)
if (!i.pop && (ctr++) >= 5) {
lua_checkstack(L, 5);
ctr = 0;
}
/* Instructions */
switch (i.type) {
case INSTR_SETTABLE:
lua_pushvalue(L, top + i.sidata1); // key
lua_pushvalue(L, top + i.sidata2); // value
lua_rawset(L, top + i.set_into);
if (i.pop) {
if (i.sidata1 != i.sidata2) {
// removing moves indices so pop higher index first
lua_remove(L, top + std::max(i.sidata1, i.sidata2));
lua_remove(L, top + std::min(i.sidata1, i.sidata2));
} else {
lua_remove(L, top + i.sidata1);
}
}
continue;
case INSTR_POP:
lua_remove(L, top + i.sidata1);
if (i.sidata2 > 0)
lua_remove(L, top + i.sidata2);
continue;
default:
break;
}
/* Lua types */
switch (i.type) {
case LUA_TNIL:
lua_pushnil(L);
break;
case LUA_TBOOLEAN:
lua_pushboolean(L, i.bdata);
break;
case LUA_TNUMBER:
lua_pushnumber(L, i.ndata);
break;
case LUA_TSTRING:
lua_pushlstring(L, i.sdata.data(), i.sdata.size());
break;
case LUA_TTABLE:
lua_createtable(L, i.uidata1, i.uidata2);
break;
case LUA_TFUNCTION:
luaL_loadbuffer(L, i.sdata.data(), i.sdata.size(), nullptr);
break;
case LUA_TUSERDATA: {
PackerTuple ser;
sanity_check(find_packer(i.sdata.c_str(), ser));
ser.second.fout(L, i.ptrdata);
i.ptrdata = nullptr; // ownership taken by callback
break;
}
default:
assert(0);
break;
}
if (i.set_into) {
if (!i.pop)
lua_pushvalue(L, -1);
if (uses_sdata(i.type))
lua_rawseti(L, top + i.set_into, i.sidata1);
else
lua_setfield(L, top + i.set_into, i.sdata.c_str());
} else {
if (i.pop)
lua_pop(L, 1);
}
}
// as part of the unpacking process we take ownership of all userdata
pv->contains_userdata = false;
// leave exactly one value on the stack
lua_settop(L, top+1);
}
//
// PackedValue
//
PackedValue::~PackedValue()
{
if (!contains_userdata)
return;
for (auto &i : this->i) {
if (i.type == LUA_TUSERDATA && i.ptrdata) {
PackerTuple ser;
if (find_packer(i.sdata.c_str(), ser)) {
// tell it to deallocate object
ser.second.fout(nullptr, i.ptrdata);
} else {
assert(false);
}
}
}
}
//
// script_dump_packed
//
#ifndef NDEBUG
void script_dump_packed(const PackedValue *val)
{
printf("instruction stream: [\n");
for (const auto &i : val->i) {
printf("\t(");
switch (i.type) {
case INSTR_SETTABLE:
printf("SETTABLE(%d, %d)", i.sidata1, i.sidata2);
break;
case INSTR_POP:
printf(i.sidata2 ? "POP(%d, %d)" : "POP(%d)", i.sidata1, i.sidata2);
break;
case LUA_TNIL:
printf("nil");
break;
case LUA_TBOOLEAN:
printf(i.bdata ? "true" : "false");
break;
case LUA_TNUMBER:
printf("%f", i.ndata);
break;
case LUA_TSTRING:
printf("\"%s\"", i.sdata.c_str());
break;
case LUA_TTABLE:
printf("table(%d, %d)", i.uidata1, i.uidata2);
break;
case LUA_TFUNCTION:
printf("function(%d byte)", i.sdata.size());
break;
case LUA_TUSERDATA:
printf("userdata %s %p", i.sdata.c_str(), i.ptrdata);
break;
default:
printf("!!UNKNOWN!!");
break;
}
if (i.set_into) {
if (i.type >= 0 && uses_sdata(i.type))
printf(", k=%d, into=%d", i.sidata1, i.set_into);
else if (i.type >= 0)
printf(", k=\"%s\", into=%d", i.sdata.c_str(), i.set_into);
else
printf(", into=%d", i.set_into);
}
if (i.pop)
printf(", pop");
printf(")\n");
}
printf("]\n");
}
#endif

@ -0,0 +1,123 @@
/*
Minetest
Copyright (C) 2022 sfan5 <sfan5@live.de>
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation; either version 2.1 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License along
with this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
#pragma once
#include <string>
#include <vector>
#include "irrlichttypes.h"
#include "util/basic_macros.h"
extern "C" {
#include <lua.h>
}
/*
This file defines an in-memory representation of Lua objects including
support for functions and userdata. It it used to move data between Lua
states and cannot be used for persistence or network transfer.
*/
#define INSTR_SETTABLE (-10)
#define INSTR_POP (-11)
/**
* Represents a single instruction that pushes a new value or works with existing ones.
*/
struct PackedInstr
{
s16 type; // LUA_T* or INSTR_*
u16 set_into; // set into table on stack
bool pop; // remove from stack?
union {
bool bdata; // boolean: value
lua_Number ndata; // number: value
struct {
u16 uidata1, uidata2; // table: narr, nrec
};
struct {
/*
SETTABLE: key index, value index
POP: indices to remove
otherwise w/ set_into: numeric key, -
*/
s32 sidata1, sidata2;
};
void *ptrdata; // userdata: implementation defined
};
/*
- string: value
- function: buffer
- w/ set_into: string key (no null bytes!)
- userdata: name in registry
*/
std::string sdata;
PackedInstr() : type(0), set_into(0), pop(false) {}
};
/**
* A packed value can be a primitive like a string or number but also a table
* including all of its contents. It is made up of a linear stream of
* 'instructions' that build the final value when executed.
*/
struct PackedValue
{
std::vector<PackedInstr> i;
// Indicates whether there are any userdata pointers that need to be deallocated
bool contains_userdata = false;
PackedValue() = default;
~PackedValue();
DISABLE_CLASS_COPY(PackedValue)
ALLOW_CLASS_MOVE(PackedValue)
};
/*
* Packing callback: Turns a Lua value at given index into a void*
*/
typedef void *(*PackInFunc)(lua_State *L, int idx);
/*
* Unpacking callback: Turns a void* back into the Lua value (left on top of stack)
*
* Note that this function must take ownership of the pointer, so make sure
* to free or keep the memory.
* `L` can be nullptr to indicate that data should just be discarded.
*/
typedef void (*PackOutFunc)(lua_State *L, void *ptr);
/*
* Register a packable type with the name of its metatable.
*
* Even though the callbacks are global this must be called for every Lua state
* that supports objects of this type.
* This function is thread-safe.
*/
void script_register_packer(lua_State *L, const char *regname,
PackInFunc fin, PackOutFunc fout);
// Pack a Lua value
PackedValue *script_pack(lua_State *L, int idx);
// Unpack a Lua value (left on top of stack)
// Note that this may modify the PackedValue, you can't reuse it!
void script_unpack(lua_State *L, PackedValue *val);
// Dump contents of PackedValue to stdout for debugging
void script_dump_packed(const PackedValue *val);

@ -21,9 +21,9 @@ with this program; if not, write to the Free Software Foundation, Inc.,
#include <cstdlib>
extern "C" {
#include "lua.h"
#include "lauxlib.h"
#include "lualib.h"
#include <lua.h>
#include <lauxlib.h>
#include <lualib.h>
}
#include "server.h"
@ -32,6 +32,7 @@ extern "C" {
#include "filesys.h"
#include "porting.h"
#include "common/c_internal.h"
#include "common/c_packer.h"
#include "lua_api/l_base.h"
/******************************************************************************/
@ -76,19 +77,34 @@ void AsyncEngine::initialize(unsigned int numEngines)
{
initDone = true;
for (unsigned int i = 0; i < numEngines; i++) {
AsyncWorkerThread *toAdd = new AsyncWorkerThread(this,
std::string("AsyncWorker-") + itos(i));
workerThreads.push_back(toAdd);
toAdd->start();
if (numEngines == 0) {
// Leave one core for the main thread and one for whatever else
autoscaleMaxWorkers = Thread::getNumberOfProcessors();
if (autoscaleMaxWorkers >= 2)
autoscaleMaxWorkers -= 2;
infostream << "AsyncEngine: using at most " << autoscaleMaxWorkers
<< " threads with automatic scaling" << std::endl;
addWorkerThread();
} else {
for (unsigned int i = 0; i < numEngines; i++)
addWorkerThread();
}
}
void AsyncEngine::addWorkerThread()
{
AsyncWorkerThread *toAdd = new AsyncWorkerThread(this,
std::string("AsyncWorker-") + itos(workerThreads.size()));
workerThreads.push_back(toAdd);
toAdd->start();
}
/******************************************************************************/
u32 AsyncEngine::queueAsyncJob(std::string &&func, std::string &&params,
const std::string &mod_origin)
{
jobQueueMutex.lock();
MutexAutoLock autolock(jobQueueMutex);
u32 jobId = jobIdCounter++;
jobQueue.emplace_back();
@ -99,7 +115,23 @@ u32 AsyncEngine::queueAsyncJob(std::string &&func, std::string &&params,
to_add.mod_origin = mod_origin;
jobQueueCounter.post();
jobQueueMutex.unlock();
return jobId;
}
u32 AsyncEngine::queueAsyncJob(std::string &&func, PackedValue *params,
const std::string &mod_origin)
{
MutexAutoLock autolock(jobQueueMutex);
u32 jobId = jobIdCounter++;
jobQueue.emplace_back();
auto &to_add = jobQueue.back();
to_add.id = jobId;
to_add.function = std::move(func);
to_add.params_ext.reset(params);
to_add.mod_origin = mod_origin;
jobQueueCounter.post();
return jobId;
}
@ -131,6 +163,12 @@ void AsyncEngine::putJobResult(LuaJobInfo &&result)
/******************************************************************************/
void AsyncEngine::step(lua_State *L)
{
stepJobResults(L);
stepAutoscale();
}
void AsyncEngine::stepJobResults(lua_State *L)
{
int error_handler = PUSH_ERROR_HANDLER(L);
lua_getglobal(L, "core");
@ -148,7 +186,10 @@ void AsyncEngine::step(lua_State *L)
luaL_checktype(L, -1, LUA_TFUNCTION);
lua_pushinteger(L, j.id);
lua_pushlstring(L, j.result.data(), j.result.size());
if (j.result_ext)
script_unpack(L, j.result_ext.get());
else
lua_pushlstring(L, j.result.data(), j.result.size());
// Call handler
const char *origin = j.mod_origin.empty() ? nullptr : j.mod_origin.c_str();
@ -161,12 +202,71 @@ void AsyncEngine::step(lua_State *L)
lua_pop(L, 2); // Pop core and error handler
}
void AsyncEngine::stepAutoscale()
{
if (workerThreads.size() >= autoscaleMaxWorkers)
return;
MutexAutoLock autolock(jobQueueMutex);
// 2) If the timer elapsed, check again
if (autoscaleTimer && porting::getTimeMs() >= autoscaleTimer) {
autoscaleTimer = 0;
// Determine overlap with previous snapshot
unsigned int n = 0;
for (const auto &it : jobQueue)
n += autoscaleSeenJobs.count(it.id);
autoscaleSeenJobs.clear();
infostream << "AsyncEngine: " << n << " jobs were still waiting after 1s" << std::endl;
// Start this many new threads
while (workerThreads.size() < autoscaleMaxWorkers && n > 0) {
addWorkerThread();
n--;
}
return;
}
// 1) Check if there's anything in the queue
if (!autoscaleTimer && !jobQueue.empty()) {
// Take a snapshot of all jobs we have seen
for (const auto &it : jobQueue)
autoscaleSeenJobs.emplace(it.id);
// and set a timer for 1 second
autoscaleTimer = porting::getTimeMs() + 1000;
}
}
/******************************************************************************/
void AsyncEngine::prepareEnvironment(lua_State* L, int top)
bool AsyncEngine::prepareEnvironment(lua_State* L, int top)
{
for (StateInitializer &stateInitializer : stateInitializers) {
stateInitializer(L, top);
}
auto *script = ModApiBase::getScriptApiBase(L);
try {
script->loadMod(Server::getBuiltinLuaPath() + DIR_DELIM + "init.lua",
BUILTIN_MOD_NAME);
} catch (const ModError &e) {
errorstream << "Execution of async base environment failed: "
<< e.what() << std::endl;
FATAL_ERROR("Execution of async base environment failed");
}
// Load per mod stuff
if (server) {
const auto &list = server->m_async_init_files;
try {
for (auto &it : list)
script->loadMod(it.second, it.first);
} catch (const ModError &e) {
errorstream << "Failed to load mod script inside async environment." << std::endl;
server->setAsyncFatalError(e.what());
return false;
}
}
return true;
}
/******************************************************************************/
@ -178,15 +278,25 @@ AsyncWorkerThread::AsyncWorkerThread(AsyncEngine* jobDispatcher,
{
lua_State *L = getStack();
if (jobDispatcher->server) {
setGameDef(jobDispatcher->server);
if (g_settings->getBool("secure.enable_security"))
initializeSecurity();
}
// Prepare job lua environment
lua_getglobal(L, "core");
int top = lua_gettop(L);
// Push builtin initialization type
lua_pushstring(L, "async");
lua_pushstring(L, jobDispatcher->server ? "async_game" : "async");
lua_setglobal(L, "INIT");
jobDispatcher->prepareEnvironment(L, top);
if (!jobDispatcher->prepareEnvironment(L, top)) {
// can't throw from here so we're stuck with this
isErrored = true;
}
}
/******************************************************************************/
@ -198,19 +308,20 @@ AsyncWorkerThread::~AsyncWorkerThread()
/******************************************************************************/
void* AsyncWorkerThread::run()
{
if (isErrored)
return nullptr;
lua_State *L = getStack();
try {
loadMod(getServer()->getBuiltinLuaPath() + DIR_DELIM + "init.lua",
BUILTIN_MOD_NAME);
} catch (const ModError &e) {
errorstream << "Execution of async base environment failed: "
<< e.what() << std::endl;
FATAL_ERROR("Execution of async base environment failed");
}
int error_handler = PUSH_ERROR_HANDLER(L);
auto report_error = [this] (const ModError &e) {
if (jobDispatcher->server)
jobDispatcher->server->setAsyncFatalError(e.what());
else
errorstream << e.what() << std::endl;
};
lua_getglobal(L, "core");
if (lua_isnil(L, -1)) {
FATAL_ERROR("Unable to find core within async environment!");
@ -223,6 +334,8 @@ void* AsyncWorkerThread::run()
if (!jobDispatcher->getJob(&j) || stopRequested())
continue;
const bool use_ext = !!j.params_ext;
lua_getfield(L, -1, "job_processor");
if (lua_isnil(L, -1))
FATAL_ERROR("Unable to get async job processor!");
@ -232,7 +345,10 @@ void* AsyncWorkerThread::run()
errorstream << "ASYNC WORKER: Unable to deserialize function" << std::endl;
lua_pushnil(L);
}
lua_pushlstring(L, j.params.data(), j.params.size());
if (use_ext)
script_unpack(L, j.params_ext.get());
else
lua_pushlstring(L, j.params.data(), j.params.size());
// Call it
setOriginDirect(j.mod_origin.empty() ? nullptr : j.mod_origin.c_str());
@ -241,19 +357,28 @@ void* AsyncWorkerThread::run()
try {
scriptError(result, "<async>");
} catch (const ModError &e) {
errorstream << e.what() << std::endl;
report_error(e);
}
} else {
// Fetch result
size_t length;
const char *retval = lua_tolstring(L, -1, &length);
j.result.assign(retval, length);
if (use_ext) {
try {
j.result_ext.reset(script_pack(L, -1));
} catch (const ModError &e) {
report_error(e);
result = LUA_ERRERR;
}
} else {
size_t length;
const char *retval = lua_tolstring(L, -1, &length);
j.result.assign(retval, length);
}
}
lua_pop(L, 1); // Pop retval
// Put job result
if (!j.result.empty())
if (result == 0)
jobDispatcher->putJobResult(std::move(j));
}

@ -21,11 +21,15 @@ with this program; if not, write to the Free Software Foundation, Inc.,
#include <vector>
#include <deque>
#include <unordered_set>
#include <memory>
#include <lua.h>
#include "threading/semaphore.h"
#include "threading/thread.h"
#include "lua.h"
#include "common/c_packer.h"
#include "cpp_api/s_base.h"
#include "cpp_api/s_security.h"
// Forward declarations
class AsyncEngine;
@ -42,8 +46,12 @@ struct LuaJobInfo
std::string function;
// Parameter to be passed to function (serialized)
std::string params;
// Alternative parameters
std::unique_ptr<PackedValue> params_ext;
// Result of function call (serialized)
std::string result;
// Alternative result
std::unique_ptr<PackedValue> result_ext;
// Name of the mod who invoked this call
std::string mod_origin;
// JobID used to identify a job and match it to callback
@ -51,7 +59,8 @@ struct LuaJobInfo
};
// Asynchronous working environment
class AsyncWorkerThread : public Thread, virtual public ScriptApiBase {
class AsyncWorkerThread : public Thread,
virtual public ScriptApiBase, public ScriptApiSecurity {
friend class AsyncEngine;
public:
virtual ~AsyncWorkerThread();
@ -63,6 +72,7 @@ protected:
private:
AsyncEngine *jobDispatcher = nullptr;
bool isErrored = false;
};
// Asynchornous thread and job management
@ -71,6 +81,7 @@ class AsyncEngine {
typedef void (*StateInitializer)(lua_State *L, int top);
public:
AsyncEngine() = default;
AsyncEngine(Server *server) : server(server) {};
~AsyncEngine();
/**
@ -81,7 +92,7 @@ public:
/**
* Create async engine tasks and lock function registration
* @param numEngines Number of async threads to be started
* @param numEngines Number of worker threads, 0 for automatic scaling
*/
void initialize(unsigned int numEngines);
@ -94,9 +105,17 @@ public:
u32 queueAsyncJob(std::string &&func, std::string &&params,
const std::string &mod_origin = "");
/**
* Queue an async job
* @param func Serialized lua function
* @param params Serialized parameters (takes ownership!)
* @return ID of queued job
*/
u32 queueAsyncJob(std::string &&func, PackedValue *params,
const std::string &mod_origin = "");
/**
* Engine step to process finished jobs
* the engine step is one way to pass events back, PushFinishedJobs another
* @param L The Lua stack
*/
void step(lua_State *L);
@ -116,19 +135,44 @@ protected:
*/
void putJobResult(LuaJobInfo &&result);
/**
* Start an additional worker thread
*/
void addWorkerThread();
/**
* Process finished jobs callbacks
*/
void stepJobResults(lua_State *L);
/**
* Handle automatic scaling of worker threads
*/
void stepAutoscale();
/**
* Initialize environment with current registred functions
* this function adds all functions registred by registerFunction to the
* passed lua stack
* @param L Lua stack to initialize
* @param top Stack position
* @return false if a mod error ocurred
*/
void prepareEnvironment(lua_State* L, int top);
bool prepareEnvironment(lua_State* L, int top);
private:
// Variable locking the engine against further modification
bool initDone = false;
// Maximum number of worker threads for automatic scaling
// 0 if disabled
unsigned int autoscaleMaxWorkers = 0;
u64 autoscaleTimer = 0;
std::unordered_set<u32> autoscaleSeenJobs;
// Only set for the server async environment (duh)
Server *server = nullptr;
// Internal store for registred state initializers
std::vector<StateInitializer> stateInitializers;

@ -525,3 +525,11 @@ void ModApiCraft::Initialize(lua_State *L, int top)
API_FCT(register_craft);
API_FCT(clear_craft);
}
void ModApiCraft::InitializeAsync(lua_State *L, int top)
{
// all read-only functions
API_FCT(get_all_craft_recipes);
API_FCT(get_craft_recipe);
API_FCT(get_craft_result);
}

@ -45,4 +45,5 @@ private:
public:
static void Initialize(lua_State *L, int top);
static void InitializeAsync(lua_State *L, int top);
};

@ -69,7 +69,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
// Retrieve Environment pointer as `env` (no map lock)
#define GET_PLAIN_ENV_PTR_NO_MAP_LOCK \
Environment *env = (Environment *)getEnv(L); \
Environment *env = getEnv(L); \
if (env == NULL) \
return 0

@ -22,6 +22,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
#include "lua_api/l_internal.h"
#include "common/c_converter.h"
#include "common/c_content.h"
#include "common/c_packer.h"
#include "itemdef.h"
#include "nodedef.h"
#include "server.h"
@ -441,6 +442,7 @@ int LuaItemStack::create_object(lua_State *L)
lua_setmetatable(L, -2);
return 1;
}
// Not callable from Lua
int LuaItemStack::create(lua_State *L, const ItemStack &item)
{
@ -457,6 +459,20 @@ LuaItemStack *LuaItemStack::checkobject(lua_State *L, int narg)
return *(LuaItemStack **)luaL_checkudata(L, narg, className);
}
void *LuaItemStack::packIn(lua_State *L, int idx)
{
LuaItemStack *o = checkobject(L, idx);
return new ItemStack(o->getItem());
}
void LuaItemStack::packOut(lua_State *L, void *ptr)
{
ItemStack *stack = reinterpret_cast<ItemStack*>(ptr);
if (L)
create(L, *stack);
delete stack;
}
void LuaItemStack::Register(lua_State *L)
{
lua_newtable(L);
@ -488,6 +504,8 @@ void LuaItemStack::Register(lua_State *L)
// Can be created from Lua (ItemStack(itemstack or itemstring or table or nil))
lua_register(L, className, create_object);
script_register_packer(L, className, packIn, packOut);
}
const char LuaItemStack::className[] = "ItemStack";
@ -673,3 +691,10 @@ void ModApiItemMod::Initialize(lua_State *L, int top)
API_FCT(get_content_id);
API_FCT(get_name_from_content_id);
}
void ModApiItemMod::InitializeAsync(lua_State *L, int top)
{
// all read-only functions
API_FCT(get_content_id);
API_FCT(get_name_from_content_id);
}

@ -141,8 +141,11 @@ public:
// Not callable from Lua
static int create(lua_State *L, const ItemStack &item);
static LuaItemStack* checkobject(lua_State *L, int narg);
static void Register(lua_State *L);
static void *packIn(lua_State *L, int idx);
static void packOut(lua_State *L, void *ptr);
static void Register(lua_State *L);
};
class ModApiItemMod : public ModApiBase {
@ -152,6 +155,8 @@ private:
static int l_register_alias_raw(lua_State *L);
static int l_get_content_id(lua_State *L);
static int l_get_name_from_content_id(lua_State *L);
public:
static void Initialize(lua_State *L, int top);
static void InitializeAsync(lua_State *L, int top);
};

@ -21,6 +21,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
#include "lua_api/l_internal.h"
#include "common/c_converter.h"
#include "common/c_content.h"
#include "common/c_packer.h"
#include "log.h"
#include "porting.h"
#include "util/numeric.h"
@ -101,6 +102,25 @@ LuaPerlinNoise *LuaPerlinNoise::checkobject(lua_State *L, int narg)
}
void *LuaPerlinNoise::packIn(lua_State *L, int idx)
{
LuaPerlinNoise *o = checkobject(L, idx);
return new NoiseParams(o->np);
}
void LuaPerlinNoise::packOut(lua_State *L, void *ptr)
{
NoiseParams *np = reinterpret_cast<NoiseParams*>(ptr);
if (L) {
LuaPerlinNoise *o = new LuaPerlinNoise(np);
*(void **)(lua_newuserdata(L, sizeof(void *))) = o;
luaL_getmetatable(L, className);
lua_setmetatable(L, -2);
}
delete np;
}
void LuaPerlinNoise::Register(lua_State *L)
{
lua_newtable(L);
@ -126,6 +146,8 @@ void LuaPerlinNoise::Register(lua_State *L)
lua_pop(L, 1);
lua_register(L, className, create_object);
script_register_packer(L, className, packIn, packOut);
}
@ -357,6 +379,35 @@ LuaPerlinNoiseMap *LuaPerlinNoiseMap::checkobject(lua_State *L, int narg)
}
struct NoiseMapParams {
NoiseParams np;
s32 seed;
v3s16 size;
};
void *LuaPerlinNoiseMap::packIn(lua_State *L, int idx)
{
LuaPerlinNoiseMap *o = checkobject(L, idx);
NoiseMapParams *ret = new NoiseMapParams();
ret->np = o->noise->np;
ret->seed = o->noise->seed;
ret->size = v3s16(o->noise->sx, o->noise->sy, o->noise->sz);
return ret;
}
void LuaPerlinNoiseMap::packOut(lua_State *L, void *ptr)
{
NoiseMapParams *p = reinterpret_cast<NoiseMapParams*>(ptr);
if (L) {
LuaPerlinNoiseMap *o = new LuaPerlinNoiseMap(&p->np, p->seed, p->size);
*(void **)(lua_newuserdata(L, sizeof(void *))) = o;
luaL_getmetatable(L, className);
lua_setmetatable(L, -2);
}
delete p;
}
void LuaPerlinNoiseMap::Register(lua_State *L)
{
lua_newtable(L);
@ -382,6 +433,8 @@ void LuaPerlinNoiseMap::Register(lua_State *L)
lua_pop(L, 1);
lua_register(L, className, create_object);
script_register_packer(L, className, packIn, packOut);
}

@ -52,6 +52,9 @@ public:
static LuaPerlinNoise *checkobject(lua_State *L, int narg);
static void *packIn(lua_State *L, int idx);
static void packOut(lua_State *L, void *ptr);
static void Register(lua_State *L);
};
@ -91,6 +94,9 @@ public:
static LuaPerlinNoiseMap *checkobject(lua_State *L, int narg);
static void *packIn(lua_State *L, int idx);
static void packOut(lua_State *L, void *ptr);
static void Register(lua_State *L);
};

@ -21,6 +21,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
#include "lua_api/l_internal.h"
#include "common/c_converter.h"
#include "common/c_content.h"
#include "common/c_packer.h"
#include "cpp_api/s_base.h"
#include "cpp_api/s_security.h"
#include "scripting_server.h"
@ -526,6 +527,76 @@ int ModApiServer::l_notify_authentication_modified(lua_State *L)
return 0;
}
// do_async_callback(func, params, mod_origin)
int ModApiServer::l_do_async_callback(lua_State *L)
{
NO_MAP_LOCK_REQUIRED;
ServerScripting *script = getScriptApi<ServerScripting>(L);
luaL_checktype(L, 1, LUA_TFUNCTION);
luaL_checktype(L, 2, LUA_TTABLE);
luaL_checktype(L, 3, LUA_TSTRING);
call_string_dump(L, 1);
size_t func_length;
const char *serialized_func_raw = lua_tolstring(L, -1, &func_length);
PackedValue *param = script_pack(L, 2);
std::string mod_origin = readParam<std::string>(L, 3);
u32 jobId = script->queueAsync(
std::string(serialized_func_raw, func_length),
param, mod_origin);
lua_settop(L, 0);
lua_pushinteger(L, jobId);
return 1;
}
// register_async_dofile(path)
int ModApiServer::l_register_async_dofile(lua_State *L)
{
NO_MAP_LOCK_REQUIRED;
std::string path = readParam<std::string>(L, 1);
CHECK_SECURE_PATH(L, path.c_str(), false);
// Find currently running mod name (only at init time)
lua_rawgeti(L, LUA_REGISTRYINDEX, CUSTOM_RIDX_CURRENT_MOD_NAME);
if (!lua_isstring(L, -1))
return 0;
std::string modname = readParam<std::string>(L, -1);
getServer(L)->m_async_init_files.emplace_back(modname, path);
lua_pushboolean(L, true);
return 1;
}
// serialize_roundtrip(value)
// Meant for unit testing the packer from Lua
int ModApiServer::l_serialize_roundtrip(lua_State *L)
{
NO_MAP_LOCK_REQUIRED;
int top = lua_gettop(L);
auto *pv = script_pack(L, 1);
if (top != lua_gettop(L))
throw LuaError("stack values leaked");
#ifndef NDEBUG
script_dump_packed(pv);
#endif
top = lua_gettop(L);
script_unpack(L, pv);
delete pv;
if (top + 1 != lua_gettop(L))
throw LuaError("stack values leaked");
return 1;
}
void ModApiServer::Initialize(lua_State *L, int top)
{
API_FCT(request_shutdown);
@ -559,4 +630,18 @@ void ModApiServer::Initialize(lua_State *L, int top)
API_FCT(remove_player);
API_FCT(unban_player_or_ip);
API_FCT(notify_authentication_modified);
API_FCT(do_async_callback);
API_FCT(register_async_dofile);
API_FCT(serialize_roundtrip);
}
void ModApiServer::InitializeAsync(lua_State *L, int top)
{
API_FCT(get_worldpath);
API_FCT(is_singleplayer);
API_FCT(get_current_modname);
API_FCT(get_modpath);
API_FCT(get_modnames);
}

@ -106,6 +106,16 @@ private:
// notify_authentication_modified(name)
static int l_notify_authentication_modified(lua_State *L);
// do_async_callback(func, params, mod_origin)
static int l_do_async_callback(lua_State *L);
// register_async_dofile(path)
static int l_register_async_dofile(lua_State *L);
// serialize_roundtrip(obj)
static int l_serialize_roundtrip(lua_State *L);
public:
static void Initialize(lua_State *L, int top);
static void InitializeAsync(lua_State *L, int top);
};

@ -671,6 +671,9 @@ void ModApiUtil::InitializeAsync(lua_State *L, int top)
API_FCT(cpdir);
API_FCT(mvdir);
API_FCT(get_dir_list);
API_FCT(safe_file_write);
API_FCT(request_insecure_environment);
API_FCT(encode_base64);
API_FCT(decode_base64);
@ -680,6 +683,8 @@ void ModApiUtil::InitializeAsync(lua_State *L, int top)
API_FCT(colorspec_to_colorstring);
API_FCT(colorspec_to_bytes);
API_FCT(encode_png);
API_FCT(get_last_run_mod);
API_FCT(set_last_run_mod);

@ -129,6 +129,4 @@ public:
static void Initialize(lua_State *L, int top);
static void InitializeAsync(lua_State *L, int top);
static void InitializeClient(lua_State *L, int top);
static void InitializeAsync(AsyncEngine &engine);
};

@ -22,6 +22,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
#include "lua_api/l_internal.h"
#include "common/c_content.h"
#include "common/c_converter.h"
#include "common/c_packer.h"
#include "emerge.h"
#include "environment.h"
#include "map.h"
@ -45,6 +46,8 @@ int LuaVoxelManip::l_read_from_map(lua_State *L)
LuaVoxelManip *o = checkobject(L, 1);
MMVManip *vm = o->vm;
if (vm->isOrphan())
return 0;
v3s16 bp1 = getNodeBlockPos(check_v3s16(L, 2));
v3s16 bp2 = getNodeBlockPos(check_v3s16(L, 3));
@ -429,6 +432,34 @@ LuaVoxelManip *LuaVoxelManip::checkobject(lua_State *L, int narg)
return *(LuaVoxelManip **)ud; // unbox pointer
}
void *LuaVoxelManip::packIn(lua_State *L, int idx)
{
LuaVoxelManip *o = checkobject(L, idx);
if (o->is_mapgen_vm)
throw LuaError("nope");
return o->vm->clone();
}
void LuaVoxelManip::packOut(lua_State *L, void *ptr)
{
MMVManip *vm = reinterpret_cast<MMVManip*>(ptr);
if (!L) {
delete vm;
return;
}
// Associate vmanip with map if the Lua env has one
Environment *env = getEnv(L);
if (env)
vm->reparent(&(env->getMap()));
LuaVoxelManip *o = new LuaVoxelManip(vm, false);
*(void **)(lua_newuserdata(L, sizeof(void *))) = o;
luaL_getmetatable(L, className);
lua_setmetatable(L, -2);
}
void LuaVoxelManip::Register(lua_State *L)
{
lua_newtable(L);
@ -455,6 +486,8 @@ void LuaVoxelManip::Register(lua_State *L)
// Can be created from Lua (VoxelManip())
lua_register(L, className, create_object);
script_register_packer(L, className, packIn, packOut);
}
const char LuaVoxelManip::className[] = "VoxelManip";

@ -75,5 +75,8 @@ public:
static LuaVoxelManip *checkobject(lua_State *L, int narg);
static void *packIn(lua_State *L, int idx);
static void packOut(lua_State *L, void *ptr);
static void Register(lua_State *L);
};

@ -47,11 +47,12 @@ with this program; if not, write to the Free Software Foundation, Inc.,
#include "lua_api/l_storage.h"
extern "C" {
#include "lualib.h"
#include <lualib.h>
}
ServerScripting::ServerScripting(Server* server):
ScriptApiBase(ScriptingType::Server)
ScriptApiBase(ScriptingType::Server),
asyncEngine(server)
{
setGameDef(server);
@ -88,6 +89,47 @@ ServerScripting::ServerScripting(Server* server):
infostream << "SCRIPTAPI: Initialized game modules" << std::endl;
}
void ServerScripting::initAsync()
{
// Save globals to transfer
{
lua_State *L = getStack();
lua_getglobal(L, "core");
luaL_checktype(L, -1, LUA_TTABLE);
lua_getfield(L, -1, "get_globals_to_transfer");
lua_call(L, 0, 1);
luaL_checktype(L, -1, LUA_TSTRING);
getServer()->m_async_globals_data.set(readParam<std::string>(L, -1));
lua_pushnil(L);
lua_setfield(L, -3, "get_globals_to_transfer"); // unset function too
lua_pop(L, 2); // pop 'core', return value
}
infostream << "SCRIPTAPI: Initializing async engine" << std::endl;
asyncEngine.registerStateInitializer(InitializeAsync);
asyncEngine.registerStateInitializer(ModApiUtil::InitializeAsync);
asyncEngine.registerStateInitializer(ModApiCraft::InitializeAsync);
asyncEngine.registerStateInitializer(ModApiItemMod::InitializeAsync);
asyncEngine.registerStateInitializer(ModApiServer::InitializeAsync);
// not added: ModApiMapgen is a minefield for thread safety
// not added: ModApiHttp async api can't really work together with our jobs
// not added: ModApiStorage is probably not thread safe(?)
asyncEngine.initialize(0);
}
void ServerScripting::stepAsync()
{
asyncEngine.step(getStack());
}
u32 ServerScripting::queueAsync(std::string &&serialized_func,
PackedValue *param, const std::string &mod_origin)
{
return asyncEngine.queueAsyncJob(std::move(serialized_func),
param, mod_origin);
}
void ServerScripting::InitializeModApi(lua_State *L, int top)
{
// Register reference classes (userdata)
@ -125,3 +167,24 @@ void ServerScripting::InitializeModApi(lua_State *L, int top)
ModApiStorage::Initialize(L, top);
ModApiChannels::Initialize(L, top);
}
void ServerScripting::InitializeAsync(lua_State *L, int top)
{
// classes
LuaItemStack::Register(L);
LuaPerlinNoise::Register(L);
LuaPerlinNoiseMap::Register(L);
LuaPseudoRandom::Register(L);
LuaPcgRandom::Register(L);
LuaSecureRandom::Register(L);
LuaVoxelManip::Register(L);
LuaSettings::Register(L);
// globals data
lua_getglobal(L, "core");
luaL_checktype(L, -1, LUA_TTABLE);
std::string s = ModApiBase::getServer(L)->m_async_globals_data.get();
lua_pushlstring(L, s.c_str(), s.size());
lua_setfield(L, -2, "transferred_globals");
lua_pop(L, 1); // pop 'core'
}

@ -27,6 +27,9 @@ with this program; if not, write to the Free Software Foundation, Inc.,
#include "cpp_api/s_player.h"
#include "cpp_api/s_server.h"
#include "cpp_api/s_security.h"
#include "cpp_api/s_async.h"
struct PackedValue;
/*****************************************************************************/
/* Scripting <-> Server Game Interface */
@ -48,6 +51,20 @@ public:
// use ScriptApiBase::loadMod() to load mods
// Initialize async engine, call this AFTER loading all mods
void initAsync();
// Global step handler to collect async results
void stepAsync();
// Pass job to async threads
u32 queueAsync(std::string &&serialized_func,
PackedValue *param, const std::string &mod_origin);
private:
void InitializeModApi(lua_State *L, int top);
static void InitializeAsync(lua_State *L, int top);
AsyncEngine asyncEngine;
};

@ -243,6 +243,7 @@ Server::Server(
m_clients(m_con),
m_admin_chat(iface),
m_on_shutdown_errmsg(on_shutdown_errmsg),
m_async_globals_data(""),
m_modchannel_mgr(new ModChannelMgr())
{
if (m_path_world.empty())
@ -480,6 +481,9 @@ void Server::init()
// Give environment reference to scripting api
m_script->initializeEnvironment(m_env);
// Do this after regular script init is done
m_script->initAsync();
// Register us to receive map edit events
servermap->addEventReceiver(this);

@ -292,7 +292,7 @@ public:
virtual const std::vector<ModSpec> &getMods() const;
virtual const ModSpec* getModSpec(const std::string &modname) const;
std::string getBuiltinLuaPath();
static std::string getBuiltinLuaPath();
virtual std::string getWorldPath() const { return m_path_world; }
inline bool isSingleplayer() const
@ -385,6 +385,12 @@ public:
static bool migrateModStorageDatabase(const GameParams &game_params,
const Settings &cmd_args);
// Lua files registered for init of async env, pair of modname + path
std::vector<std::pair<std::string, std::string>> m_async_init_files;
// Serialized data transferred into async envs at init time
MutexedVariable<std::string> m_async_globals_data;
// Bind address
Address m_bind_addr;

@ -1481,6 +1481,8 @@ void ServerEnvironment::step(float dtime)
*/
m_script->environment_Step(dtime);
m_script->stepAsync();
/*
Step active objects
*/

@ -29,13 +29,19 @@ with this program; if not, write to the Free Software Foundation, Inc.,
#define CONTAINS(c, v) (std::find((c).begin(), (c).end(), (v)) != (c).end())
// To disable copy constructors and assignment operations for some class
// 'Foobar', add the macro DISABLE_CLASS_COPY(Foobar) as a private member.
// 'Foobar', add the macro DISABLE_CLASS_COPY(Foobar) in the class definition.
// Note this also disables copying for any classes derived from 'Foobar' as well
// as classes having a 'Foobar' member.
#define DISABLE_CLASS_COPY(C) \
C(const C &) = delete; \
C &operator=(const C &) = delete;
// If you have used DISABLE_CLASS_COPY with a class but still want to permit moving
// use this macro to add the default move constructors back.
#define ALLOW_CLASS_MOVE(C) \
C(C &&other) = default; \
C &operator=(C &&) = default;
#ifndef _MSC_VER
#define UNUSED_ATTRIBUTE __attribute__ ((unused))
#else