BRNSystems/minetest
1
0
Fork 0
forked from Mirrorlandia_minetest/minetest
Code Pull Requests 1 Activity

Support packing arbitrary graphs (#12289)

Browse Source
This commit is contained in:
Jude Melton-Houghton 2022-05-10 16:37:33 -04:00 committed by GitHub
parent d17d7eba14
commit 7f58887ae3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 89 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
games/devtest/mods/unittests/async_env.lua
View File

@ -60,15 +60,34 @@ local function test_object_passing()
local tmp = core.serialize_roundtrip(test_object) local tmp = core.serialize_roundtrip(test_object)
assert(deepequal(test_object, tmp)) assert(deepequal(test_object, tmp))
-- Circular key, should error local circular_key = {"foo", "bar"}
tmp = {"foo", "bar"} circular_key[circular_key] = true
tmp[tmp] = true tmp = core.serialize_roundtrip(circular_key)
assert(not pcall(core.serialize_roundtrip, tmp)) assert(tmp[1] == "foo")
assert(tmp[2] == "bar")
assert(tmp[tmp] == true)
-- Circular value, should error local circular_value = {"foo"}
tmp = {"foo"} circular_value[2] = circular_value
tmp[2] = tmp tmp = core.serialize_roundtrip(circular_value)
assert(not pcall(core.serialize_roundtrip, tmp)) assert(tmp[1] == "foo")
assert(tmp[2] == tmp)
-- Two-segment cycle
local cycle_seg_1, cycle_seg_2 = {}, {}
cycle_seg_1[1] = cycle_seg_2
cycle_seg_2[1] = cycle_seg_1
tmp = core.serialize_roundtrip(cycle_seg_1)
assert(tmp[1][1] == tmp)
-- Duplicated value without a cycle
local acyclic_dup_holder = {}
tmp = ItemStack("")
acyclic_dup_holder[tmp] = tmp
tmp = core.serialize_roundtrip(acyclic_dup_holder)
for k, v in pairs(tmp) do
assert(rawequal(k, v))
end
end end
unittests.register("test_object_passing", test_object_passing) unittests.register("test_object_passing", test_object_passing)

103
src/script/common/c_packer.cpp
View File

@ -123,6 +123,7 @@ namespace {
size_t idx; size_t idx;
VectorRef(std::vector<T> *vec, size_t idx) : vec(vec), idx(idx) {} VectorRef(std::vector<T> *vec, size_t idx) : vec(vec), idx(idx) {}
public: public:
constexpr VectorRef() : vec(nullptr), idx(0) {}
static VectorRef<T> front(std::vector<T> &vec) { static VectorRef<T> front(std::vector<T> &vec) {
return VectorRef(&vec, 0); return VectorRef(&vec, 0);
} }
@ -131,6 +132,7 @@ namespace {
} }
T &operator*() { return (*vec)[idx]; } T &operator*() { return (*vec)[idx]; }
T *operator->() { return &(*vec)[idx]; } T *operator->() { return &(*vec)[idx]; }
operator bool() const { return vec != nullptr; }
}; };
struct Packer { struct Packer {
@ -252,38 +254,27 @@ static bool find_packer(lua_State *L, int idx, PackerTuple &out)
// Packing implementation // Packing implementation
// //
// recursively goes through the structure and ensures there are no circular references static VectorRef<PackedInstr> record_object(lua_State *L, int idx, PackedValue &pv,
static void pack_validate(lua_State *L, int idx, std::unordered_set<const void*> &seen) std::unordered_map<const void *, s32> &seen)
{ {
#ifndef NDEBUG
StackChecker checker(L);
assert(idx > 0);
#endif
if (lua_type(L, idx) != LUA_TTABLE)
return;
const void *ptr = lua_topointer(L, idx); const void *ptr = lua_topointer(L, idx);
assert(ptr); assert(ptr);
auto found = seen.find(ptr);
if (seen.find(ptr) != seen.end()) if (found == seen.end()) {
throw LuaError("Circular references cannot be packed (yet)"); seen[ptr] = pv.i.size();
seen.insert(ptr); return VectorRef<PackedInstr>();
lua_checkstack(L, 5);
lua_pushnil(L);
while (lua_next(L, idx) != 0) {
// key at -2, value at -1
pack_validate(L, absidx(L, -2), seen);
pack_validate(L, absidx(L, -1), seen);
lua_pop(L, 1);
} }
s32 ref = found->second;
seen.erase(ptr); assert(ref < (s32)pv.i.size());
// reuse the value from first time
auto r = emplace(pv, INSTR_PUSHREF);
r->ref = ref;
pv.i[ref].keep_ref = true;
return r;
} }
static VectorRef<PackedInstr> pack_inner(lua_State *L, int idx, int vidx, PackedValue &pv) static VectorRef<PackedInstr> pack_inner(lua_State *L, int idx, int vidx, PackedValue &pv,
std::unordered_map<const void *, s32> &seen)
{ {
#ifndef NDEBUG #ifndef NDEBUG
StackChecker checker(L); StackChecker checker(L);
@ -313,10 +304,17 @@ static VectorRef<PackedInstr> pack_inner(lua_State *L, int idx, int vidx, Packed
r->sdata.assign(str, len); r->sdata.assign(str, len);
return r; return r;
} }
case LUA_TTABLE: case LUA_TTABLE: {
auto r = record_object(L, idx, pv, seen);
if (r)
return r;
break; // execution continues break; // execution continues
}
case LUA_TFUNCTION: { case LUA_TFUNCTION: {
auto r = emplace(pv, LUA_TFUNCTION); auto r = record_object(L, idx, pv, seen);
if (r)
return r;
r = emplace(pv, LUA_TFUNCTION);
call_string_dump(L, idx); call_string_dump(L, idx);
size_t len; size_t len;
const char *str = lua_tolstring(L, -1, &len); const char *str = lua_tolstring(L, -1, &len);
@ -326,11 +324,14 @@ static VectorRef<PackedInstr> pack_inner(lua_State *L, int idx, int vidx, Packed
return r; return r;
} }
case LUA_TUSERDATA: { case LUA_TUSERDATA: {
auto r = record_object(L, idx, pv, seen);
if (r)
return r;
PackerTuple ser; PackerTuple ser;
if (!find_packer(L, idx, ser)) if (!find_packer(L, idx, ser))
throw LuaError("Cannot serialize unsupported userdata"); throw LuaError("Cannot serialize unsupported userdata");
pv.contains_userdata = true; pv.contains_userdata = true;
auto r = emplace(pv, LUA_TUSERDATA); r = emplace(pv, LUA_TUSERDATA);
r->sdata = ser.first; r->sdata = ser.first;
r->ptrdata = ser.second.fin(L, idx); r->ptrdata = ser.second.fin(L, idx);
return r; return r;
@ -360,8 +361,8 @@ static VectorRef<PackedInstr> pack_inner(lua_State *L, int idx, int vidx, Packed
// check if we can use a shortcut // check if we can use a shortcut
if (can_set_into(ktype, vtype) && suitable_key(L, -2)) { if (can_set_into(ktype, vtype) && suitable_key(L, -2)) {
// push only the value // push only the value
auto rval = pack_inner(L, absidx(L, -1), vidx, pv); auto rval = pack_inner(L, absidx(L, -1), vidx, pv, seen);
rval->pop = vtype != LUA_TTABLE; rval->pop = rval->type != LUA_TTABLE;
// and where to put it: // and where to put it:
rval->set_into = vi_table; rval->set_into = vi_table;
if (ktype == LUA_TSTRING) if (ktype == LUA_TSTRING)
@ -375,9 +376,9 @@ static VectorRef<PackedInstr> pack_inner(lua_State *L, int idx, int vidx, Packed
} }
} else { } else {
// push the key and value // push the key and value
pack_inner(L, absidx(L, -2), vidx, pv); pack_inner(L, absidx(L, -2), vidx, pv, seen);
vidx++; vidx++;
pack_inner(L, absidx(L, -1), vidx, pv); pack_inner(L, absidx(L, -1), vidx, pv, seen);
vidx++; vidx++;
// push an instruction to set them // push an instruction to set them
auto ri1 = emplace(pv, INSTR_SETTABLE); auto ri1 = emplace(pv, INSTR_SETTABLE);
@ -400,13 +401,9 @@ PackedValue *script_pack(lua_State *L, int idx)
if (idx < 0) if (idx < 0)
idx = absidx(L, idx); idx = absidx(L, idx);
std::unordered_set<const void*> seen;
pack_validate(L, idx, seen);
assert(seen.size() == 0);
// Actual serialization
PackedValue pv; PackedValue pv;
pack_inner(L, idx, 1, pv); std::unordered_map<const void *, s32> seen;
pack_inner(L, idx, 1, pv, seen);
return new PackedValue(std::move(pv)); return new PackedValue(std::move(pv));
} }
@ -417,18 +414,21 @@ PackedValue *script_pack(lua_State *L, int idx)
void script_unpack(lua_State *L, PackedValue *pv) void script_unpack(lua_State *L, PackedValue *pv)
{ {
lua_newtable(L); // table at index top to track ref indices -> objects
const int top = lua_gettop(L); const int top = lua_gettop(L);
int ctr = 0; int ctr = 0;
for (auto &i : pv->i) { for (size_t packed_idx = 0; packed_idx < pv->i.size(); packed_idx++) {
auto &i = pv->i[packed_idx];
// If leaving values on stack make sure there's space (every 5th iteration) // If leaving values on stack make sure there's space (every 5th iteration)
if (!i.pop && (ctr++) >= 5) { if (!i.pop && (ctr++) >= 5) {
lua_checkstack(L, 5); lua_checkstack(L, 5);
ctr = 0; ctr = 0;
} }
/* Instructions */
switch (i.type) { switch (i.type) {
/* Instructions */
case INSTR_SETTABLE: case INSTR_SETTABLE:
lua_pushvalue(L, top + i.sidata1); // key lua_pushvalue(L, top + i.sidata1); // key
lua_pushvalue(L, top + i.sidata2); // value lua_pushvalue(L, top + i.sidata2); // value
@ -448,12 +448,12 @@ void script_unpack(lua_State *L, PackedValue *pv)
if (i.sidata2 > 0) if (i.sidata2 > 0)
lua_remove(L, top + i.sidata2); lua_remove(L, top + i.sidata2);
continue; continue;
default: case INSTR_PUSHREF:
lua_pushinteger(L, i.ref);
lua_rawget(L, top);
break; break;
}
/* Lua types */ /* Lua types */
switch (i.type) {
case LUA_TNIL: case LUA_TNIL:
lua_pushnil(L); lua_pushnil(L);
break; break;
@ -479,11 +479,18 @@ void script_unpack(lua_State *L, PackedValue *pv)
i.ptrdata = nullptr; // ownership taken by callback i.ptrdata = nullptr; // ownership taken by callback
break; break;
} }
default: default:
assert(0); assert(0);
break; break;
} }
if (i.keep_ref) {
lua_pushinteger(L, packed_idx);
lua_pushvalue(L, -2);
lua_rawset(L, top);
}
if (i.set_into) { if (i.set_into) {
if (!i.pop) if (!i.pop)
lua_pushvalue(L, -1); lua_pushvalue(L, -1);
@ -501,6 +508,7 @@ void script_unpack(lua_State *L, PackedValue *pv)
pv->contains_userdata = false; pv->contains_userdata = false;
// leave exactly one value on the stack // leave exactly one value on the stack
lua_settop(L, top+1); lua_settop(L, top+1);
lua_remove(L, top);
} }
// //
@ -541,6 +549,9 @@ void script_dump_packed(const PackedValue *val)
case INSTR_POP: case INSTR_POP:
printf(i.sidata2 ? "POP(%d, %d)" : "POP(%d)", i.sidata1, i.sidata2); printf(i.sidata2 ? "POP(%d, %d)" : "POP(%d)", i.sidata1, i.sidata2);
break; break;
case INSTR_PUSHREF:
printf("PUSHREF(%d)", i.ref);
break;
case LUA_TNIL: case LUA_TNIL:
printf("nil"); printf("nil");
break; break;
@ -574,6 +585,8 @@ void script_dump_packed(const PackedValue *val)
else else
printf(", into=%d", i.set_into); printf(", into=%d", i.set_into);
} }
if (i.keep_ref)
printf(", keep_ref");
if (i.pop) if (i.pop)
printf(", pop"); printf(", pop");
printf(")\n"); printf(")\n");

5
src/script/common/c_packer.h
View File

@ -36,6 +36,7 @@ extern "C" {
#define INSTR_SETTABLE (-10) #define INSTR_SETTABLE (-10)
#define INSTR_POP (-11) #define INSTR_POP (-11)
#define INSTR_PUSHREF (-12)
/** /**
* Represents a single instruction that pushes a new value or works with existing ones. * Represents a single instruction that pushes a new value or works with existing ones.
@ -44,6 +45,7 @@ struct PackedInstr
{ {
s16 type; // LUA_T* or INSTR_* s16 type; // LUA_T* or INSTR_*
u16 set_into; // set into table on stack u16 set_into; // set into table on stack
bool keep_ref; // is referenced later by INSTR_PUSHREF?
bool pop; // remove from stack? bool pop; // remove from stack?
union { union {
bool bdata; // boolean: value bool bdata; // boolean: value
@ -60,6 +62,7 @@ struct PackedInstr
s32 sidata1, sidata2; s32 sidata1, sidata2;
}; };
void *ptrdata; // userdata: implementation defined void *ptrdata; // userdata: implementation defined
s32 ref; // PUSHREF: index of referenced instr
}; };
/* /*
- string: value - string: value
@ -69,7 +72,7 @@ struct PackedInstr
*/ */
std::string sdata; std::string sdata;
PackedInstr() : type(0), set_into(0), pop(false) {} PackedInstr() : type(0), set_into(0), keep_ref(false), pop(false) {}
}; };
/** /**