Add zstd compression support (#12515)

This commit is contained in:
20kdc 2022-09-28 14:06:14 +01:00 committed by GitHub
parent 0251b01da6
commit b1233056b7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 95 additions and 13 deletions

@ -26,6 +26,7 @@ core.features = {
get_sky_as_table = true, get_sky_as_table = true,
get_light_data_buffer = true, get_light_data_buffer = true,
mod_storage_on_disk = true, mod_storage_on_disk = true,
compress_zstd = true,
} }
function core.has_feature(arg) function core.has_feature(arg)

@ -898,13 +898,18 @@ Call these functions only at load time!
* `method` is a string identifying the compression method to be used. * `method` is a string identifying the compression method to be used.
* Supported compression methods: * Supported compression methods:
* Deflate (zlib): `"deflate"` * Deflate (zlib): `"deflate"`
* `...` indicates method-specific arguments. Currently defined arguments are: * Zstandard: `"zstd"`
* `...` indicates method-specific arguments. Currently defined arguments
are:
* Deflate: `level` - Compression level, `0`-`9` or `nil`. * Deflate: `level` - Compression level, `0`-`9` or `nil`.
* Zstandard: `level` - Compression level. Integer or `nil`. Default `3`.
Note any supported Zstandard compression level could be used here,
but these are subject to change between Zstandard versions.
* `minetest.decompress(compressed_data, method, ...)`: returns data * `minetest.decompress(compressed_data, method, ...)`: returns data
* Decompress a string of data (using ZLib). * Decompress a string of data using the algorithm specified by `method`.
* See documentation on `minetest.compress()` for supported compression methods. * See documentation on `minetest.compress()` for supported compression
* currently supported. methods.
* `...` indicates method-specific arguments. Currently, no methods use this. * `...` indicates method-specific arguments. Currently, no methods use this
* `minetest.rgba(red, green, blue[, alpha])`: returns a string * `minetest.rgba(red, green, blue[, alpha])`: returns a string
* Each argument is a 8 Bit unsigned integer * Each argument is a 8 Bit unsigned integer
* Returns the ColorString from rgb or rgba values * Returns the ColorString from rgb or rgba values

@ -4910,6 +4910,8 @@ Utilities
-- the amount of data in mod storage is not constrained by -- the amount of data in mod storage is not constrained by
-- the amount of RAM available. (5.7.0) -- the amount of RAM available. (5.7.0)
mod_storage_on_disk = true, mod_storage_on_disk = true,
-- "zstd" method for compress/decompress (5.7.0)
compress_zstd = true,
} }
* `minetest.has_feature(arg)`: returns `boolean, missing_features` * `minetest.has_feature(arg)`: returns `boolean, missing_features`
@ -6370,11 +6372,15 @@ Misc.
* `method` is a string identifying the compression method to be used. * `method` is a string identifying the compression method to be used.
* Supported compression methods: * Supported compression methods:
* Deflate (zlib): `"deflate"` * Deflate (zlib): `"deflate"`
* Zstandard: `"zstd"`
* `...` indicates method-specific arguments. Currently defined arguments * `...` indicates method-specific arguments. Currently defined arguments
are: are:
* Deflate: `level` - Compression level, `0`-`9` or `nil`. * Deflate: `level` - Compression level, `0`-`9` or `nil`.
* Zstandard: `level` - Compression level. Integer or `nil`. Default `3`.
Note any supported Zstandard compression level could be used here,
but these are subject to change between Zstandard versions.
* `minetest.decompress(compressed_data, method, ...)`: returns data * `minetest.decompress(compressed_data, method, ...)`: returns data
* Decompress a string of data (using ZLib). * Decompress a string of data using the algorithm specified by `method`.
* See documentation on `minetest.compress()` for supported compression * See documentation on `minetest.compress()` for supported compression
methods. methods.
* `...` indicates method-specific arguments. Currently, no methods use this * `...` indicates method-specific arguments. Currently, no methods use this

@ -80,3 +80,26 @@ unittests.register("test_punch_node", function(_, pos)
minetest.remove_node(pos) minetest.remove_node(pos)
-- currently failing: assert(on_punch_called) -- currently failing: assert(on_punch_called)
end, {map=true}) end, {map=true})
local function test_compress()
-- This text should be compressible, to make sure the results are... normal
local text = "The\000 icey canoe couldn't move very well on the\128 lake. The\000 ice was too stiff and the icey canoe's paddles simply wouldn't punch through."
local methods = {
"deflate",
"zstd",
-- "noodle", -- for warning alarm test
}
local zstd_magic = string.char(0x28, 0xB5, 0x2F, 0xFD)
for _, method in ipairs(methods) do
local compressed = core.compress(text, method)
assert(core.decompress(compressed, method) == text, "input/output mismatch for compression method " .. method)
local has_zstd_magic = compressed:sub(1, 4) == zstd_magic
if method == "zstd" then
assert(has_zstd_magic, "zstd magic number not in zstd method")
else
assert(not has_zstd_magic, "zstd magic number in method " .. method .. " (which is not zstd)")
end
end
end
unittests.register("test_compress", test_compress)

@ -26,6 +26,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
#include "cpp_api/s_async.h" #include "cpp_api/s_async.h"
#include "serialization.h" #include "serialization.h"
#include <json/json.h> #include <json/json.h>
#include <zstd.h>
#include "cpp_api/s_security.h" #include "cpp_api/s_security.h"
#include "porting.h" #include "porting.h"
#include "convert_json.h" #include "convert_json.h"
@ -278,6 +279,34 @@ int ModApiUtil::l_get_user_path(lua_State *L)
return 1; return 1;
} }
enum LuaCompressMethod
{
LUA_COMPRESS_METHOD_DEFLATE,
LUA_COMPRESS_METHOD_ZSTD,
};
static const struct EnumString es_LuaCompressMethod[] =
{
{LUA_COMPRESS_METHOD_DEFLATE, "deflate"},
{LUA_COMPRESS_METHOD_ZSTD, "zstd"},
{0, nullptr},
};
static LuaCompressMethod get_compress_method(lua_State *L, int index)
{
if (lua_isnoneornil(L, index))
return LUA_COMPRESS_METHOD_DEFLATE;
const char *name = luaL_checkstring(L, index);
int value;
if (!string_to_enum(es_LuaCompressMethod, value, name)) {
// Pretend it's deflate if we don't know, for compatibility reasons.
log_deprecated(L, "Unknown compression method \"" + std::string(name)
+ "\", defaulting to \"deflate\". You should pass a valid value.");
return LUA_COMPRESS_METHOD_DEFLATE;
}
return (LuaCompressMethod) value;
}
// compress(data, method, level) // compress(data, method, level)
int ModApiUtil::l_compress(lua_State *L) int ModApiUtil::l_compress(lua_State *L)
{ {
@ -286,12 +315,23 @@ int ModApiUtil::l_compress(lua_State *L)
size_t size; size_t size;
const char *data = luaL_checklstring(L, 1, &size); const char *data = luaL_checklstring(L, 1, &size);
LuaCompressMethod method = get_compress_method(L, 2);
std::ostringstream os(std::ios_base::binary);
if (method == LUA_COMPRESS_METHOD_DEFLATE) {
int level = -1; int level = -1;
if (!lua_isnoneornil(L, 3)) if (!lua_isnoneornil(L, 3))
level = readParam<int>(L, 3); level = readParam<int>(L, 3);
std::ostringstream os(std::ios_base::binary);
compressZlib(reinterpret_cast<const u8 *>(data), size, os, level); compressZlib(reinterpret_cast<const u8 *>(data), size, os, level);
} else if (method == LUA_COMPRESS_METHOD_ZSTD) {
int level = ZSTD_CLEVEL_DEFAULT;
if (!lua_isnoneornil(L, 3))
level = readParam<int>(L, 3);
compressZstd(reinterpret_cast<const u8 *>(data), size, os, level);
}
std::string out = os.str(); std::string out = os.str();
@ -307,9 +347,16 @@ int ModApiUtil::l_decompress(lua_State *L)
size_t size; size_t size;
const char *data = luaL_checklstring(L, 1, &size); const char *data = luaL_checklstring(L, 1, &size);
LuaCompressMethod method = get_compress_method(L, 2);
std::istringstream is(std::string(data, size), std::ios_base::binary); std::istringstream is(std::string(data, size), std::ios_base::binary);
std::ostringstream os(std::ios_base::binary); std::ostringstream os(std::ios_base::binary);
if (method == LUA_COMPRESS_METHOD_DEFLATE) {
decompressZlib(is, os); decompressZlib(is, os);
} else if (method == LUA_COMPRESS_METHOD_ZSTD) {
decompressZstd(is, os);
}
std::string out = os.str(); std::string out = os.str();