mirror of
https://github.com/minetest/minetest.git
synced 2024-08-18 20:44:16 +02:00
Add zstd compression support (#12515)
This commit is contained in:
parent
0251b01da6
commit
b1233056b7
@ -26,6 +26,7 @@ core.features = {
|
||||
get_sky_as_table = true,
|
||||
get_light_data_buffer = true,
|
||||
mod_storage_on_disk = true,
|
||||
compress_zstd = true,
|
||||
}
|
||||
|
||||
function core.has_feature(arg)
|
||||
|
@ -897,14 +897,19 @@ Call these functions only at load time!
|
||||
* Compress a string of data.
|
||||
* `method` is a string identifying the compression method to be used.
|
||||
* Supported compression methods:
|
||||
* Deflate (zlib): `"deflate"`
|
||||
* `...` indicates method-specific arguments. Currently defined arguments are:
|
||||
* Deflate: `level` - Compression level, `0`-`9` or `nil`.
|
||||
* Deflate (zlib): `"deflate"`
|
||||
* Zstandard: `"zstd"`
|
||||
* `...` indicates method-specific arguments. Currently defined arguments
|
||||
are:
|
||||
* Deflate: `level` - Compression level, `0`-`9` or `nil`.
|
||||
* Zstandard: `level` - Compression level. Integer or `nil`. Default `3`.
|
||||
Note any supported Zstandard compression level could be used here,
|
||||
but these are subject to change between Zstandard versions.
|
||||
* `minetest.decompress(compressed_data, method, ...)`: returns data
|
||||
* Decompress a string of data (using ZLib).
|
||||
* See documentation on `minetest.compress()` for supported compression methods.
|
||||
* currently supported.
|
||||
* `...` indicates method-specific arguments. Currently, no methods use this.
|
||||
* Decompress a string of data using the algorithm specified by `method`.
|
||||
* See documentation on `minetest.compress()` for supported compression
|
||||
methods.
|
||||
* `...` indicates method-specific arguments. Currently, no methods use this
|
||||
* `minetest.rgba(red, green, blue[, alpha])`: returns a string
|
||||
* Each argument is a 8 Bit unsigned integer
|
||||
* Returns the ColorString from rgb or rgba values
|
||||
|
@ -4910,6 +4910,8 @@ Utilities
|
||||
-- the amount of data in mod storage is not constrained by
|
||||
-- the amount of RAM available. (5.7.0)
|
||||
mod_storage_on_disk = true,
|
||||
-- "zstd" method for compress/decompress (5.7.0)
|
||||
compress_zstd = true,
|
||||
}
|
||||
|
||||
* `minetest.has_feature(arg)`: returns `boolean, missing_features`
|
||||
@ -6370,11 +6372,15 @@ Misc.
|
||||
* `method` is a string identifying the compression method to be used.
|
||||
* Supported compression methods:
|
||||
* Deflate (zlib): `"deflate"`
|
||||
* Zstandard: `"zstd"`
|
||||
* `...` indicates method-specific arguments. Currently defined arguments
|
||||
are:
|
||||
* Deflate: `level` - Compression level, `0`-`9` or `nil`.
|
||||
* Zstandard: `level` - Compression level. Integer or `nil`. Default `3`.
|
||||
Note any supported Zstandard compression level could be used here,
|
||||
but these are subject to change between Zstandard versions.
|
||||
* `minetest.decompress(compressed_data, method, ...)`: returns data
|
||||
* Decompress a string of data (using ZLib).
|
||||
* Decompress a string of data using the algorithm specified by `method`.
|
||||
* See documentation on `minetest.compress()` for supported compression
|
||||
methods.
|
||||
* `...` indicates method-specific arguments. Currently, no methods use this
|
||||
|
@ -80,3 +80,26 @@ unittests.register("test_punch_node", function(_, pos)
|
||||
minetest.remove_node(pos)
|
||||
-- currently failing: assert(on_punch_called)
|
||||
end, {map=true})
|
||||
|
||||
local function test_compress()
|
||||
-- This text should be compressible, to make sure the results are... normal
|
||||
local text = "The\000 icey canoe couldn't move very well on the\128 lake. The\000 ice was too stiff and the icey canoe's paddles simply wouldn't punch through."
|
||||
local methods = {
|
||||
"deflate",
|
||||
"zstd",
|
||||
-- "noodle", -- for warning alarm test
|
||||
}
|
||||
local zstd_magic = string.char(0x28, 0xB5, 0x2F, 0xFD)
|
||||
for _, method in ipairs(methods) do
|
||||
local compressed = core.compress(text, method)
|
||||
assert(core.decompress(compressed, method) == text, "input/output mismatch for compression method " .. method)
|
||||
local has_zstd_magic = compressed:sub(1, 4) == zstd_magic
|
||||
if method == "zstd" then
|
||||
assert(has_zstd_magic, "zstd magic number not in zstd method")
|
||||
else
|
||||
assert(not has_zstd_magic, "zstd magic number in method " .. method .. " (which is not zstd)")
|
||||
end
|
||||
end
|
||||
end
|
||||
unittests.register("test_compress", test_compress)
|
||||
|
||||
|
@ -26,6 +26,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
#include "cpp_api/s_async.h"
|
||||
#include "serialization.h"
|
||||
#include <json/json.h>
|
||||
#include <zstd.h>
|
||||
#include "cpp_api/s_security.h"
|
||||
#include "porting.h"
|
||||
#include "convert_json.h"
|
||||
@ -278,6 +279,34 @@ int ModApiUtil::l_get_user_path(lua_State *L)
|
||||
return 1;
|
||||
}
|
||||
|
||||
enum LuaCompressMethod
|
||||
{
|
||||
LUA_COMPRESS_METHOD_DEFLATE,
|
||||
LUA_COMPRESS_METHOD_ZSTD,
|
||||
};
|
||||
|
||||
static const struct EnumString es_LuaCompressMethod[] =
|
||||
{
|
||||
{LUA_COMPRESS_METHOD_DEFLATE, "deflate"},
|
||||
{LUA_COMPRESS_METHOD_ZSTD, "zstd"},
|
||||
{0, nullptr},
|
||||
};
|
||||
|
||||
static LuaCompressMethod get_compress_method(lua_State *L, int index)
|
||||
{
|
||||
if (lua_isnoneornil(L, index))
|
||||
return LUA_COMPRESS_METHOD_DEFLATE;
|
||||
const char *name = luaL_checkstring(L, index);
|
||||
int value;
|
||||
if (!string_to_enum(es_LuaCompressMethod, value, name)) {
|
||||
// Pretend it's deflate if we don't know, for compatibility reasons.
|
||||
log_deprecated(L, "Unknown compression method \"" + std::string(name)
|
||||
+ "\", defaulting to \"deflate\". You should pass a valid value.");
|
||||
return LUA_COMPRESS_METHOD_DEFLATE;
|
||||
}
|
||||
return (LuaCompressMethod) value;
|
||||
}
|
||||
|
||||
// compress(data, method, level)
|
||||
int ModApiUtil::l_compress(lua_State *L)
|
||||
{
|
||||
@ -286,12 +315,23 @@ int ModApiUtil::l_compress(lua_State *L)
|
||||
size_t size;
|
||||
const char *data = luaL_checklstring(L, 1, &size);
|
||||
|
||||
int level = -1;
|
||||
if (!lua_isnoneornil(L, 3))
|
||||
level = readParam<int>(L, 3);
|
||||
LuaCompressMethod method = get_compress_method(L, 2);
|
||||
|
||||
std::ostringstream os(std::ios_base::binary);
|
||||
compressZlib(reinterpret_cast<const u8 *>(data), size, os, level);
|
||||
|
||||
if (method == LUA_COMPRESS_METHOD_DEFLATE) {
|
||||
int level = -1;
|
||||
if (!lua_isnoneornil(L, 3))
|
||||
level = readParam<int>(L, 3);
|
||||
|
||||
compressZlib(reinterpret_cast<const u8 *>(data), size, os, level);
|
||||
} else if (method == LUA_COMPRESS_METHOD_ZSTD) {
|
||||
int level = ZSTD_CLEVEL_DEFAULT;
|
||||
if (!lua_isnoneornil(L, 3))
|
||||
level = readParam<int>(L, 3);
|
||||
|
||||
compressZstd(reinterpret_cast<const u8 *>(data), size, os, level);
|
||||
}
|
||||
|
||||
std::string out = os.str();
|
||||
|
||||
@ -307,9 +347,16 @@ int ModApiUtil::l_decompress(lua_State *L)
|
||||
size_t size;
|
||||
const char *data = luaL_checklstring(L, 1, &size);
|
||||
|
||||
LuaCompressMethod method = get_compress_method(L, 2);
|
||||
|
||||
std::istringstream is(std::string(data, size), std::ios_base::binary);
|
||||
std::ostringstream os(std::ios_base::binary);
|
||||
decompressZlib(is, os);
|
||||
|
||||
if (method == LUA_COMPRESS_METHOD_DEFLATE) {
|
||||
decompressZlib(is, os);
|
||||
} else if (method == LUA_COMPRESS_METHOD_ZSTD) {
|
||||
decompressZstd(is, os);
|
||||
}
|
||||
|
||||
std::string out = os.str();
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user