Add zstd compression support (#12515)

builtin/game/features.lua

@@ -26,6 +26,7 @@ core.features = {
 	get_sky_as_table = true,
 	get_light_data_buffer = true,
 	mod_storage_on_disk = true,
+	compress_zstd = true,
 }
 
 function core.has_feature(arg)

doc/client_lua_api.txt

@@ -898,13 +898,18 @@ Call these functions only at load time!
     * `method` is a string identifying the compression method to be used.
     * Supported compression methods:
         * Deflate (zlib): `"deflate"`
-    * `...` indicates method-specific arguments.  Currently defined arguments are:
+        * Zstandard: `"zstd"`
+    * `...` indicates method-specific arguments. Currently defined arguments
+      are:
         * Deflate: `level` - Compression level, `0`-`9` or `nil`.
+        * Zstandard: `level` - Compression level. Integer or `nil`. Default `3`.
+        Note any supported Zstandard compression level could be used here,
+        but these are subject to change between Zstandard versions.
 * `minetest.decompress(compressed_data, method, ...)`: returns data
-    * Decompress a string of data (using ZLib).
-    * See documentation on `minetest.compress()` for supported compression methods.
-    * currently supported.
-    * `...` indicates method-specific arguments. Currently, no methods use this.
+    * Decompress a string of data using the algorithm specified by `method`.
+    * See documentation on `minetest.compress()` for supported compression
+      methods.
+    * `...` indicates method-specific arguments. Currently, no methods use this
 * `minetest.rgba(red, green, blue[, alpha])`: returns a string
     * Each argument is a 8 Bit unsigned integer
     * Returns the ColorString from rgb or rgba values

doc/lua_api.txt

@@ -4910,6 +4910,8 @@ Utilities
           -- the amount of data in mod storage is not constrained by
           -- the amount of RAM available. (5.7.0)
           mod_storage_on_disk = true,
+          -- "zstd" method for compress/decompress (5.7.0)
+          compress_zstd = true,
       }
 
 * `minetest.has_feature(arg)`: returns `boolean, missing_features`
@@ -6370,11 +6372,15 @@ Misc.
     * `method` is a string identifying the compression method to be used.
     * Supported compression methods:
         * Deflate (zlib): `"deflate"`
+        * Zstandard: `"zstd"`
     * `...` indicates method-specific arguments. Currently defined arguments
       are:
         * Deflate: `level` - Compression level, `0`-`9` or `nil`.
+        * Zstandard: `level` - Compression level. Integer or `nil`. Default `3`.
+        Note any supported Zstandard compression level could be used here,
+        but these are subject to change between Zstandard versions.
 * `minetest.decompress(compressed_data, method, ...)`: returns data
-    * Decompress a string of data (using ZLib).
+    * Decompress a string of data using the algorithm specified by `method`.
     * See documentation on `minetest.compress()` for supported compression
       methods.
     * `...` indicates method-specific arguments. Currently, no methods use this

games/devtest/mods/unittests/misc.lua

@@ -80,3 +80,26 @@ unittests.register("test_punch_node", function(_, pos)
 	minetest.remove_node(pos)
 	-- currently failing: assert(on_punch_called)
 end, {map=true})
+
+local function test_compress()
+	-- This text should be compressible, to make sure the results are... normal
+	local text = "The\000 icey canoe couldn't move very well on the\128 lake. The\000 ice was too stiff and the icey canoe's paddles simply wouldn't punch through."
+	local methods = {
+		"deflate",
+		"zstd",
+		-- "noodle", -- for warning alarm test
+	}
+	local zstd_magic = string.char(0x28, 0xB5, 0x2F, 0xFD)
+	for _, method in ipairs(methods) do
+		local compressed = core.compress(text, method)
+		assert(core.decompress(compressed, method) == text, "input/output mismatch for compression method " .. method)
+		local has_zstd_magic = compressed:sub(1, 4) == zstd_magic
+		if method == "zstd" then
+			assert(has_zstd_magic, "zstd magic number not in zstd method")
+		else
+			assert(not has_zstd_magic, "zstd magic number in method " .. method .. " (which is not zstd)")
+		end
+	end
+end
+unittests.register("test_compress", test_compress)
+

src/script/lua_api/l_util.cpp

@@ -26,6 +26,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "cpp_api/s_async.h"
 #include "serialization.h"
 #include <json/json.h>
+#include <zstd.h>
 #include "cpp_api/s_security.h"
 #include "porting.h"
 #include "convert_json.h"
@@ -278,6 +279,34 @@ int ModApiUtil::l_get_user_path(lua_State *L)
 	return 1;
 }
 
+enum LuaCompressMethod
+{
+	LUA_COMPRESS_METHOD_DEFLATE,
+	LUA_COMPRESS_METHOD_ZSTD,
+};
+
+static const struct EnumString es_LuaCompressMethod[] =
+{
+	{LUA_COMPRESS_METHOD_DEFLATE, "deflate"},
+	{LUA_COMPRESS_METHOD_ZSTD, "zstd"},
+	{0, nullptr},
+};
+
+static LuaCompressMethod get_compress_method(lua_State *L, int index)
+{
+	if (lua_isnoneornil(L, index))
+		return LUA_COMPRESS_METHOD_DEFLATE;
+	const char *name = luaL_checkstring(L, index);
+	int value;
+	if (!string_to_enum(es_LuaCompressMethod, value, name)) {
+		// Pretend it's deflate if we don't know, for compatibility reasons.
+		log_deprecated(L, "Unknown compression method \"" + std::string(name)
+			+ "\", defaulting to \"deflate\". You should pass a valid value.");
+		return LUA_COMPRESS_METHOD_DEFLATE;
+	}
+	return (LuaCompressMethod) value;
+}
+
 // compress(data, method, level)
 int ModApiUtil::l_compress(lua_State *L)
 {
@@ -286,12 +315,23 @@ int ModApiUtil::l_compress(lua_State *L)
 	size_t size;
 	const char *data = luaL_checklstring(L, 1, &size);
 
+	LuaCompressMethod method = get_compress_method(L, 2);
+
+	std::ostringstream os(std::ios_base::binary);
+
+	if (method == LUA_COMPRESS_METHOD_DEFLATE) {
 		int level = -1;
 		if (!lua_isnoneornil(L, 3))
 			level = readParam<int>(L, 3);
 
-	std::ostringstream os(std::ios_base::binary);
 		compressZlib(reinterpret_cast<const u8 *>(data), size, os, level);
+	} else if (method == LUA_COMPRESS_METHOD_ZSTD) {
+		int level = ZSTD_CLEVEL_DEFAULT;
+		if (!lua_isnoneornil(L, 3))
+			level = readParam<int>(L, 3);
+
+		compressZstd(reinterpret_cast<const u8 *>(data), size, os, level);
+	}
 
 	std::string out = os.str();
 
@@ -307,9 +347,16 @@ int ModApiUtil::l_decompress(lua_State *L)
 	size_t size;
 	const char *data = luaL_checklstring(L, 1, &size);
 
+	LuaCompressMethod method = get_compress_method(L, 2);
+
 	std::istringstream is(std::string(data, size), std::ios_base::binary);
 	std::ostringstream os(std::ios_base::binary);
+
+	if (method == LUA_COMPRESS_METHOD_DEFLATE) {
 		decompressZlib(is, os);
+	} else if (method == LUA_COMPRESS_METHOD_ZSTD) {
+		decompressZstd(is, os);
+	}
 
 	std::string out = os.str();