Add support for map block version 29

BlockDecoder.cpp

@@ -1,4 +1,3 @@
-#include <stdint.h>
 #include <string>
 #include <iostream>
 #include <sstream>
@@ -39,7 +38,7 @@ void BlockDecoder::reset()
 
 	m_version = 0;
 	m_contentWidth = 0;
-	m_mapData = ustring();
+	m_mapData.clear();
 }
 
 void BlockDecoder::decode(const ustring &datastr)
@@ -49,7 +48,6 @@ void BlockDecoder::decode(const ustring &datastr)
 	// TODO: bounds checks
 
 	uint8_t version = data[0];
-	//uint8_t flags = data[1];
 	if (version < 22) {
 		std::ostringstream oss;
 		oss << "Unsupported map version " << (int)version;
@@ -57,12 +55,46 @@ void BlockDecoder::decode(const ustring &datastr)
 	}
 	m_version = version;
 
+	ustring datastr2;
+	if (version >= 29) {
+		// decompress whole block at once
+		m_zstd_decompressor.setData(data, length, 1);
+		datastr2 = m_zstd_decompressor.decompress();
+		data = datastr2.c_str();
+		length = datastr2.size();
+	}
+
 	size_t dataOffset = 0;
-	if (version >= 27)
+	if (version >= 29)
+		dataOffset = 7;
+	else if (version >= 27)
 		dataOffset = 4;
 	else
 		dataOffset = 2;
 
+	auto decode_mapping = [&] () {
+		dataOffset++; // mapping version
+		uint16_t numMappings = readU16(data + dataOffset);
+		dataOffset += 2;
+		for (int i = 0; i < numMappings; ++i) {
+			uint16_t nodeId = readU16(data + dataOffset);
+			dataOffset += 2;
+			uint16_t nameLen = readU16(data + dataOffset);
+			dataOffset += 2;
+			std::string name(reinterpret_cast<const char *>(data) + dataOffset, nameLen);
+			if (name == "air")
+				m_blockAirId = nodeId;
+			else if (name == "ignore")
+				m_blockIgnoreId = nodeId;
+			else
+				m_nameMap[nodeId] = name;
+			dataOffset += nameLen;
+		}
+	};
+
+	if (version >= 29)
+		decode_mapping();
+
 	uint8_t contentWidth = data[dataOffset];
 	dataOffset++;
 	uint8_t paramsWidth = data[dataOffset];
@@ -73,14 +105,20 @@ void BlockDecoder::decode(const ustring &datastr)
 		throw std::runtime_error("unsupported map version (paramsWidth)");
 	m_contentWidth = contentWidth;
 
+	if (version >= 29) {
+		m_mapData.resize((contentWidth + paramsWidth) * 4096);
+		m_mapData.assign(data + dataOffset, m_mapData.size());
+		return; // we have read everything we need and can return early
+	}
 
+	// version < 29
 	ZlibDecompressor decompressor(data, length);
 	decompressor.setSeekPos(dataOffset);
 	m_mapData = decompressor.decompress();
 	decompressor.decompress(); // unused metadata
 	dataOffset = decompressor.seekPos();
 
-	// Skip unused data
+	// Skip unused node timers
 	if (version == 23)
 		dataOffset += 1;
 	if (version == 24) {
@@ -104,33 +142,7 @@ void BlockDecoder::decode(const ustring &datastr)
 	dataOffset += 4; // Skip timestamp
 
 	// Read mapping
-	{
-		dataOffset++; // mapping version
-		uint16_t numMappings = readU16(data + dataOffset);
-		dataOffset += 2;
-		for (int i = 0; i < numMappings; ++i) {
-			uint16_t nodeId = readU16(data + dataOffset);
-			dataOffset += 2;
-			uint16_t nameLen = readU16(data + dataOffset);
-			dataOffset += 2;
-			std::string name(reinterpret_cast<const char *>(data) + dataOffset, nameLen);
-			if (name == "air")
-				m_blockAirId = nodeId;
-			else if (name == "ignore")
-				m_blockIgnoreId = nodeId;
-			else
-				m_nameMap[nodeId] = name;
-			dataOffset += nameLen;
-		}
-	}
-
-	// Node timers
-	if (version >= 25) {
-		uint8_t timerLength = data[dataOffset++];
-		uint16_t numTimers = readU16(data + dataOffset);
-		dataOffset += 2;
-		dataOffset += numTimers * timerLength;
-	}
+	decode_mapping();
 }
 
 bool BlockDecoder::isEmpty() const

CMakeLists.txt

@@ -43,7 +43,7 @@ if(NOT CUSTOM_DOCDIR STREQUAL "")
         message(STATUS "Using DOCDIR=${DOCDIR}")
 endif()
 
-#set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake)
+set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake)
 
 # Libraries: gd
 
@@ -59,6 +59,10 @@ endif(NOT LIBGD_LIBRARY OR NOT LIBGD_INCLUDE_DIR)
 
 find_package(ZLIB REQUIRED)
 
+# Libraries: zstd
+
+find_package(Zstd REQUIRED)
+
 # Libraries: sqlite3
 
 find_library(SQLITE3_LIBRARY sqlite3)
@@ -148,6 +152,7 @@ include_directories(
 	${SQLITE3_INCLUDE_DIR}
 	${LIBGD_INCLUDE_DIR}
 	${ZLIB_INCLUDE_DIR}
+	${ZSTD_INCLUDE_DIR}
 )
 
 configure_file(
@@ -171,6 +176,7 @@ add_executable(minetestmapper
 	PlayerAttributes.cpp
 	TileGenerator.cpp
 	ZlibDecompressor.cpp
+	ZstdDecompressor.cpp
 	Image.cpp
 	mapper.cpp
 	util.cpp
@@ -188,6 +194,7 @@ target_link_libraries(
 	${REDIS_LIBRARY}
 	${LIBGD_LIBRARY}
 	${ZLIB_LIBRARY}
+	${ZSTD_LIBRARY}
 )
 
 # Installing & Packaging

ZstdDecompressor.cpp

@@ -0,0 +1,58 @@
+#include <zstd.h>
+#include "ZstdDecompressor.h"
+
+ZstdDecompressor::ZstdDecompressor():
+	m_data(nullptr),
+	m_seekPos(0),
+	m_size(0)
+{
+	m_stream = ZSTD_createDStream();
+}
+
+ZstdDecompressor::~ZstdDecompressor()
+{
+	ZSTD_freeDStream(reinterpret_cast<ZSTD_DStream*>(m_stream));
+}
+
+void ZstdDecompressor::setData(const u8 *data, size_t size, size_t seekPos)
+{
+	m_data = data;
+	m_seekPos = seekPos;
+	m_size = size;
+}
+
+std::size_t ZstdDecompressor::seekPos() const
+{
+	return m_seekPos;
+}
+
+ustring ZstdDecompressor::decompress()
+{
+	ZSTD_DStream *stream = reinterpret_cast<ZSTD_DStream*>(m_stream);
+	ZSTD_inBuffer inbuf = { m_data, m_size, m_seekPos };
+
+	ustring buffer;
+	constexpr size_t BUFSIZE = 32 * 1024;
+
+	buffer.resize(BUFSIZE);
+	ZSTD_outBuffer outbuf = { &buffer[0], buffer.size(), 0 };
+
+	ZSTD_initDStream(stream);
+
+	size_t ret;
+	do {
+		ret = ZSTD_decompressStream(stream, &outbuf, &inbuf);
+		if (outbuf.size == outbuf.pos) {
+			outbuf.size += BUFSIZE;
+			buffer.resize(outbuf.size);
+			outbuf.dst = &buffer[0];
+		}
+	} while (ret != 0);
+	if (ZSTD_isError(ret))
+		throw DecompressError();
+
+	m_seekPos = inbuf.pos;
+	buffer.resize(outbuf.pos);
+
+	return buffer;
+}

cmake/FindZstd.cmake

@@ -0,0 +1,24 @@
+mark_as_advanced(ZSTD_LIBRARY ZSTD_INCLUDE_DIR)
+
+find_path(ZSTD_INCLUDE_DIR NAMES zstd.h)
+
+find_library(ZSTD_LIBRARY NAMES zstd)
+
+if(ZSTD_INCLUDE_DIR AND ZSTD_LIBRARY)
+	# Check that the API we use exists
+	include(CheckSymbolExists)
+	unset(HAVE_ZSTD_INITDSTREAM CACHE)
+	set(CMAKE_REQUIRED_INCLUDES ${ZSTD_INCLUDE_DIR})
+	set(CMAKE_REQUIRED_LIBRARIES ${ZSTD_LIBRARY})
+	check_symbol_exists(ZSTD_initDStream zstd.h HAVE_ZSTD_INITDSTREAM)
+	unset(CMAKE_REQUIRED_INCLUDES)
+	unset(CMAKE_REQUIRED_LIBRARIES)
+
+	if(NOT HAVE_ZSTD_INITDSTREAM)
+		unset(ZSTD_INCLUDE_DIR CACHE)
+		unset(ZSTD_LIBRARY CACHE)
+	endif()
+endif()
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(Zstd DEFAULT_MSG ZSTD_LIBRARY ZSTD_INCLUDE_DIR)

include/BlockDecoder.h

@@ -1,8 +1,9 @@
 #pragma once
 
+#include <cstdint>
 #include <unordered_map>
-
 #include "types.h"
+#include <ZstdDecompressor.h>
 
 class BlockDecoder {
 public:
@@ -17,9 +18,11 @@ public:
 private:
 	typedef std::unordered_map<uint16_t, std::string> NameMap;
 	NameMap m_nameMap;
-	int m_blockAirId;
-	int m_blockIgnoreId;
+	uint16_t m_blockAirId, m_blockIgnoreId;
 
 	u8 m_version, m_contentWidth;
 	ustring m_mapData;
+
+	// one instance for performance
+	ZstdDecompressor m_zstd_decompressor;
 };

include/ZstdDecompressor.h

@@ -0,0 +1,22 @@
+#pragma once
+
+#include <cstdlib>
+#include <string>
+#include "types.h"
+
+class ZstdDecompressor
+{
+public:
+	class DecompressError {};
+
+	ZstdDecompressor();
+	~ZstdDecompressor();
+	void setData(const u8 *data, size_t size, size_t seekPos);
+	size_t seekPos() const;
+	ustring decompress();
+
+private:
+	void *m_stream; // ZSTD_DStream
+	const u8 *m_data;
+	size_t m_seekPos, m_size;
+};

util/ci/script.sh

@@ -1,7 +1,7 @@
 #!/bin/bash -e
 
 install_linux_deps() {
-	local pkgs=(cmake libgd-dev libsqlite3-dev libleveldb-dev libpq-dev libhiredis-dev)
+	local pkgs=(cmake libgd-dev libsqlite3-dev libleveldb-dev libpq-dev libhiredis-dev libzstd-dev)
 
 	sudo apt-get update
 	sudo apt-get install -y --no-install-recommends ${pkgs[@]} "$@"