From b0ca3d706604d6c1878f03990fa36345872df664 Mon Sep 17 00:00:00 2001 From: sfan5 Date: Wed, 1 Sep 2021 23:55:32 +0200 Subject: [PATCH] Add support for map block version 29 --- BlockDecoder.cpp | 76 ++++++++++++++++++++++---------------- CMakeLists.txt | 9 ++++- ZstdDecompressor.cpp | 58 +++++++++++++++++++++++++++++ cmake/FindZstd.cmake | 24 ++++++++++++ include/BlockDecoder.h | 9 +++-- include/ZstdDecompressor.h | 22 +++++++++++ util/ci/script.sh | 2 +- 7 files changed, 163 insertions(+), 37 deletions(-) create mode 100644 ZstdDecompressor.cpp create mode 100644 cmake/FindZstd.cmake create mode 100644 include/ZstdDecompressor.h diff --git a/BlockDecoder.cpp b/BlockDecoder.cpp index a904007..b0fb41e 100644 --- a/BlockDecoder.cpp +++ b/BlockDecoder.cpp @@ -1,4 +1,3 @@ -#include #include #include #include @@ -39,7 +38,7 @@ void BlockDecoder::reset() m_version = 0; m_contentWidth = 0; - m_mapData = ustring(); + m_mapData.clear(); } void BlockDecoder::decode(const ustring &datastr) @@ -49,7 +48,6 @@ void BlockDecoder::decode(const ustring &datastr) // TODO: bounds checks uint8_t version = data[0]; - //uint8_t flags = data[1]; if (version < 22) { std::ostringstream oss; oss << "Unsupported map version " << (int)version; @@ -57,12 +55,46 @@ void BlockDecoder::decode(const ustring &datastr) } m_version = version; + ustring datastr2; + if (version >= 29) { + // decompress whole block at once + m_zstd_decompressor.setData(data, length, 1); + datastr2 = m_zstd_decompressor.decompress(); + data = datastr2.c_str(); + length = datastr2.size(); + } + size_t dataOffset = 0; - if (version >= 27) + if (version >= 29) + dataOffset = 7; + else if (version >= 27) dataOffset = 4; else dataOffset = 2; + auto decode_mapping = [&] () { + dataOffset++; // mapping version + uint16_t numMappings = readU16(data + dataOffset); + dataOffset += 2; + for (int i = 0; i < numMappings; ++i) { + uint16_t nodeId = readU16(data + dataOffset); + dataOffset += 2; + uint16_t nameLen = readU16(data + dataOffset); + dataOffset += 2; + std::string name(reinterpret_cast(data) + dataOffset, nameLen); + if (name == "air") + m_blockAirId = nodeId; + else if (name == "ignore") + m_blockIgnoreId = nodeId; + else + m_nameMap[nodeId] = name; + dataOffset += nameLen; + } + }; + + if (version >= 29) + decode_mapping(); + uint8_t contentWidth = data[dataOffset]; dataOffset++; uint8_t paramsWidth = data[dataOffset]; @@ -73,14 +105,20 @@ void BlockDecoder::decode(const ustring &datastr) throw std::runtime_error("unsupported map version (paramsWidth)"); m_contentWidth = contentWidth; + if (version >= 29) { + m_mapData.resize((contentWidth + paramsWidth) * 4096); + m_mapData.assign(data + dataOffset, m_mapData.size()); + return; // we have read everything we need and can return early + } + // version < 29 ZlibDecompressor decompressor(data, length); decompressor.setSeekPos(dataOffset); m_mapData = decompressor.decompress(); decompressor.decompress(); // unused metadata dataOffset = decompressor.seekPos(); - // Skip unused data + // Skip unused node timers if (version == 23) dataOffset += 1; if (version == 24) { @@ -104,33 +142,7 @@ void BlockDecoder::decode(const ustring &datastr) dataOffset += 4; // Skip timestamp // Read mapping - { - dataOffset++; // mapping version - uint16_t numMappings = readU16(data + dataOffset); - dataOffset += 2; - for (int i = 0; i < numMappings; ++i) { - uint16_t nodeId = readU16(data + dataOffset); - dataOffset += 2; - uint16_t nameLen = readU16(data + dataOffset); - dataOffset += 2; - std::string name(reinterpret_cast(data) + dataOffset, nameLen); - if (name == "air") - m_blockAirId = nodeId; - else if (name == "ignore") - m_blockIgnoreId = nodeId; - else - m_nameMap[nodeId] = name; - dataOffset += nameLen; - } - } - - // Node timers - if (version >= 25) { - uint8_t timerLength = data[dataOffset++]; - uint16_t numTimers = readU16(data + dataOffset); - dataOffset += 2; - dataOffset += numTimers * timerLength; - } + decode_mapping(); } bool BlockDecoder::isEmpty() const diff --git a/CMakeLists.txt b/CMakeLists.txt index 67b0667..f1a3b82 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -43,7 +43,7 @@ if(NOT CUSTOM_DOCDIR STREQUAL "") message(STATUS "Using DOCDIR=${DOCDIR}") endif() -#set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake) +set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake) # Libraries: gd @@ -59,6 +59,10 @@ endif(NOT LIBGD_LIBRARY OR NOT LIBGD_INCLUDE_DIR) find_package(ZLIB REQUIRED) +# Libraries: zstd + +find_package(Zstd REQUIRED) + # Libraries: sqlite3 find_library(SQLITE3_LIBRARY sqlite3) @@ -148,6 +152,7 @@ include_directories( ${SQLITE3_INCLUDE_DIR} ${LIBGD_INCLUDE_DIR} ${ZLIB_INCLUDE_DIR} + ${ZSTD_INCLUDE_DIR} ) configure_file( @@ -171,6 +176,7 @@ add_executable(minetestmapper PlayerAttributes.cpp TileGenerator.cpp ZlibDecompressor.cpp + ZstdDecompressor.cpp Image.cpp mapper.cpp util.cpp @@ -188,6 +194,7 @@ target_link_libraries( ${REDIS_LIBRARY} ${LIBGD_LIBRARY} ${ZLIB_LIBRARY} + ${ZSTD_LIBRARY} ) # Installing & Packaging diff --git a/ZstdDecompressor.cpp b/ZstdDecompressor.cpp new file mode 100644 index 0000000..950af73 --- /dev/null +++ b/ZstdDecompressor.cpp @@ -0,0 +1,58 @@ +#include +#include "ZstdDecompressor.h" + +ZstdDecompressor::ZstdDecompressor(): + m_data(nullptr), + m_seekPos(0), + m_size(0) +{ + m_stream = ZSTD_createDStream(); +} + +ZstdDecompressor::~ZstdDecompressor() +{ + ZSTD_freeDStream(reinterpret_cast(m_stream)); +} + +void ZstdDecompressor::setData(const u8 *data, size_t size, size_t seekPos) +{ + m_data = data; + m_seekPos = seekPos; + m_size = size; +} + +std::size_t ZstdDecompressor::seekPos() const +{ + return m_seekPos; +} + +ustring ZstdDecompressor::decompress() +{ + ZSTD_DStream *stream = reinterpret_cast(m_stream); + ZSTD_inBuffer inbuf = { m_data, m_size, m_seekPos }; + + ustring buffer; + constexpr size_t BUFSIZE = 32 * 1024; + + buffer.resize(BUFSIZE); + ZSTD_outBuffer outbuf = { &buffer[0], buffer.size(), 0 }; + + ZSTD_initDStream(stream); + + size_t ret; + do { + ret = ZSTD_decompressStream(stream, &outbuf, &inbuf); + if (outbuf.size == outbuf.pos) { + outbuf.size += BUFSIZE; + buffer.resize(outbuf.size); + outbuf.dst = &buffer[0]; + } + } while (ret != 0); + if (ZSTD_isError(ret)) + throw DecompressError(); + + m_seekPos = inbuf.pos; + buffer.resize(outbuf.pos); + + return buffer; +} diff --git a/cmake/FindZstd.cmake b/cmake/FindZstd.cmake new file mode 100644 index 0000000..534de10 --- /dev/null +++ b/cmake/FindZstd.cmake @@ -0,0 +1,24 @@ +mark_as_advanced(ZSTD_LIBRARY ZSTD_INCLUDE_DIR) + +find_path(ZSTD_INCLUDE_DIR NAMES zstd.h) + +find_library(ZSTD_LIBRARY NAMES zstd) + +if(ZSTD_INCLUDE_DIR AND ZSTD_LIBRARY) + # Check that the API we use exists + include(CheckSymbolExists) + unset(HAVE_ZSTD_INITDSTREAM CACHE) + set(CMAKE_REQUIRED_INCLUDES ${ZSTD_INCLUDE_DIR}) + set(CMAKE_REQUIRED_LIBRARIES ${ZSTD_LIBRARY}) + check_symbol_exists(ZSTD_initDStream zstd.h HAVE_ZSTD_INITDSTREAM) + unset(CMAKE_REQUIRED_INCLUDES) + unset(CMAKE_REQUIRED_LIBRARIES) + + if(NOT HAVE_ZSTD_INITDSTREAM) + unset(ZSTD_INCLUDE_DIR CACHE) + unset(ZSTD_LIBRARY CACHE) + endif() +endif() + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(Zstd DEFAULT_MSG ZSTD_LIBRARY ZSTD_INCLUDE_DIR) diff --git a/include/BlockDecoder.h b/include/BlockDecoder.h index 17992f5..eae6e69 100644 --- a/include/BlockDecoder.h +++ b/include/BlockDecoder.h @@ -1,8 +1,9 @@ #pragma once +#include #include - #include "types.h" +#include class BlockDecoder { public: @@ -17,9 +18,11 @@ public: private: typedef std::unordered_map NameMap; NameMap m_nameMap; - int m_blockAirId; - int m_blockIgnoreId; + uint16_t m_blockAirId, m_blockIgnoreId; u8 m_version, m_contentWidth; ustring m_mapData; + + // one instance for performance + ZstdDecompressor m_zstd_decompressor; }; diff --git a/include/ZstdDecompressor.h b/include/ZstdDecompressor.h new file mode 100644 index 0000000..653bf43 --- /dev/null +++ b/include/ZstdDecompressor.h @@ -0,0 +1,22 @@ +#pragma once + +#include +#include +#include "types.h" + +class ZstdDecompressor +{ +public: + class DecompressError {}; + + ZstdDecompressor(); + ~ZstdDecompressor(); + void setData(const u8 *data, size_t size, size_t seekPos); + size_t seekPos() const; + ustring decompress(); + +private: + void *m_stream; // ZSTD_DStream + const u8 *m_data; + size_t m_seekPos, m_size; +}; diff --git a/util/ci/script.sh b/util/ci/script.sh index 3d8d35b..4557bd7 100755 --- a/util/ci/script.sh +++ b/util/ci/script.sh @@ -1,7 +1,7 @@ #!/bin/bash -e install_linux_deps() { - local pkgs=(cmake libgd-dev libsqlite3-dev libleveldb-dev libpq-dev libhiredis-dev) + local pkgs=(cmake libgd-dev libsqlite3-dev libleveldb-dev libpq-dev libhiredis-dev libzstd-dev) sudo apt-get update sudo apt-get install -y --no-install-recommends ${pkgs[@]} "$@"