From e6380565236d6d963acf75538f1f8fec807190cc Mon Sep 17 00:00:00 2001 From: Lars Date: Wed, 9 Dec 2020 14:30:37 -0800 Subject: [PATCH] Allow configuring block disk and net compression. Change default disk level. --- builtin/settingtypes.txt | 14 ++++++++++++++ src/defaultsettings.cpp | 4 ++++ src/map.cpp | 8 +++++--- src/map.h | 3 ++- src/mapblock.cpp | 10 +++++----- src/mapblock.h | 2 +- src/mapgen/mg_schematic.cpp | 4 ++-- src/mapnode.cpp | 34 ++++++++++------------------------ src/mapnode.h | 4 ++-- src/server.cpp | 4 ++-- 10 files changed, 47 insertions(+), 40 deletions(-) diff --git a/builtin/settingtypes.txt b/builtin/settingtypes.txt index c9f16578c..251b6d868 100644 --- a/builtin/settingtypes.txt +++ b/builtin/settingtypes.txt @@ -1048,6 +1048,13 @@ full_block_send_enable_min_time_from_building (Delay in sending blocks after bui # client number. max_packets_per_iteration (Max. packets per iteration) int 1024 +# ZLib compression level to use when sending mapblocks to the client. +# -1 - Zlib's default compression level +# 0 - no compresson, fastest +# 9 - best compression, slowest +# (levels 1-3 use Zlib's "fast" method, 4-9 use the normal method) +map_compression_level_net (Map Compression Level for Network Transfer) int -1 -1 9 + [*Game] # Default game when creating a new world. @@ -1240,6 +1247,13 @@ max_objects_per_block (Maximum objects per block) int 64 # See https://www.sqlite.org/pragma.html#pragma_synchronous sqlite_synchronous (Synchronous SQLite) enum 2 0,1,2 +# ZLib compression level to use when saving mapblocks to disk. +# -1 - Zlib's default compression level +# 0 - no compresson, fastest +# 9 - best compression, slowest +# (levels 1-3 use Zlib's "fast" method, 4-9 use the normal method) +map_compression_level_disk (Map Compression Level for Disk Storage) int 3 -1 9 + # Length of a server tick and the interval at which objects are generally updated over # network. dedicated_server_step (Dedicated server step) float 0.09 diff --git a/src/defaultsettings.cpp b/src/defaultsettings.cpp index 177955589..e13977fe3 100644 --- a/src/defaultsettings.cpp +++ b/src/defaultsettings.cpp @@ -385,6 +385,8 @@ void set_default_settings(Settings *settings) settings->setDefault("chat_message_limit_per_10sec", "8.0"); settings->setDefault("chat_message_limit_trigger_kick", "50"); settings->setDefault("sqlite_synchronous", "2"); + settings->setDefault("map_compression_level_disk", "3"); + settings->setDefault("map_compression_level_net", "-1"); settings->setDefault("full_block_send_enable_min_time_from_building", "2.0"); settings->setDefault("dedicated_server_step", "0.09"); settings->setDefault("active_block_mgmt_interval", "2.0"); @@ -470,6 +472,8 @@ void set_default_settings(Settings *settings) settings->setDefault("fps_max_unfocused", "10"); settings->setDefault("max_objects_per_block", "20"); settings->setDefault("sqlite_synchronous", "1"); + settings->setDefault("map_compression_level_disk", "-1"); + settings->setDefault("map_compression_level_net", "3"); settings->setDefault("server_map_save_interval", "15"); settings->setDefault("client_mapblock_limit", "1000"); settings->setDefault("active_block_range", "2"); diff --git a/src/map.cpp b/src/map.cpp index 37b6e9b6b..6a7cadca5 100644 --- a/src/map.cpp +++ b/src/map.cpp @@ -1235,6 +1235,8 @@ ServerMap::ServerMap(const std::string &savedir, IGameDef *gamedef, m_save_time_counter = mb->addCounter("minetest_core_map_save_time", "Map save time (in nanoseconds)"); + m_map_compression_level = rangelim(g_settings->getS16("map_compression_level_disk"), -1, 9); + try { // If directory exists, check contents and load if possible if (fs::PathExists(m_savedir)) { @@ -1863,10 +1865,10 @@ void ServerMap::endSave() bool ServerMap::saveBlock(MapBlock *block) { - return saveBlock(block, dbase); + return saveBlock(block, dbase, m_map_compression_level); } -bool ServerMap::saveBlock(MapBlock *block, MapDatabase *db) +bool ServerMap::saveBlock(MapBlock *block, MapDatabase *db, int compression_level) { v3s16 p3d = block->getPos(); @@ -1886,7 +1888,7 @@ bool ServerMap::saveBlock(MapBlock *block, MapDatabase *db) */ std::ostringstream o(std::ios_base::binary); o.write((char*) &version, 1); - block->serialize(o, version, true); + block->serialize(o, version, true, compression_level); bool ret = db->saveBlock(p3d, o.str()); if (ret) { diff --git a/src/map.h b/src/map.h index 3bc30c482..c8bae9451 100644 --- a/src/map.h +++ b/src/map.h @@ -381,7 +381,7 @@ public: MapgenParams *getMapgenParams(); bool saveBlock(MapBlock *block); - static bool saveBlock(MapBlock *block, MapDatabase *db); + static bool saveBlock(MapBlock *block, MapDatabase *db, int compression_level = -1); MapBlock* loadBlock(v3s16 p); // Database version void loadBlock(std::string *blob, v3s16 p3d, MapSector *sector, bool save_after_load=false); @@ -416,6 +416,7 @@ private: std::string m_savedir; bool m_map_saving_enabled; + int m_map_compression_level; #if 0 // Chunk size in MapSectors // If 0, chunks are disabled. diff --git a/src/mapblock.cpp b/src/mapblock.cpp index 8bfecd755..0ca71e643 100644 --- a/src/mapblock.cpp +++ b/src/mapblock.cpp @@ -355,7 +355,7 @@ static void correctBlockNodeIds(const NameIdMapping *nimap, MapNode *nodes, } } -void MapBlock::serialize(std::ostream &os, u8 version, bool disk) +void MapBlock::serialize(std::ostream &os, u8 version, bool disk, int compression_level) { if(!ser_ver_supported(version)) throw VersionMismatchException("ERROR: MapBlock format not supported"); @@ -394,7 +394,7 @@ void MapBlock::serialize(std::ostream &os, u8 version, bool disk) writeU8(os, content_width); writeU8(os, params_width); MapNode::serializeBulk(os, version, tmp_nodes, nodecount, - content_width, params_width, true); + content_width, params_width, compression_level); delete[] tmp_nodes; } else @@ -404,7 +404,7 @@ void MapBlock::serialize(std::ostream &os, u8 version, bool disk) writeU8(os, content_width); writeU8(os, params_width); MapNode::serializeBulk(os, version, data, nodecount, - content_width, params_width, true); + content_width, params_width, compression_level); } /* @@ -412,7 +412,7 @@ void MapBlock::serialize(std::ostream &os, u8 version, bool disk) */ std::ostringstream oss(std::ios_base::binary); m_node_metadata.serialize(oss, version, disk); - compressZlib(oss.str(), os); + compressZlib(oss.str(), os, compression_level); /* Data that goes to disk, but not the network @@ -485,7 +485,7 @@ void MapBlock::deSerialize(std::istream &is, u8 version, bool disk) if(params_width != 2) throw SerializationError("MapBlock::deSerialize(): invalid params_width"); MapNode::deSerializeBulk(is, version, data, nodecount, - content_width, params_width, true); + content_width, params_width); /* NodeMetadata diff --git a/src/mapblock.h b/src/mapblock.h index 6b5015cab..641a1b69b 100644 --- a/src/mapblock.h +++ b/src/mapblock.h @@ -482,7 +482,7 @@ public: // These don't write or read version by itself // Set disk to true for on-disk format, false for over-the-network format // Precondition: version >= SER_FMT_VER_LOWEST_WRITE - void serialize(std::ostream &os, u8 version, bool disk); + void serialize(std::ostream &os, u8 version, bool disk, int compression_level); // If disk == true: In addition to doing other things, will add // unknown blocks from id-name mapping to wndef void deSerialize(std::istream &is, u8 version, bool disk); diff --git a/src/mapgen/mg_schematic.cpp b/src/mapgen/mg_schematic.cpp index dfd414709..e70e97e48 100644 --- a/src/mapgen/mg_schematic.cpp +++ b/src/mapgen/mg_schematic.cpp @@ -334,7 +334,7 @@ bool Schematic::deserializeFromMts(std::istream *is, schemdata = new MapNode[nodecount]; MapNode::deSerializeBulk(ss, SER_FMT_VER_HIGHEST_READ, schemdata, - nodecount, 2, 2, true); + nodecount, 2, 2); // Fix probability values for nodes that were ignore; removed in v2 if (version < 2) { @@ -376,7 +376,7 @@ bool Schematic::serializeToMts(std::ostream *os, // compressed bulk node data MapNode::serializeBulk(ss, SER_FMT_VER_HIGHEST_WRITE, - schemdata, size.X * size.Y * size.Z, 2, 2, true); + schemdata, size.X * size.Y * size.Z, 2, 2, -1); return true; } diff --git a/src/mapnode.cpp b/src/mapnode.cpp index dcf1f6d6e..0551f3b6f 100644 --- a/src/mapnode.cpp +++ b/src/mapnode.cpp @@ -706,7 +706,7 @@ void MapNode::deSerialize(u8 *source, u8 version) } void MapNode::serializeBulk(std::ostream &os, int version, const MapNode *nodes, u32 nodecount, - u8 content_width, u8 params_width, bool compressed) + u8 content_width, u8 params_width, int compression_level) { if (!ser_ver_supported(version)) throw VersionMismatchException("ERROR: MapNode format not supported"); @@ -737,10 +737,7 @@ void MapNode::serializeBulk(std::ostream &os, int version, Compress data to output stream */ - if (compressed) - compressZlib(databuf, databuf_size, os); - else - os.write((const char*) &databuf[0], databuf_size); + compressZlib(databuf, databuf_size, os, compression_level); delete [] databuf; } @@ -748,7 +745,7 @@ void MapNode::serializeBulk(std::ostream &os, int version, // Deserialize bulk node data void MapNode::deSerializeBulk(std::istream &is, int version, MapNode *nodes, u32 nodecount, - u8 content_width, u8 params_width, bool compressed) + u8 content_width, u8 params_width) { if(!ser_ver_supported(version)) throw VersionMismatchException("ERROR: MapNode format not supported"); @@ -760,24 +757,13 @@ void MapNode::deSerializeBulk(std::istream &is, int version, // Uncompress or read data u32 len = nodecount * (content_width + params_width); - SharedBuffer databuf(len); - if(compressed) - { - std::ostringstream os(std::ios_base::binary); - decompressZlib(is, os); - std::string s = os.str(); - if(s.size() != len) - throw SerializationError("deSerializeBulkNodes: " - "decompress resulted in invalid size"); - memcpy(&databuf[0], s.c_str(), len); - } - else - { - is.read((char*) &databuf[0], len); - if(is.eof() || is.fail()) - throw SerializationError("deSerializeBulkNodes: " - "failed to read bulk node data"); - } + std::ostringstream os(std::ios_base::binary); + decompressZlib(is, os); + std::string s = os.str(); + if(s.size() != len) + throw SerializationError("deSerializeBulkNodes: " + "decompress resulted in invalid size"); + const u8 *databuf = reinterpret_cast(s.c_str()); // Deserialize content if(content_width == 1) diff --git a/src/mapnode.h b/src/mapnode.h index 32ac1b4f6..a9ae63ba3 100644 --- a/src/mapnode.h +++ b/src/mapnode.h @@ -292,10 +292,10 @@ struct MapNode // compressed = true to zlib-compress output static void serializeBulk(std::ostream &os, int version, const MapNode *nodes, u32 nodecount, - u8 content_width, u8 params_width, bool compressed); + u8 content_width, u8 params_width, int compression_level); static void deSerializeBulk(std::istream &is, int version, MapNode *nodes, u32 nodecount, - u8 content_width, u8 params_width, bool compressed); + u8 content_width, u8 params_width); private: // Deprecated serialization methods diff --git a/src/server.cpp b/src/server.cpp index 8f6257afe..b5352749c 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -2332,9 +2332,9 @@ void Server::SendBlockNoLock(session_t peer_id, MapBlock *block, u8 ver, /* Create a packet with the block in the right format */ - + thread_local const int net_compression_level = rangelim(g_settings->getS16("map_compression_level_net"), -1, 9); std::ostringstream os(std::ios_base::binary); - block->serialize(os, ver, false); + block->serialize(os, ver, false, net_compression_level); block->serializeNetworkSpecific(os); std::string s = os.str();