Allow configuring block disk and net compression. Change default disk level.

This commit is contained in:
Lars 2020-12-09 14:30:37 -08:00 committed by lhofhansl
parent d0a38f694d
commit e638056523
10 changed files with 47 additions and 40 deletions

@ -1048,6 +1048,13 @@ full_block_send_enable_min_time_from_building (Delay in sending blocks after bui
# client number.
max_packets_per_iteration (Max. packets per iteration) int 1024
# ZLib compression level to use when sending mapblocks to the client.
# -1 - Zlib's default compression level
# 0 - no compresson, fastest
# 9 - best compression, slowest
# (levels 1-3 use Zlib's "fast" method, 4-9 use the normal method)
map_compression_level_net (Map Compression Level for Network Transfer) int -1 -1 9
[*Game]
# Default game when creating a new world.
@ -1240,6 +1247,13 @@ max_objects_per_block (Maximum objects per block) int 64
# See https://www.sqlite.org/pragma.html#pragma_synchronous
sqlite_synchronous (Synchronous SQLite) enum 2 0,1,2
# ZLib compression level to use when saving mapblocks to disk.
# -1 - Zlib's default compression level
# 0 - no compresson, fastest
# 9 - best compression, slowest
# (levels 1-3 use Zlib's "fast" method, 4-9 use the normal method)
map_compression_level_disk (Map Compression Level for Disk Storage) int 3 -1 9
# Length of a server tick and the interval at which objects are generally updated over
# network.
dedicated_server_step (Dedicated server step) float 0.09

@ -385,6 +385,8 @@ void set_default_settings(Settings *settings)
settings->setDefault("chat_message_limit_per_10sec", "8.0");
settings->setDefault("chat_message_limit_trigger_kick", "50");
settings->setDefault("sqlite_synchronous", "2");
settings->setDefault("map_compression_level_disk", "3");
settings->setDefault("map_compression_level_net", "-1");
settings->setDefault("full_block_send_enable_min_time_from_building", "2.0");
settings->setDefault("dedicated_server_step", "0.09");
settings->setDefault("active_block_mgmt_interval", "2.0");
@ -470,6 +472,8 @@ void set_default_settings(Settings *settings)
settings->setDefault("fps_max_unfocused", "10");
settings->setDefault("max_objects_per_block", "20");
settings->setDefault("sqlite_synchronous", "1");
settings->setDefault("map_compression_level_disk", "-1");
settings->setDefault("map_compression_level_net", "3");
settings->setDefault("server_map_save_interval", "15");
settings->setDefault("client_mapblock_limit", "1000");
settings->setDefault("active_block_range", "2");

@ -1235,6 +1235,8 @@ ServerMap::ServerMap(const std::string &savedir, IGameDef *gamedef,
m_save_time_counter = mb->addCounter("minetest_core_map_save_time", "Map save time (in nanoseconds)");
m_map_compression_level = rangelim(g_settings->getS16("map_compression_level_disk"), -1, 9);
try {
// If directory exists, check contents and load if possible
if (fs::PathExists(m_savedir)) {
@ -1863,10 +1865,10 @@ void ServerMap::endSave()
bool ServerMap::saveBlock(MapBlock *block)
{
return saveBlock(block, dbase);
return saveBlock(block, dbase, m_map_compression_level);
}
bool ServerMap::saveBlock(MapBlock *block, MapDatabase *db)
bool ServerMap::saveBlock(MapBlock *block, MapDatabase *db, int compression_level)
{
v3s16 p3d = block->getPos();
@ -1886,7 +1888,7 @@ bool ServerMap::saveBlock(MapBlock *block, MapDatabase *db)
*/
std::ostringstream o(std::ios_base::binary);
o.write((char*) &version, 1);
block->serialize(o, version, true);
block->serialize(o, version, true, compression_level);
bool ret = db->saveBlock(p3d, o.str());
if (ret) {

@ -381,7 +381,7 @@ public:
MapgenParams *getMapgenParams();
bool saveBlock(MapBlock *block);
static bool saveBlock(MapBlock *block, MapDatabase *db);
static bool saveBlock(MapBlock *block, MapDatabase *db, int compression_level = -1);
MapBlock* loadBlock(v3s16 p);
// Database version
void loadBlock(std::string *blob, v3s16 p3d, MapSector *sector, bool save_after_load=false);
@ -416,6 +416,7 @@ private:
std::string m_savedir;
bool m_map_saving_enabled;
int m_map_compression_level;
#if 0
// Chunk size in MapSectors
// If 0, chunks are disabled.

@ -355,7 +355,7 @@ static void correctBlockNodeIds(const NameIdMapping *nimap, MapNode *nodes,
}
}
void MapBlock::serialize(std::ostream &os, u8 version, bool disk)
void MapBlock::serialize(std::ostream &os, u8 version, bool disk, int compression_level)
{
if(!ser_ver_supported(version))
throw VersionMismatchException("ERROR: MapBlock format not supported");
@ -394,7 +394,7 @@ void MapBlock::serialize(std::ostream &os, u8 version, bool disk)
writeU8(os, content_width);
writeU8(os, params_width);
MapNode::serializeBulk(os, version, tmp_nodes, nodecount,
content_width, params_width, true);
content_width, params_width, compression_level);
delete[] tmp_nodes;
}
else
@ -404,7 +404,7 @@ void MapBlock::serialize(std::ostream &os, u8 version, bool disk)
writeU8(os, content_width);
writeU8(os, params_width);
MapNode::serializeBulk(os, version, data, nodecount,
content_width, params_width, true);
content_width, params_width, compression_level);
}
/*
@ -412,7 +412,7 @@ void MapBlock::serialize(std::ostream &os, u8 version, bool disk)
*/
std::ostringstream oss(std::ios_base::binary);
m_node_metadata.serialize(oss, version, disk);
compressZlib(oss.str(), os);
compressZlib(oss.str(), os, compression_level);
/*
Data that goes to disk, but not the network
@ -485,7 +485,7 @@ void MapBlock::deSerialize(std::istream &is, u8 version, bool disk)
if(params_width != 2)
throw SerializationError("MapBlock::deSerialize(): invalid params_width");
MapNode::deSerializeBulk(is, version, data, nodecount,
content_width, params_width, true);
content_width, params_width);
/*
NodeMetadata

@ -482,7 +482,7 @@ public:
// These don't write or read version by itself
// Set disk to true for on-disk format, false for over-the-network format
// Precondition: version >= SER_FMT_VER_LOWEST_WRITE
void serialize(std::ostream &os, u8 version, bool disk);
void serialize(std::ostream &os, u8 version, bool disk, int compression_level);
// If disk == true: In addition to doing other things, will add
// unknown blocks from id-name mapping to wndef
void deSerialize(std::istream &is, u8 version, bool disk);

@ -334,7 +334,7 @@ bool Schematic::deserializeFromMts(std::istream *is,
schemdata = new MapNode[nodecount];
MapNode::deSerializeBulk(ss, SER_FMT_VER_HIGHEST_READ, schemdata,
nodecount, 2, 2, true);
nodecount, 2, 2);
// Fix probability values for nodes that were ignore; removed in v2
if (version < 2) {
@ -376,7 +376,7 @@ bool Schematic::serializeToMts(std::ostream *os,
// compressed bulk node data
MapNode::serializeBulk(ss, SER_FMT_VER_HIGHEST_WRITE,
schemdata, size.X * size.Y * size.Z, 2, 2, true);
schemdata, size.X * size.Y * size.Z, 2, 2, -1);
return true;
}

@ -706,7 +706,7 @@ void MapNode::deSerialize(u8 *source, u8 version)
}
void MapNode::serializeBulk(std::ostream &os, int version,
const MapNode *nodes, u32 nodecount,
u8 content_width, u8 params_width, bool compressed)
u8 content_width, u8 params_width, int compression_level)
{
if (!ser_ver_supported(version))
throw VersionMismatchException("ERROR: MapNode format not supported");
@ -737,10 +737,7 @@ void MapNode::serializeBulk(std::ostream &os, int version,
Compress data to output stream
*/
if (compressed)
compressZlib(databuf, databuf_size, os);
else
os.write((const char*) &databuf[0], databuf_size);
compressZlib(databuf, databuf_size, os, compression_level);
delete [] databuf;
}
@ -748,7 +745,7 @@ void MapNode::serializeBulk(std::ostream &os, int version,
// Deserialize bulk node data
void MapNode::deSerializeBulk(std::istream &is, int version,
MapNode *nodes, u32 nodecount,
u8 content_width, u8 params_width, bool compressed)
u8 content_width, u8 params_width)
{
if(!ser_ver_supported(version))
throw VersionMismatchException("ERROR: MapNode format not supported");
@ -760,24 +757,13 @@ void MapNode::deSerializeBulk(std::istream &is, int version,
// Uncompress or read data
u32 len = nodecount * (content_width + params_width);
SharedBuffer<u8> databuf(len);
if(compressed)
{
std::ostringstream os(std::ios_base::binary);
decompressZlib(is, os);
std::string s = os.str();
if(s.size() != len)
throw SerializationError("deSerializeBulkNodes: "
"decompress resulted in invalid size");
memcpy(&databuf[0], s.c_str(), len);
}
else
{
is.read((char*) &databuf[0], len);
if(is.eof() || is.fail())
throw SerializationError("deSerializeBulkNodes: "
"failed to read bulk node data");
}
std::ostringstream os(std::ios_base::binary);
decompressZlib(is, os);
std::string s = os.str();
if(s.size() != len)
throw SerializationError("deSerializeBulkNodes: "
"decompress resulted in invalid size");
const u8 *databuf = reinterpret_cast<const u8*>(s.c_str());
// Deserialize content
if(content_width == 1)

@ -292,10 +292,10 @@ struct MapNode
// compressed = true to zlib-compress output
static void serializeBulk(std::ostream &os, int version,
const MapNode *nodes, u32 nodecount,
u8 content_width, u8 params_width, bool compressed);
u8 content_width, u8 params_width, int compression_level);
static void deSerializeBulk(std::istream &is, int version,
MapNode *nodes, u32 nodecount,
u8 content_width, u8 params_width, bool compressed);
u8 content_width, u8 params_width);
private:
// Deprecated serialization methods

@ -2332,9 +2332,9 @@ void Server::SendBlockNoLock(session_t peer_id, MapBlock *block, u8 ver,
/*
Create a packet with the block in the right format
*/
thread_local const int net_compression_level = rangelim(g_settings->getS16("map_compression_level_net"), -1, 9);
std::ostringstream os(std::ios_base::binary);
block->serialize(os, ver, false);
block->serialize(os, ver, false, net_compression_level);
block->serializeNetworkSpecific(os);
std::string s = os.str();