Add count based unload limit for mapblocks

This commit is contained in:
est31 2015-08-10 22:24:47 +02:00
parent 2b04ab874d
commit a8e238ed06
8 changed files with 154 additions and 65 deletions

@ -94,6 +94,9 @@
#random_input = false #random_input = false
# Timeout for client to remove unused map data from memory # Timeout for client to remove unused map data from memory
#client_unload_unused_data_timeout = 600 #client_unload_unused_data_timeout = 600
# Maximum number of mapblocks for client to be kept in memory
# Set to -1 for unlimited amount
#client_mapblock_limit = 1000
# Whether to fog out the end of the visible area # Whether to fog out the end of the visible area
#enable_fog = true #enable_fog = true
# Whether to show the client debug info (has the same effect as hitting F5) # Whether to show the client debug info (has the same effect as hitting F5)

@ -421,8 +421,9 @@ void Client::step(float dtime)
ScopeProfiler sp(g_profiler, "Client: map timer and unload"); ScopeProfiler sp(g_profiler, "Client: map timer and unload");
std::vector<v3s16> deleted_blocks; std::vector<v3s16> deleted_blocks;
m_env.getMap().timerUpdate(map_timer_and_unload_dtime, m_env.getMap().timerUpdate(map_timer_and_unload_dtime,
g_settings->getFloat("client_unload_unused_data_timeout"), g_settings->getFloat("client_unload_unused_data_timeout"),
&deleted_blocks); g_settings->getS32("client_mapblock_limit"),
&deleted_blocks);
/* /*
Send info to server Send info to server

@ -104,6 +104,7 @@ void set_default_settings(Settings *settings)
settings->setDefault("address", ""); settings->setDefault("address", "");
settings->setDefault("random_input", "false"); settings->setDefault("random_input", "false");
settings->setDefault("client_unload_unused_data_timeout", "600"); settings->setDefault("client_unload_unused_data_timeout", "600");
settings->setDefault("client_mapblock_limit", "1000");
settings->setDefault("enable_fog", "true"); settings->setDefault("enable_fog", "true");
settings->setDefault("fov", "72"); settings->setDefault("fov", "72");
settings->setDefault("view_bobbing", "true"); settings->setDefault("view_bobbing", "true");

@ -43,6 +43,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
#include "database-dummy.h" #include "database-dummy.h"
#include "database-sqlite3.h" #include "database-sqlite3.h"
#include <deque> #include <deque>
#include <queue>
#if USE_LEVELDB #if USE_LEVELDB
#include "database-leveldb.h" #include "database-leveldb.h"
#endif #endif
@ -1399,10 +1400,25 @@ bool Map::getDayNightDiff(v3s16 blockpos)
return false; return false;
} }
struct TimeOrderedMapBlock {
MapSector *sect;
MapBlock *block;
TimeOrderedMapBlock(MapSector *sect, MapBlock *block) :
sect(sect),
block(block)
{}
bool operator<(const TimeOrderedMapBlock &b) const
{
return block->getUsageTimer() < b.block->getUsageTimer();
};
};
/* /*
Updates usage timers Updates usage timers
*/ */
void Map::timerUpdate(float dtime, float unload_timeout, void Map::timerUpdate(float dtime, float unload_timeout, u32 max_loaded_blocks,
std::vector<v3s16> *unloaded_blocks) std::vector<v3s16> *unloaded_blocks)
{ {
bool save_before_unloading = (mapType() == MAPTYPE_SERVER); bool save_before_unloading = (mapType() == MAPTYPE_SERVER);
@ -1416,48 +1432,108 @@ void Map::timerUpdate(float dtime, float unload_timeout,
u32 block_count_all = 0; u32 block_count_all = 0;
beginSave(); beginSave();
for(std::map<v2s16, MapSector*>::iterator si = m_sectors.begin();
si != m_sectors.end(); ++si) {
MapSector *sector = si->second;
bool all_blocks_deleted = true; // If there is no practical limit, we spare creation of mapblock_queue
if (max_loaded_blocks == (u32)-1) {
for (std::map<v2s16, MapSector*>::iterator si = m_sectors.begin();
si != m_sectors.end(); ++si) {
MapSector *sector = si->second;
MapBlockVect blocks; bool all_blocks_deleted = true;
sector->getBlocks(blocks);
for(MapBlockVect::iterator i = blocks.begin(); MapBlockVect blocks;
i != blocks.end(); ++i) { sector->getBlocks(blocks);
MapBlock *block = (*i);
block->incrementUsageTimer(dtime); for (MapBlockVect::iterator i = blocks.begin();
i != blocks.end(); ++i) {
MapBlock *block = (*i);
if(block->refGet() == 0 && block->getUsageTimer() > unload_timeout) { block->incrementUsageTimer(dtime);
v3s16 p = block->getPos();
// Save if modified if (block->refGet() == 0
if (block->getModified() != MOD_STATE_CLEAN && save_before_unloading) { && block->getUsageTimer() > unload_timeout) {
modprofiler.add(block->getModifiedReasonString(), 1); v3s16 p = block->getPos();
if (!saveBlock(block))
continue; // Save if modified
saved_blocks_count++; if (block->getModified() != MOD_STATE_CLEAN
&& save_before_unloading) {
modprofiler.add(block->getModifiedReasonString(), 1);
if (!saveBlock(block))
continue;
saved_blocks_count++;
}
// Delete from memory
sector->deleteBlock(block);
if (unloaded_blocks)
unloaded_blocks->push_back(p);
deleted_blocks_count++;
} else {
all_blocks_deleted = false;
block_count_all++;
} }
// Delete from memory
sector->deleteBlock(block);
if(unloaded_blocks)
unloaded_blocks->push_back(p);
deleted_blocks_count++;
} }
else {
all_blocks_deleted = false; if (all_blocks_deleted) {
block_count_all++; sector_deletion_queue.push_back(si->first);
} }
} }
} else {
std::priority_queue<TimeOrderedMapBlock> mapblock_queue;
for (std::map<v2s16, MapSector*>::iterator si = m_sectors.begin();
si != m_sectors.end(); ++si) {
MapSector *sector = si->second;
if(all_blocks_deleted) { MapBlockVect blocks;
sector_deletion_queue.push_back(si->first); sector->getBlocks(blocks);
for(MapBlockVect::iterator i = blocks.begin();
i != blocks.end(); ++i) {
MapBlock *block = (*i);
block->incrementUsageTimer(dtime);
mapblock_queue.push(TimeOrderedMapBlock(sector, block));
}
}
block_count_all = mapblock_queue.size();
// Delete old blocks, and blocks over the limit from the memory
while (mapblock_queue.size() > max_loaded_blocks
|| mapblock_queue.top().block->getUsageTimer() > unload_timeout) {
TimeOrderedMapBlock b = mapblock_queue.top();
mapblock_queue.pop();
MapBlock *block = b.block;
if (block->refGet() != 0)
continue;
v3s16 p = block->getPos();
// Save if modified
if (block->getModified() != MOD_STATE_CLEAN && save_before_unloading) {
modprofiler.add(block->getModifiedReasonString(), 1);
if (!saveBlock(block))
continue;
saved_blocks_count++;
}
// Delete from memory
b.sect->deleteBlock(block);
if (unloaded_blocks)
unloaded_blocks->push_back(p);
deleted_blocks_count++;
block_count_all--;
}
// Delete empty sectors
for (std::map<v2s16, MapSector*>::iterator si = m_sectors.begin();
si != m_sectors.end(); ++si) {
if (si->second->empty()) {
sector_deletion_queue.push_back(si->first);
}
} }
} }
endSave(); endSave();
@ -1484,7 +1560,7 @@ void Map::timerUpdate(float dtime, float unload_timeout,
void Map::unloadUnreferencedBlocks(std::vector<v3s16> *unloaded_blocks) void Map::unloadUnreferencedBlocks(std::vector<v3s16> *unloaded_blocks)
{ {
timerUpdate(0.0, -1.0, unloaded_blocks); timerUpdate(0.0, -1.0, 0, unloaded_blocks);
} }
void Map::deleteSectors(std::vector<v2s16> &sectorList) void Map::deleteSectors(std::vector<v2s16> &sectorList)

@ -277,7 +277,7 @@ public:
Updates usage timers and unloads unused blocks and sectors. Updates usage timers and unloads unused blocks and sectors.
Saves modified blocks before unloading on MAPTYPE_SERVER. Saves modified blocks before unloading on MAPTYPE_SERVER.
*/ */
void timerUpdate(float dtime, float unload_timeout, void timerUpdate(float dtime, float unload_timeout, u32 max_loaded_blocks,
std::vector<v3s16> *unloaded_blocks=NULL); std::vector<v3s16> *unloaded_blocks=NULL);
/* /*

@ -59,7 +59,7 @@ MapBlock * MapSector::getBlockBuffered(s16 y)
if(m_block_cache != NULL && y == m_block_cache_y){ if(m_block_cache != NULL && y == m_block_cache_y){
return m_block_cache; return m_block_cache;
} }
// If block doesn't exist, return NULL // If block doesn't exist, return NULL
std::map<s16, MapBlock*>::iterator n = m_blocks.find(y); std::map<s16, MapBlock*>::iterator n = m_blocks.find(y);
if(n == m_blocks.end()) if(n == m_blocks.end())
@ -70,11 +70,11 @@ MapBlock * MapSector::getBlockBuffered(s16 y)
else{ else{
block = n->second; block = n->second;
} }
// Cache the last result // Cache the last result
m_block_cache_y = y; m_block_cache_y = y;
m_block_cache = block; m_block_cache = block;
return block; return block;
} }
@ -88,16 +88,16 @@ MapBlock * MapSector::createBlankBlockNoInsert(s16 y)
assert(getBlockBuffered(y) == NULL); // Pre-condition assert(getBlockBuffered(y) == NULL); // Pre-condition
v3s16 blockpos_map(m_pos.X, y, m_pos.Y); v3s16 blockpos_map(m_pos.X, y, m_pos.Y);
MapBlock *block = new MapBlock(m_parent, blockpos_map, m_gamedef); MapBlock *block = new MapBlock(m_parent, blockpos_map, m_gamedef);
return block; return block;
} }
MapBlock * MapSector::createBlankBlock(s16 y) MapBlock * MapSector::createBlankBlock(s16 y)
{ {
MapBlock *block = createBlankBlockNoInsert(y); MapBlock *block = createBlankBlockNoInsert(y);
m_blocks[y] = block; m_blocks[y] = block;
return block; return block;
@ -114,7 +114,7 @@ void MapSector::insertBlock(MapBlock *block)
v2s16 p2d(block->getPos().X, block->getPos().Z); v2s16 p2d(block->getPos().X, block->getPos().Z);
assert(p2d == m_pos); assert(p2d == m_pos);
// Insert into container // Insert into container
m_blocks[block_y] = block; m_blocks[block_y] = block;
} }
@ -125,7 +125,7 @@ void MapSector::deleteBlock(MapBlock *block)
// Clear from cache // Clear from cache
m_block_cache = NULL; m_block_cache = NULL;
// Remove from container // Remove from container
m_blocks.erase(block_y); m_blocks.erase(block_y);
@ -142,6 +142,11 @@ void MapSector::getBlocks(MapBlockVect &dest)
} }
} }
bool MapSector::empty()
{
return m_blocks.empty();
}
/* /*
ServerMapSector ServerMapSector
*/ */
@ -159,18 +164,18 @@ void ServerMapSector::serialize(std::ostream &os, u8 version)
{ {
if(!ser_ver_supported(version)) if(!ser_ver_supported(version))
throw VersionMismatchException("ERROR: MapSector format not supported"); throw VersionMismatchException("ERROR: MapSector format not supported");
/* /*
[0] u8 serialization version [0] u8 serialization version
+ heightmap data + heightmap data
*/ */
// Server has both of these, no need to support not having them. // Server has both of these, no need to support not having them.
//assert(m_objects != NULL); //assert(m_objects != NULL);
// Write version // Write version
os.write((char*)&version, 1); os.write((char*)&version, 1);
/* /*
Add stuff here, if needed Add stuff here, if needed
*/ */
@ -193,18 +198,18 @@ ServerMapSector* ServerMapSector::deSerialize(
/* /*
Read stuff Read stuff
*/ */
// Read version // Read version
u8 version = SER_FMT_VER_INVALID; u8 version = SER_FMT_VER_INVALID;
is.read((char*)&version, 1); is.read((char*)&version, 1);
if(!ser_ver_supported(version)) if(!ser_ver_supported(version))
throw VersionMismatchException("ERROR: MapSector format not supported"); throw VersionMismatchException("ERROR: MapSector format not supported");
/* /*
Add necessary reading stuff here Add necessary reading stuff here
*/ */
/* /*
Get or create sector Get or create sector
*/ */

@ -40,7 +40,7 @@ class IGameDef;
class MapSector class MapSector
{ {
public: public:
MapSector(Map *parent, v2s16 pos, IGameDef *gamedef); MapSector(Map *parent, v2s16 pos, IGameDef *gamedef);
virtual ~MapSector(); virtual ~MapSector();
@ -58,16 +58,18 @@ public:
MapBlock * createBlankBlock(s16 y); MapBlock * createBlankBlock(s16 y);
void insertBlock(MapBlock *block); void insertBlock(MapBlock *block);
void deleteBlock(MapBlock *block); void deleteBlock(MapBlock *block);
void getBlocks(MapBlockVect &dest); void getBlocks(MapBlockVect &dest);
bool empty();
// Always false at the moment, because sector contains no metadata. // Always false at the moment, because sector contains no metadata.
bool differs_from_disk; bool differs_from_disk;
protected: protected:
// The pile of MapBlocks // The pile of MapBlocks
std::map<s16, MapBlock*> m_blocks; std::map<s16, MapBlock*> m_blocks;
@ -76,12 +78,12 @@ protected:
v2s16 m_pos; v2s16 m_pos;
IGameDef *m_gamedef; IGameDef *m_gamedef;
// Last-used block is cached here for quicker access. // Last-used block is cached here for quicker access.
// Be sure to set this to NULL when the cached block is deleted // Be sure to set this to NULL when the cached block is deleted
MapBlock *m_block_cache; MapBlock *m_block_cache;
s16 m_block_cache_y; s16 m_block_cache_y;
/* /*
Private methods Private methods
*/ */
@ -94,7 +96,7 @@ class ServerMapSector : public MapSector
public: public:
ServerMapSector(Map *parent, v2s16 pos, IGameDef *gamedef); ServerMapSector(Map *parent, v2s16 pos, IGameDef *gamedef);
~ServerMapSector(); ~ServerMapSector();
u32 getId() const u32 getId() const
{ {
return MAPSECTOR_SERVER; return MAPSECTOR_SERVER;
@ -106,7 +108,7 @@ public:
*/ */
void serialize(std::ostream &os, u8 version); void serialize(std::ostream &os, u8 version);
static ServerMapSector* deSerialize( static ServerMapSector* deSerialize(
std::istream &is, std::istream &is,
Map *parent, Map *parent,
@ -114,7 +116,7 @@ public:
std::map<v2s16, MapSector*> & sectors, std::map<v2s16, MapSector*> & sectors,
IGameDef *gamedef IGameDef *gamedef
); );
private: private:
}; };
@ -124,7 +126,7 @@ class ClientMapSector : public MapSector
public: public:
ClientMapSector(Map *parent, v2s16 pos, IGameDef *gamedef); ClientMapSector(Map *parent, v2s16 pos, IGameDef *gamedef);
~ClientMapSector(); ~ClientMapSector();
u32 getId() const u32 getId() const
{ {
return MAPSECTOR_CLIENT; return MAPSECTOR_CLIENT;
@ -133,6 +135,6 @@ public:
private: private:
}; };
#endif #endif
#endif #endif

@ -594,7 +594,8 @@ void Server::AsyncRunStep(bool initial_step)
// Run Map's timers and unload unused data // Run Map's timers and unload unused data
ScopeProfiler sp(g_profiler, "Server: map timer and unload"); ScopeProfiler sp(g_profiler, "Server: map timer and unload");
m_env->getMap().timerUpdate(map_timer_and_unload_dtime, m_env->getMap().timerUpdate(map_timer_and_unload_dtime,
g_settings->getFloat("server_unload_unused_data_timeout")); g_settings->getFloat("server_unload_unused_data_timeout"),
(u32)-1);
} }
/* /*