Mirrorlandia_minetest/minetestmapper
1
0
Fork 0
mirror of https://github.com/minetest/minetestmapper.git synced 2024-11-24 16:33:47 +01:00
Code Issues Packages Projects Releases Wiki Activity

Optimize database access further by allowing "brute-force" queries instead of listing available blocks

Browse Source 
Also adds a heuristic that will enable this behaviour automatically.
This commit is contained in:
sfan5 2020-03-27 21:10:00 +01:00
parent 5b264fd443
commit 7ff2288627
14 changed files with 317 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
README.rst
View File

@ -106,3 +106,7 @@ colors:
scales:
Draw scales on specified image edges (letters *t b l r* meaning top, bottom, left and right), e.g. ``--scales tbr``
exhaustive:
Select if database should be traversed exhaustively or using range queries, available: *never*, *y*, *full*, *auto*
Defaults to *auto*. You shouldn't need to change this, but doing so can improve rendering times on large maps.

91
TileGenerator.cpp
View File

@ -96,6 +96,7 @@ TileGenerator::TileGenerator():
m_geomY(-2048),
m_geomX2(2048),
m_geomY2(2048),
m_exhaustiveSearch(EXH_AUTO),
m_zoom(1),
m_scales(SCALE_LEFT | SCALE_TOP)
{
@ -206,6 +207,11 @@ void TileGenerator::setMaxY(int y)
std::swap(m_yMin, m_yMax);
}
void TileGenerator::setExhaustiveSearch(int mode)
{
m_exhaustiveSearch = mode;
}
void TileGenerator::parseColorsFile(const std::string &fileName)
{
ifstream in;
@ -222,6 +228,7 @@ void TileGenerator::printGeometry(const std::string &input)
input_path += PATH_SEPARATOR;
}
setExhaustiveSearch(EXH_NEVER);
openDb(input_path);
loadBlocks();
@ -247,6 +254,8 @@ void TileGenerator::generate(const std::string &input, const std::string &output
input_path += PATH_SEPARATOR;
}
if (m_dontWriteEmpty) // FIXME: possible too, just needs to be done differently
setExhaustiveSearch(EXH_NEVER);
openDb(input_path);
loadBlocks();
@ -329,6 +338,35 @@ void TileGenerator::openDb(const std::string &input)
#endif
else
throw std::runtime_error(((std::string) "Unknown map backend: ") + backend);
// Determine how we're going to traverse the database (heuristic)
if (m_exhaustiveSearch == EXH_AUTO) {
using u64 = uint64_t;
u64 y_range = (m_yMax / 16 + 1) - (m_yMin / 16);
u64 blocks = (u64)(m_geomX2 - m_geomX) * y_range * (u64)(m_geomY2 - m_geomY);
#ifndef NDEBUG
std::cout << "Heuristic parameters:"
<< " preferRangeQueries()=" << m_db->preferRangeQueries()
<< " y_range=" << y_range << " blocks=" << blocks << std::endl;
#endif
if (m_db->preferRangeQueries())
m_exhaustiveSearch = EXH_NEVER;
else if (blocks < 200000)
m_exhaustiveSearch = EXH_FULL;
else if (y_range < 600)
m_exhaustiveSearch = EXH_Y;
else
m_exhaustiveSearch = EXH_NEVER;
} else if (m_exhaustiveSearch == EXH_FULL || m_exhaustiveSearch == EXH_Y) {
if (m_db->preferRangeQueries()) {
std::cerr << "Note: The current database backend supports efficient "
"range queries, forcing exhaustive search should always result "
" in worse performance." << std::endl;
}
}
if (m_exhaustiveSearch == EXH_Y)
m_exhaustiveSearch = EXH_NEVER; // (TODO remove when implemented)
assert(m_exhaustiveSearch != EXH_AUTO);
}
void TileGenerator::closeDatabase()
@ -340,6 +378,8 @@ void TileGenerator::closeDatabase()
void TileGenerator::loadBlocks()
{
const int16_t yMax = m_yMax / 16 + 1;
if (m_exhaustiveSearch == EXH_NEVER || m_exhaustiveSearch == EXH_Y) {
std::vector<BlockPos> vec = m_db->getBlockPos(
BlockPos(m_geomX, m_yMin / 16, m_geomY),
BlockPos(m_geomX2, yMax, m_geomY2)
@ -371,6 +411,7 @@ void TileGenerator::loadBlocks()
std::cout << "Loaded " << count
<< " positions (across Z: " << m_positions.size() << ") for rendering" << std::endl;
#endif
}
}
void TileGenerator::createImage()
@ -422,11 +463,7 @@ void TileGenerator::renderMap()
BlockDecoder blk;
const int16_t yMax = m_yMax / 16 + 1;
for (auto it = m_positions.rbegin(); it != m_positions.rend(); ++it) {
int16_t zPos = it->first;
for (auto it2 = it->second.rbegin(); it2 != it->second.rend(); ++it2) {
int16_t xPos = *it2;
auto renderSingle = [&] (int16_t xPos, int16_t zPos, BlockList &blockStack) {
m_readPixels.reset();
m_readInfo.reset();
for (int i = 0; i < 16; i++) {
@ -437,9 +474,6 @@ void TileGenerator::renderMap()
}
}
BlockList blockStack;
m_db->getBlocksOnXZ(blockStack, xPos, zPos, m_yMin / 16, yMax);
blockStack.sort();
for (const auto &it : blockStack) {
const BlockPos pos = it.first;
assert(pos.x == xPos && pos.z == zPos);
@ -457,9 +491,48 @@ void TileGenerator::renderMap()
}
if (!m_readPixels.full())
renderMapBlockBottom(blockStack.begin()->first);
}
};
auto postRenderRow = [&] (int16_t zPos) {
if (m_shading)
renderShading(zPos);
};
if (m_exhaustiveSearch == EXH_NEVER) {
for (auto it = m_positions.rbegin(); it != m_positions.rend(); ++it) {
int16_t zPos = it->first;
for (auto it2 = it->second.rbegin(); it2 != it->second.rend(); ++it2) {
int16_t xPos = *it2;
BlockList blockStack;
m_db->getBlocksOnXZ(blockStack, xPos, zPos, m_yMin / 16, yMax);
blockStack.sort();
renderSingle(xPos, zPos, blockStack);
}
postRenderRow(zPos);
}
} else if (m_exhaustiveSearch == EXH_FULL) {
#ifndef NDEBUG
std::cerr << "Exhaustively searching "
<< (m_geomX2 - m_geomX) << "x" << (yMax - (m_yMin / 16)) << "x"
<< (m_geomY2 - m_geomY) << " blocks" << std::endl;
#endif
std::vector<BlockPos> positions;
positions.reserve(yMax - (m_yMin / 16));
for (int16_t zPos = m_geomY2 - 1; zPos >= m_geomY; zPos--) {
for (int16_t xPos = m_geomX2 - 1; xPos >= m_geomX; xPos--) {
positions.clear();
for (int16_t yPos = m_yMin / 16; yPos < yMax; yPos++)
positions.emplace_back(xPos, yPos, zPos);
BlockList blockStack;
m_db->getBlocksByPos(blockStack, positions);
blockStack.sort();
renderSingle(xPos, zPos, blockStack);
}
postRenderRow(zPos);
}
}
}

16
db-leveldb.cpp
View File

@ -96,3 +96,19 @@ void DBLevelDB::getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
}
}
}
void DBLevelDB::getBlocksByPos(BlockList &blocks,
const std::vector<BlockPos> &positions)
{
std::string datastr;
leveldb::Status status;
for (auto pos : positions) {
status = db->Get(leveldb::ReadOptions(), i64tos(encodeBlockPos(pos)), &datastr);
if (status.ok()) {
blocks.emplace_back(
pos, ustring((unsigned char *) datastr.data(), datastr.size())
);
}
}
}

45
db-postgresql.cpp
View File

@ -38,6 +38,11 @@ DBPostgreSQL::DBPostgreSQL(const std::string &mapdir)
" posX = $1::int4 AND posZ = $2::int4"
" AND (posY BETWEEN $3::int4 AND $4::int4)"
);
prepareStatement(
"get_block_exact",
"SELECT data FROM blocks WHERE"
" posX = $1::int4 AND posY = $2::int4 AND posZ = $3::int4"
);
checkResults(PQexec(db, "START TRANSACTION;"));
checkResults(PQexec(db, "SET TRANSACTION ISOLATION LEVEL REPEATABLE READ;"));
@ -48,12 +53,13 @@ DBPostgreSQL::~DBPostgreSQL()
{
try {
checkResults(PQexec(db, "COMMIT;"));
} catch (std::exception& caught) {
} catch (const std::exception& caught) {
std::cerr << "could not finalize: " << caught.what() << std::endl;
}
PQfinish(db);
}
std::vector<BlockPos> DBPostgreSQL::getBlockPos(BlockPos min, BlockPos max)
{
int32_t const x1 = htonl(min.x);
@ -124,6 +130,43 @@ void DBPostgreSQL::getBlocksOnXZ(BlockList &blocks, int16_t xPos, int16_t zPos,
PQclear(results);
}
void DBPostgreSQL::getBlocksByPos(BlockList &blocks,
const std::vector<BlockPos> &positions)
{
int32_t x, y, z;
const void *args[] = { &x, &y, &z };
const int argLen[] = { 4, 4, 4 };
const int argFmt[] = { 1, 1, 1 };
for (auto pos : positions) {
x = htonl(pos.x);
y = htonl(pos.y);
z = htonl(pos.z);
PGresult *results = execPrepared(
"get_block_exact", ARRLEN(args), args,
argLen, argFmt, false
);
if (PQntuples(results) > 0) {
blocks.emplace_back(
pos,
ustring(
reinterpret_cast<unsigned char*>(
PQgetvalue(results, 0, 0)
),
PQgetlength(results, 0, 0)
)
);
}
PQclear(results);
}
}
PGresult *DBPostgreSQL::checkResults(PGresult *res, bool clear)
{
ExecStatusType statusType = PQresultStatus(res);

46
db-redis.cpp
View File

@ -82,7 +82,7 @@ std::vector<BlockPos> DBRedis::getBlockPos(BlockPos min, BlockPos max)
}
std::string DBRedis::replyTypeStr(int type) {
const char *DBRedis::replyTypeStr(int type) {
switch(type) {
case REDIS_REPLY_STATUS:
return "REDIS_REPLY_STATUS";
@ -121,7 +121,8 @@ void DBRedis::loadPosCache()
}
void DBRedis::HMGET(const std::vector<BlockPos> &positions, std::vector<ustring> *result)
void DBRedis::HMGET(const std::vector<BlockPos> &positions,
std::function<void(std::size_t, ustring)> result)
{
const char *argv[DB_REDIS_HMGET_NUMFIELDS + 2];
argv[0] = "HMGET";
@ -146,27 +147,21 @@ void DBRedis::HMGET(const std::vector<BlockPos> &positions, std::vector<ustring>
if(!reply)
throw std::runtime_error("Redis command HMGET failed");
if (reply->type != REDIS_REPLY_ARRAY) {
freeReplyObject(reply);
REPLY_TYPE_ERR(reply, "HKEYS subreply");
}
if (reply->type != REDIS_REPLY_ARRAY)
REPLY_TYPE_ERR(reply, "HMGET reply");
if (reply->elements != batch_size) {
freeReplyObject(reply);
throw std::runtime_error("HMGET wrong number of elements");
}
for (std::size_t i = 0; i < batch_size; ++i) {
for (std::size_t i = 0; i < reply->elements; ++i) {
redisReply *subreply = reply->element[i];
if(!subreply)
throw std::runtime_error("Redis command HMGET failed");
if (subreply->type != REDIS_REPLY_STRING) {
freeReplyObject(reply);
REPLY_TYPE_ERR(reply, "HKEYS subreply");
}
if (subreply->len == 0) {
freeReplyObject(reply);
if (subreply->type == REDIS_REPLY_NIL)
continue;
else if (subreply->type != REDIS_REPLY_STRING)
REPLY_TYPE_ERR(subreply, "HMGET subreply");
if (subreply->len == 0)
throw std::runtime_error("HMGET empty string");
}
result->emplace_back((const unsigned char *) subreply->str, subreply->len);
result(i, ustring((const unsigned char *) subreply->str, subreply->len));
}
freeReplyObject(reply);
remaining -= batch_size;
@ -187,10 +182,15 @@ void DBRedis::getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
positions.emplace_back(x, pos2.second, z);
}
std::vector<ustring> db_blocks;
HMGET(positions, &db_blocks);
auto block = db_blocks.cbegin();
for (auto pos = positions.cbegin(); pos != positions.cend(); ++pos, ++block)
blocks.emplace_back(*pos, *block);
getBlocksByPos(blocks, positions);
}
void DBRedis::getBlocksByPos(BlockList &blocks,
const std::vector<BlockPos> &positions)
{
auto result = [&] (std::size_t i, ustring data) {
blocks.emplace_back(positions[i], std::move(data));
};
HMGET(positions, result);
}

33
db-sqlite3.cpp
View File

@ -25,6 +25,10 @@ DBSQLite3::DBSQLite3(const std::string &mapdir)
"SELECT pos, data FROM blocks WHERE pos BETWEEN ? AND ?",
-1, &stmt_get_blocks_z, NULL))
SQLOK(prepare_v2(db,
"SELECT data FROM blocks WHERE pos = ?",
-1, &stmt_get_block_exact, NULL))
SQLOK(prepare_v2(db,
"SELECT pos FROM blocks",
-1, &stmt_get_block_pos, NULL))
@ -40,6 +44,7 @@ DBSQLite3::~DBSQLite3()
sqlite3_finalize(stmt_get_blocks_z);
sqlite3_finalize(stmt_get_block_pos);
sqlite3_finalize(stmt_get_block_pos_z);
sqlite3_finalize(stmt_get_block_exact);
if (sqlite3_close(db) != SQLITE_OK) {
std::cerr << "Error closing SQLite database." << std::endl;
@ -161,3 +166,31 @@ void DBSQLite3::getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
it++;
}
}
void DBSQLite3::getBlocksByPos(BlockList &blocks,
const std::vector<BlockPos> &positions)
{
int result;
for (auto pos : positions) {
int64_t dbPos = encodeBlockPos(pos);
SQLOK(bind_int64(stmt_get_block_exact, 1, dbPos));
while ((result = sqlite3_step(stmt_get_block_exact)) == SQLITE_BUSY) {
usleep(10000); // Wait some time and try again
}
if (result == SQLITE_DONE) {
// no data
} else if (result != SQLITE_ROW) {
throw std::runtime_error(sqlite3_errmsg(db));
} else {
const unsigned char *data = reinterpret_cast<const unsigned char *>(
sqlite3_column_blob(stmt_get_block_exact, 0));
size_t size = sqlite3_column_bytes(stmt_get_block_exact, 0);
blocks.emplace_back(pos, ustring(data, size));
}
SQLOK(reset(stmt_get_block_exact))
}
}

9
include/TileGenerator.h
View File

@ -23,6 +23,13 @@ enum {
SCALE_RIGHT = (1 << 3),
};
enum {
EXH_NEVER, // Always use range queries
EXH_Y, // Exhaustively search Y space, range queries for X/Z (future TODO)
EXH_FULL, // Exhaustively search entire requested geometry
EXH_AUTO, // Automatically pick one of the previous modes
};
struct ColorEntry {
ColorEntry(): r(0), g(0), b(0), a(0), t(0) {};
ColorEntry(uint8_t r, uint8_t g, uint8_t b, uint8_t a, uint8_t t): r(r), g(g), b(b), a(a), t(t) {};
@ -75,6 +82,7 @@ public:
void setGeometry(int x, int y, int w, int h);
void setMinY(int y);
void setMaxY(int y);
void setExhaustiveSearch(int mode);
void parseColorsFile(const std::string &fileName);
void setBackend(std::string backend);
void generate(const std::string &input, const std::string &output);
@ -135,6 +143,7 @@ private:
/* */
int m_mapWidth;
int m_mapHeight;
int m_exhaustiveSearch;
std::map<int16_t, std::set<int16_t>> m_positions; /* indexed by Z, contains X coords */
ColorMap m_colorMap;
BitmapThing m_readPixels;

4
include/db-leveldb.h
View File

@ -12,8 +12,12 @@ public:
std::vector<BlockPos> getBlockPos(BlockPos min, BlockPos max) override;
void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
int16_t min_y, int16_t max_y) override;
void getBlocksByPos(BlockList &blocks,
const std::vector<BlockPos> &positions) override;
~DBLevelDB() override;
bool preferRangeQueries() const override { return false; }
private:
using pos2d = std::pair<int16_t, int16_t>;

4
include/db-postgresql.h
View File

@ -10,8 +10,12 @@ public:
std::vector<BlockPos> getBlockPos(BlockPos min, BlockPos max) override;
void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
int16_t min_y, int16_t max_y) override;
void getBlocksByPos(BlockList &blocks,
const std::vector<BlockPos> &positions) override;
~DBPostgreSQL() override;
bool preferRangeQueries() const override { return true; }
protected:
PGresult *checkResults(PGresult *res, bool clear = true);
void prepareStatement(const std::string &name, const std::string &sql);

10
include/db-redis.h
View File

@ -4,6 +4,7 @@
#include "db.h"
#include <unordered_map>
#include <utility>
#include <functional>
#include <hiredis/hiredis.h>
class DBRedis : public DB {
@ -12,14 +13,19 @@ public:
std::vector<BlockPos> getBlockPos(BlockPos min, BlockPos max) override;
void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
int16_t min_y, int16_t max_y) override;
void getBlocksByPos(BlockList &blocks,
const std::vector<BlockPos> &positions) override;
~DBRedis() override;
bool preferRangeQueries() const override { return false; }
private:
using pos2d = std::pair<int16_t, int16_t>;
static std::string replyTypeStr(int type);
static const char *replyTypeStr(int type);
void loadPosCache();
void HMGET(const std::vector<BlockPos> &positions, std::vector<ustring> *result);
void HMGET(const std::vector<BlockPos> &positions,
std::function<void(std::size_t, ustring)> result);
// indexed by Z, contains all (x,y) position pairs
std::unordered_map<int16_t, std::vector<pos2d>> posCache;

5
include/db-sqlite3.h
View File

@ -11,8 +11,12 @@ public:
std::vector<BlockPos> getBlockPos(BlockPos min, BlockPos max) override;
void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
int16_t min_y, int16_t max_y) override;
void getBlocksByPos(BlockList &blocks,
const std::vector<BlockPos> &positions) override;
~DBSQLite3() override;
bool preferRangeQueries() const override { return false; }
private:
inline void getPosRange(int64_t &min, int64_t &max, int16_t zPos,
int16_t zPos2) const;
@ -23,6 +27,7 @@ private:
sqlite3_stmt *stmt_get_block_pos;
sqlite3_stmt *stmt_get_block_pos_z;
sqlite3_stmt *stmt_get_blocks_z;
sqlite3_stmt *stmt_get_block_exact;
int16_t blockCachedZ = -10000;
std::unordered_map<int16_t, BlockList> blockCache; // indexed by X

14
include/db.h
View File

@ -52,11 +52,21 @@ public:
* so that min.x <= x < max.x, ...
*/
virtual std::vector<BlockPos> getBlockPos(BlockPos min, BlockPos max) = 0;
/* Return all blocks in column given by x and z
* and inside the given Y range (min_y <= y < max_y)
/* Read all blocks in column given by x and z
* and inside the given Y range (min_y <= y < max_y) into list
*/
virtual void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
int16_t min_y, int16_t max_y) = 0;
/* Read blocks at given positions into list
*/
virtual void getBlocksByPos(BlockList &blocks,
const std::vector<BlockPos> &positions) = 0;
/* Can this database efficiently do range queries?
* (for large data sets, more efficient that brute force)
*/
virtual bool preferRangeQueries() const = 0;
virtual ~DB() {}
};

15
mapper.cpp
View File

@ -33,6 +33,7 @@ static void usage()
" --zoom <zoomlevel>\n"
" --colors <colors.txt>\n"
" --scales [t][b][l][r]\n"
" --exhaustive never|y|full|auto\n"
"Color format: '#000000'\n";
std::cout << usage_text;
}
@ -90,6 +91,7 @@ int main(int argc, char *argv[])
{"colors", required_argument, 0, 'C'},
{"scales", required_argument, 0, 'f'},
{"noemptyimage", no_argument, 0, 'n'},
{"exhaustive", required_argument, 0, 'j'},
{0, 0, 0, 0}
};
@ -201,6 +203,19 @@ int main(int argc, char *argv[])
case 'n':
generator.setDontWriteEmpty(true);
break;
case 'j': {
int mode;
if (!strcmp(optarg, "never"))
mode = EXH_NEVER;
else if (!strcmp(optarg, "y"))
mode = EXH_Y;
else if (!strcmp(optarg, "full"))
mode = EXH_FULL;
else
mode = EXH_AUTO;
generator.setExhaustiveSearch(mode);
}
break;
default:
exit(1);
}

12
minetestmapper.6
View File

@ -70,11 +70,11 @@ Don't draw nodes above this y value, e.g. "--max-y 75"
.TP
.BR \-\-backend " " \fIbackend\fR
Use specific map backend; supported: *sqlite3*, *leveldb*, *redis*, *postgresql*, e.g. "--backend leveldb"
Use specific map backend; supported: \fIsqlite3\fP, \fIleveldb\fP, \fIredis\fP, \fIpostgresql\fP, e.g. "--backend leveldb"
.TP
.BR \-\-geometry " " \fIgeometry\fR
Limit area to specific geometry (*x:y+w+h* where x and y specify the lower left corner), e.g. "--geometry -800:-800+1600+1600"
Limit area to specific geometry (\fIx:y+w+h\fP where x and y specify the lower left corner), e.g. "--geometry -800:-800+1600+1600"
.TP
.BR \-\-extent " " \fIextent\fR
@ -90,7 +90,13 @@ Forcefully set path to colors.txt file (it's autodetected otherwise), e.g. "--co
.TP
.BR \-\-scales " " \fIedges\fR
Draw scales on specified image edges (letters *t b l r* meaning top, bottom, left and right), e.g. "--scales tbr"
Draw scales on specified image edges (letters \fIt b l r\fP meaning top, bottom, left and right), e.g. "--scales tbr"
.TP
.BR \-\-exhaustive " \fImode\fR
Select if database should be traversed exhaustively or using range queries, available: \fInever\fP, \fIy\fP, \fIfull\fP, \fIauto\fP
Defaults to \fIauto\fP. You shouldn't need to change this, but doing so can improve rendering times on large maps.
.SH MORE INFORMATION
Website: https://github.com/minetest/minetestmapper