mirror of
https://github.com/minetest/minetestmapper.git
synced 2024-11-21 23:13:53 +01:00
Optimize database access further by allowing "brute-force" queries instead of listing available blocks
Also adds a heuristic that will enable this behaviour automatically.
This commit is contained in:
parent
5b264fd443
commit
7ff2288627
@ -106,3 +106,7 @@ colors:
|
|||||||
|
|
||||||
scales:
|
scales:
|
||||||
Draw scales on specified image edges (letters *t b l r* meaning top, bottom, left and right), e.g. ``--scales tbr``
|
Draw scales on specified image edges (letters *t b l r* meaning top, bottom, left and right), e.g. ``--scales tbr``
|
||||||
|
|
||||||
|
exhaustive:
|
||||||
|
Select if database should be traversed exhaustively or using range queries, available: *never*, *y*, *full*, *auto*
|
||||||
|
Defaults to *auto*. You shouldn't need to change this, but doing so can improve rendering times on large maps.
|
||||||
|
@ -96,6 +96,7 @@ TileGenerator::TileGenerator():
|
|||||||
m_geomY(-2048),
|
m_geomY(-2048),
|
||||||
m_geomX2(2048),
|
m_geomX2(2048),
|
||||||
m_geomY2(2048),
|
m_geomY2(2048),
|
||||||
|
m_exhaustiveSearch(EXH_AUTO),
|
||||||
m_zoom(1),
|
m_zoom(1),
|
||||||
m_scales(SCALE_LEFT | SCALE_TOP)
|
m_scales(SCALE_LEFT | SCALE_TOP)
|
||||||
{
|
{
|
||||||
@ -206,6 +207,11 @@ void TileGenerator::setMaxY(int y)
|
|||||||
std::swap(m_yMin, m_yMax);
|
std::swap(m_yMin, m_yMax);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void TileGenerator::setExhaustiveSearch(int mode)
|
||||||
|
{
|
||||||
|
m_exhaustiveSearch = mode;
|
||||||
|
}
|
||||||
|
|
||||||
void TileGenerator::parseColorsFile(const std::string &fileName)
|
void TileGenerator::parseColorsFile(const std::string &fileName)
|
||||||
{
|
{
|
||||||
ifstream in;
|
ifstream in;
|
||||||
@ -222,6 +228,7 @@ void TileGenerator::printGeometry(const std::string &input)
|
|||||||
input_path += PATH_SEPARATOR;
|
input_path += PATH_SEPARATOR;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
setExhaustiveSearch(EXH_NEVER);
|
||||||
openDb(input_path);
|
openDb(input_path);
|
||||||
loadBlocks();
|
loadBlocks();
|
||||||
|
|
||||||
@ -247,6 +254,8 @@ void TileGenerator::generate(const std::string &input, const std::string &output
|
|||||||
input_path += PATH_SEPARATOR;
|
input_path += PATH_SEPARATOR;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (m_dontWriteEmpty) // FIXME: possible too, just needs to be done differently
|
||||||
|
setExhaustiveSearch(EXH_NEVER);
|
||||||
openDb(input_path);
|
openDb(input_path);
|
||||||
loadBlocks();
|
loadBlocks();
|
||||||
|
|
||||||
@ -329,6 +338,35 @@ void TileGenerator::openDb(const std::string &input)
|
|||||||
#endif
|
#endif
|
||||||
else
|
else
|
||||||
throw std::runtime_error(((std::string) "Unknown map backend: ") + backend);
|
throw std::runtime_error(((std::string) "Unknown map backend: ") + backend);
|
||||||
|
|
||||||
|
// Determine how we're going to traverse the database (heuristic)
|
||||||
|
if (m_exhaustiveSearch == EXH_AUTO) {
|
||||||
|
using u64 = uint64_t;
|
||||||
|
u64 y_range = (m_yMax / 16 + 1) - (m_yMin / 16);
|
||||||
|
u64 blocks = (u64)(m_geomX2 - m_geomX) * y_range * (u64)(m_geomY2 - m_geomY);
|
||||||
|
#ifndef NDEBUG
|
||||||
|
std::cout << "Heuristic parameters:"
|
||||||
|
<< " preferRangeQueries()=" << m_db->preferRangeQueries()
|
||||||
|
<< " y_range=" << y_range << " blocks=" << blocks << std::endl;
|
||||||
|
#endif
|
||||||
|
if (m_db->preferRangeQueries())
|
||||||
|
m_exhaustiveSearch = EXH_NEVER;
|
||||||
|
else if (blocks < 200000)
|
||||||
|
m_exhaustiveSearch = EXH_FULL;
|
||||||
|
else if (y_range < 600)
|
||||||
|
m_exhaustiveSearch = EXH_Y;
|
||||||
|
else
|
||||||
|
m_exhaustiveSearch = EXH_NEVER;
|
||||||
|
} else if (m_exhaustiveSearch == EXH_FULL || m_exhaustiveSearch == EXH_Y) {
|
||||||
|
if (m_db->preferRangeQueries()) {
|
||||||
|
std::cerr << "Note: The current database backend supports efficient "
|
||||||
|
"range queries, forcing exhaustive search should always result "
|
||||||
|
" in worse performance." << std::endl;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (m_exhaustiveSearch == EXH_Y)
|
||||||
|
m_exhaustiveSearch = EXH_NEVER; // (TODO remove when implemented)
|
||||||
|
assert(m_exhaustiveSearch != EXH_AUTO);
|
||||||
}
|
}
|
||||||
|
|
||||||
void TileGenerator::closeDatabase()
|
void TileGenerator::closeDatabase()
|
||||||
@ -340,6 +378,8 @@ void TileGenerator::closeDatabase()
|
|||||||
void TileGenerator::loadBlocks()
|
void TileGenerator::loadBlocks()
|
||||||
{
|
{
|
||||||
const int16_t yMax = m_yMax / 16 + 1;
|
const int16_t yMax = m_yMax / 16 + 1;
|
||||||
|
|
||||||
|
if (m_exhaustiveSearch == EXH_NEVER || m_exhaustiveSearch == EXH_Y) {
|
||||||
std::vector<BlockPos> vec = m_db->getBlockPos(
|
std::vector<BlockPos> vec = m_db->getBlockPos(
|
||||||
BlockPos(m_geomX, m_yMin / 16, m_geomY),
|
BlockPos(m_geomX, m_yMin / 16, m_geomY),
|
||||||
BlockPos(m_geomX2, yMax, m_geomY2)
|
BlockPos(m_geomX2, yMax, m_geomY2)
|
||||||
@ -371,6 +411,7 @@ void TileGenerator::loadBlocks()
|
|||||||
std::cout << "Loaded " << count
|
std::cout << "Loaded " << count
|
||||||
<< " positions (across Z: " << m_positions.size() << ") for rendering" << std::endl;
|
<< " positions (across Z: " << m_positions.size() << ") for rendering" << std::endl;
|
||||||
#endif
|
#endif
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void TileGenerator::createImage()
|
void TileGenerator::createImage()
|
||||||
@ -422,11 +463,7 @@ void TileGenerator::renderMap()
|
|||||||
BlockDecoder blk;
|
BlockDecoder blk;
|
||||||
const int16_t yMax = m_yMax / 16 + 1;
|
const int16_t yMax = m_yMax / 16 + 1;
|
||||||
|
|
||||||
for (auto it = m_positions.rbegin(); it != m_positions.rend(); ++it) {
|
auto renderSingle = [&] (int16_t xPos, int16_t zPos, BlockList &blockStack) {
|
||||||
int16_t zPos = it->first;
|
|
||||||
for (auto it2 = it->second.rbegin(); it2 != it->second.rend(); ++it2) {
|
|
||||||
int16_t xPos = *it2;
|
|
||||||
|
|
||||||
m_readPixels.reset();
|
m_readPixels.reset();
|
||||||
m_readInfo.reset();
|
m_readInfo.reset();
|
||||||
for (int i = 0; i < 16; i++) {
|
for (int i = 0; i < 16; i++) {
|
||||||
@ -437,9 +474,6 @@ void TileGenerator::renderMap()
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
BlockList blockStack;
|
|
||||||
m_db->getBlocksOnXZ(blockStack, xPos, zPos, m_yMin / 16, yMax);
|
|
||||||
blockStack.sort();
|
|
||||||
for (const auto &it : blockStack) {
|
for (const auto &it : blockStack) {
|
||||||
const BlockPos pos = it.first;
|
const BlockPos pos = it.first;
|
||||||
assert(pos.x == xPos && pos.z == zPos);
|
assert(pos.x == xPos && pos.z == zPos);
|
||||||
@ -457,9 +491,48 @@ void TileGenerator::renderMap()
|
|||||||
}
|
}
|
||||||
if (!m_readPixels.full())
|
if (!m_readPixels.full())
|
||||||
renderMapBlockBottom(blockStack.begin()->first);
|
renderMapBlockBottom(blockStack.begin()->first);
|
||||||
}
|
};
|
||||||
|
auto postRenderRow = [&] (int16_t zPos) {
|
||||||
if (m_shading)
|
if (m_shading)
|
||||||
renderShading(zPos);
|
renderShading(zPos);
|
||||||
|
};
|
||||||
|
|
||||||
|
if (m_exhaustiveSearch == EXH_NEVER) {
|
||||||
|
for (auto it = m_positions.rbegin(); it != m_positions.rend(); ++it) {
|
||||||
|
int16_t zPos = it->first;
|
||||||
|
for (auto it2 = it->second.rbegin(); it2 != it->second.rend(); ++it2) {
|
||||||
|
int16_t xPos = *it2;
|
||||||
|
|
||||||
|
BlockList blockStack;
|
||||||
|
m_db->getBlocksOnXZ(blockStack, xPos, zPos, m_yMin / 16, yMax);
|
||||||
|
blockStack.sort();
|
||||||
|
|
||||||
|
renderSingle(xPos, zPos, blockStack);
|
||||||
|
}
|
||||||
|
postRenderRow(zPos);
|
||||||
|
}
|
||||||
|
} else if (m_exhaustiveSearch == EXH_FULL) {
|
||||||
|
#ifndef NDEBUG
|
||||||
|
std::cerr << "Exhaustively searching "
|
||||||
|
<< (m_geomX2 - m_geomX) << "x" << (yMax - (m_yMin / 16)) << "x"
|
||||||
|
<< (m_geomY2 - m_geomY) << " blocks" << std::endl;
|
||||||
|
#endif
|
||||||
|
std::vector<BlockPos> positions;
|
||||||
|
positions.reserve(yMax - (m_yMin / 16));
|
||||||
|
for (int16_t zPos = m_geomY2 - 1; zPos >= m_geomY; zPos--) {
|
||||||
|
for (int16_t xPos = m_geomX2 - 1; xPos >= m_geomX; xPos--) {
|
||||||
|
positions.clear();
|
||||||
|
for (int16_t yPos = m_yMin / 16; yPos < yMax; yPos++)
|
||||||
|
positions.emplace_back(xPos, yPos, zPos);
|
||||||
|
|
||||||
|
BlockList blockStack;
|
||||||
|
m_db->getBlocksByPos(blockStack, positions);
|
||||||
|
blockStack.sort();
|
||||||
|
|
||||||
|
renderSingle(xPos, zPos, blockStack);
|
||||||
|
}
|
||||||
|
postRenderRow(zPos);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -96,3 +96,19 @@ void DBLevelDB::getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void DBLevelDB::getBlocksByPos(BlockList &blocks,
|
||||||
|
const std::vector<BlockPos> &positions)
|
||||||
|
{
|
||||||
|
std::string datastr;
|
||||||
|
leveldb::Status status;
|
||||||
|
|
||||||
|
for (auto pos : positions) {
|
||||||
|
status = db->Get(leveldb::ReadOptions(), i64tos(encodeBlockPos(pos)), &datastr);
|
||||||
|
if (status.ok()) {
|
||||||
|
blocks.emplace_back(
|
||||||
|
pos, ustring((unsigned char *) datastr.data(), datastr.size())
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
@ -38,6 +38,11 @@ DBPostgreSQL::DBPostgreSQL(const std::string &mapdir)
|
|||||||
" posX = $1::int4 AND posZ = $2::int4"
|
" posX = $1::int4 AND posZ = $2::int4"
|
||||||
" AND (posY BETWEEN $3::int4 AND $4::int4)"
|
" AND (posY BETWEEN $3::int4 AND $4::int4)"
|
||||||
);
|
);
|
||||||
|
prepareStatement(
|
||||||
|
"get_block_exact",
|
||||||
|
"SELECT data FROM blocks WHERE"
|
||||||
|
" posX = $1::int4 AND posY = $2::int4 AND posZ = $3::int4"
|
||||||
|
);
|
||||||
|
|
||||||
checkResults(PQexec(db, "START TRANSACTION;"));
|
checkResults(PQexec(db, "START TRANSACTION;"));
|
||||||
checkResults(PQexec(db, "SET TRANSACTION ISOLATION LEVEL REPEATABLE READ;"));
|
checkResults(PQexec(db, "SET TRANSACTION ISOLATION LEVEL REPEATABLE READ;"));
|
||||||
@ -48,12 +53,13 @@ DBPostgreSQL::~DBPostgreSQL()
|
|||||||
{
|
{
|
||||||
try {
|
try {
|
||||||
checkResults(PQexec(db, "COMMIT;"));
|
checkResults(PQexec(db, "COMMIT;"));
|
||||||
} catch (std::exception& caught) {
|
} catch (const std::exception& caught) {
|
||||||
std::cerr << "could not finalize: " << caught.what() << std::endl;
|
std::cerr << "could not finalize: " << caught.what() << std::endl;
|
||||||
}
|
}
|
||||||
PQfinish(db);
|
PQfinish(db);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
std::vector<BlockPos> DBPostgreSQL::getBlockPos(BlockPos min, BlockPos max)
|
std::vector<BlockPos> DBPostgreSQL::getBlockPos(BlockPos min, BlockPos max)
|
||||||
{
|
{
|
||||||
int32_t const x1 = htonl(min.x);
|
int32_t const x1 = htonl(min.x);
|
||||||
@ -124,6 +130,43 @@ void DBPostgreSQL::getBlocksOnXZ(BlockList &blocks, int16_t xPos, int16_t zPos,
|
|||||||
PQclear(results);
|
PQclear(results);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
void DBPostgreSQL::getBlocksByPos(BlockList &blocks,
|
||||||
|
const std::vector<BlockPos> &positions)
|
||||||
|
{
|
||||||
|
int32_t x, y, z;
|
||||||
|
|
||||||
|
const void *args[] = { &x, &y, &z };
|
||||||
|
const int argLen[] = { 4, 4, 4 };
|
||||||
|
const int argFmt[] = { 1, 1, 1 };
|
||||||
|
|
||||||
|
for (auto pos : positions) {
|
||||||
|
x = htonl(pos.x);
|
||||||
|
y = htonl(pos.y);
|
||||||
|
z = htonl(pos.z);
|
||||||
|
|
||||||
|
PGresult *results = execPrepared(
|
||||||
|
"get_block_exact", ARRLEN(args), args,
|
||||||
|
argLen, argFmt, false
|
||||||
|
);
|
||||||
|
|
||||||
|
if (PQntuples(results) > 0) {
|
||||||
|
blocks.emplace_back(
|
||||||
|
pos,
|
||||||
|
ustring(
|
||||||
|
reinterpret_cast<unsigned char*>(
|
||||||
|
PQgetvalue(results, 0, 0)
|
||||||
|
),
|
||||||
|
PQgetlength(results, 0, 0)
|
||||||
|
)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
PQclear(results);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
PGresult *DBPostgreSQL::checkResults(PGresult *res, bool clear)
|
PGresult *DBPostgreSQL::checkResults(PGresult *res, bool clear)
|
||||||
{
|
{
|
||||||
ExecStatusType statusType = PQresultStatus(res);
|
ExecStatusType statusType = PQresultStatus(res);
|
||||||
|
46
db-redis.cpp
46
db-redis.cpp
@ -82,7 +82,7 @@ std::vector<BlockPos> DBRedis::getBlockPos(BlockPos min, BlockPos max)
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
std::string DBRedis::replyTypeStr(int type) {
|
const char *DBRedis::replyTypeStr(int type) {
|
||||||
switch(type) {
|
switch(type) {
|
||||||
case REDIS_REPLY_STATUS:
|
case REDIS_REPLY_STATUS:
|
||||||
return "REDIS_REPLY_STATUS";
|
return "REDIS_REPLY_STATUS";
|
||||||
@ -121,7 +121,8 @@ void DBRedis::loadPosCache()
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
void DBRedis::HMGET(const std::vector<BlockPos> &positions, std::vector<ustring> *result)
|
void DBRedis::HMGET(const std::vector<BlockPos> &positions,
|
||||||
|
std::function<void(std::size_t, ustring)> result)
|
||||||
{
|
{
|
||||||
const char *argv[DB_REDIS_HMGET_NUMFIELDS + 2];
|
const char *argv[DB_REDIS_HMGET_NUMFIELDS + 2];
|
||||||
argv[0] = "HMGET";
|
argv[0] = "HMGET";
|
||||||
@ -146,27 +147,21 @@ void DBRedis::HMGET(const std::vector<BlockPos> &positions, std::vector<ustring>
|
|||||||
|
|
||||||
if(!reply)
|
if(!reply)
|
||||||
throw std::runtime_error("Redis command HMGET failed");
|
throw std::runtime_error("Redis command HMGET failed");
|
||||||
if (reply->type != REDIS_REPLY_ARRAY) {
|
if (reply->type != REDIS_REPLY_ARRAY)
|
||||||
freeReplyObject(reply);
|
REPLY_TYPE_ERR(reply, "HMGET reply");
|
||||||
REPLY_TYPE_ERR(reply, "HKEYS subreply");
|
|
||||||
}
|
|
||||||
if (reply->elements != batch_size) {
|
if (reply->elements != batch_size) {
|
||||||
freeReplyObject(reply);
|
freeReplyObject(reply);
|
||||||
throw std::runtime_error("HMGET wrong number of elements");
|
throw std::runtime_error("HMGET wrong number of elements");
|
||||||
}
|
}
|
||||||
for (std::size_t i = 0; i < batch_size; ++i) {
|
for (std::size_t i = 0; i < reply->elements; ++i) {
|
||||||
redisReply *subreply = reply->element[i];
|
redisReply *subreply = reply->element[i];
|
||||||
if(!subreply)
|
if (subreply->type == REDIS_REPLY_NIL)
|
||||||
throw std::runtime_error("Redis command HMGET failed");
|
continue;
|
||||||
if (subreply->type != REDIS_REPLY_STRING) {
|
else if (subreply->type != REDIS_REPLY_STRING)
|
||||||
freeReplyObject(reply);
|
REPLY_TYPE_ERR(subreply, "HMGET subreply");
|
||||||
REPLY_TYPE_ERR(reply, "HKEYS subreply");
|
if (subreply->len == 0)
|
||||||
}
|
|
||||||
if (subreply->len == 0) {
|
|
||||||
freeReplyObject(reply);
|
|
||||||
throw std::runtime_error("HMGET empty string");
|
throw std::runtime_error("HMGET empty string");
|
||||||
}
|
result(i, ustring((const unsigned char *) subreply->str, subreply->len));
|
||||||
result->emplace_back((const unsigned char *) subreply->str, subreply->len);
|
|
||||||
}
|
}
|
||||||
freeReplyObject(reply);
|
freeReplyObject(reply);
|
||||||
remaining -= batch_size;
|
remaining -= batch_size;
|
||||||
@ -187,10 +182,15 @@ void DBRedis::getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
|
|||||||
positions.emplace_back(x, pos2.second, z);
|
positions.emplace_back(x, pos2.second, z);
|
||||||
}
|
}
|
||||||
|
|
||||||
std::vector<ustring> db_blocks;
|
getBlocksByPos(blocks, positions);
|
||||||
HMGET(positions, &db_blocks);
|
}
|
||||||
|
|
||||||
auto block = db_blocks.cbegin();
|
|
||||||
for (auto pos = positions.cbegin(); pos != positions.cend(); ++pos, ++block)
|
void DBRedis::getBlocksByPos(BlockList &blocks,
|
||||||
blocks.emplace_back(*pos, *block);
|
const std::vector<BlockPos> &positions)
|
||||||
|
{
|
||||||
|
auto result = [&] (std::size_t i, ustring data) {
|
||||||
|
blocks.emplace_back(positions[i], std::move(data));
|
||||||
|
};
|
||||||
|
HMGET(positions, result);
|
||||||
}
|
}
|
||||||
|
@ -25,6 +25,10 @@ DBSQLite3::DBSQLite3(const std::string &mapdir)
|
|||||||
"SELECT pos, data FROM blocks WHERE pos BETWEEN ? AND ?",
|
"SELECT pos, data FROM blocks WHERE pos BETWEEN ? AND ?",
|
||||||
-1, &stmt_get_blocks_z, NULL))
|
-1, &stmt_get_blocks_z, NULL))
|
||||||
|
|
||||||
|
SQLOK(prepare_v2(db,
|
||||||
|
"SELECT data FROM blocks WHERE pos = ?",
|
||||||
|
-1, &stmt_get_block_exact, NULL))
|
||||||
|
|
||||||
SQLOK(prepare_v2(db,
|
SQLOK(prepare_v2(db,
|
||||||
"SELECT pos FROM blocks",
|
"SELECT pos FROM blocks",
|
||||||
-1, &stmt_get_block_pos, NULL))
|
-1, &stmt_get_block_pos, NULL))
|
||||||
@ -40,6 +44,7 @@ DBSQLite3::~DBSQLite3()
|
|||||||
sqlite3_finalize(stmt_get_blocks_z);
|
sqlite3_finalize(stmt_get_blocks_z);
|
||||||
sqlite3_finalize(stmt_get_block_pos);
|
sqlite3_finalize(stmt_get_block_pos);
|
||||||
sqlite3_finalize(stmt_get_block_pos_z);
|
sqlite3_finalize(stmt_get_block_pos_z);
|
||||||
|
sqlite3_finalize(stmt_get_block_exact);
|
||||||
|
|
||||||
if (sqlite3_close(db) != SQLITE_OK) {
|
if (sqlite3_close(db) != SQLITE_OK) {
|
||||||
std::cerr << "Error closing SQLite database." << std::endl;
|
std::cerr << "Error closing SQLite database." << std::endl;
|
||||||
@ -161,3 +166,31 @@ void DBSQLite3::getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
|
|||||||
it++;
|
it++;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
void DBSQLite3::getBlocksByPos(BlockList &blocks,
|
||||||
|
const std::vector<BlockPos> &positions)
|
||||||
|
{
|
||||||
|
int result;
|
||||||
|
|
||||||
|
for (auto pos : positions) {
|
||||||
|
int64_t dbPos = encodeBlockPos(pos);
|
||||||
|
SQLOK(bind_int64(stmt_get_block_exact, 1, dbPos));
|
||||||
|
|
||||||
|
while ((result = sqlite3_step(stmt_get_block_exact)) == SQLITE_BUSY) {
|
||||||
|
usleep(10000); // Wait some time and try again
|
||||||
|
}
|
||||||
|
if (result == SQLITE_DONE) {
|
||||||
|
// no data
|
||||||
|
} else if (result != SQLITE_ROW) {
|
||||||
|
throw std::runtime_error(sqlite3_errmsg(db));
|
||||||
|
} else {
|
||||||
|
const unsigned char *data = reinterpret_cast<const unsigned char *>(
|
||||||
|
sqlite3_column_blob(stmt_get_block_exact, 0));
|
||||||
|
size_t size = sqlite3_column_bytes(stmt_get_block_exact, 0);
|
||||||
|
blocks.emplace_back(pos, ustring(data, size));
|
||||||
|
}
|
||||||
|
|
||||||
|
SQLOK(reset(stmt_get_block_exact))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
@ -23,6 +23,13 @@ enum {
|
|||||||
SCALE_RIGHT = (1 << 3),
|
SCALE_RIGHT = (1 << 3),
|
||||||
};
|
};
|
||||||
|
|
||||||
|
enum {
|
||||||
|
EXH_NEVER, // Always use range queries
|
||||||
|
EXH_Y, // Exhaustively search Y space, range queries for X/Z (future TODO)
|
||||||
|
EXH_FULL, // Exhaustively search entire requested geometry
|
||||||
|
EXH_AUTO, // Automatically pick one of the previous modes
|
||||||
|
};
|
||||||
|
|
||||||
struct ColorEntry {
|
struct ColorEntry {
|
||||||
ColorEntry(): r(0), g(0), b(0), a(0), t(0) {};
|
ColorEntry(): r(0), g(0), b(0), a(0), t(0) {};
|
||||||
ColorEntry(uint8_t r, uint8_t g, uint8_t b, uint8_t a, uint8_t t): r(r), g(g), b(b), a(a), t(t) {};
|
ColorEntry(uint8_t r, uint8_t g, uint8_t b, uint8_t a, uint8_t t): r(r), g(g), b(b), a(a), t(t) {};
|
||||||
@ -75,6 +82,7 @@ public:
|
|||||||
void setGeometry(int x, int y, int w, int h);
|
void setGeometry(int x, int y, int w, int h);
|
||||||
void setMinY(int y);
|
void setMinY(int y);
|
||||||
void setMaxY(int y);
|
void setMaxY(int y);
|
||||||
|
void setExhaustiveSearch(int mode);
|
||||||
void parseColorsFile(const std::string &fileName);
|
void parseColorsFile(const std::string &fileName);
|
||||||
void setBackend(std::string backend);
|
void setBackend(std::string backend);
|
||||||
void generate(const std::string &input, const std::string &output);
|
void generate(const std::string &input, const std::string &output);
|
||||||
@ -135,6 +143,7 @@ private:
|
|||||||
/* */
|
/* */
|
||||||
int m_mapWidth;
|
int m_mapWidth;
|
||||||
int m_mapHeight;
|
int m_mapHeight;
|
||||||
|
int m_exhaustiveSearch;
|
||||||
std::map<int16_t, std::set<int16_t>> m_positions; /* indexed by Z, contains X coords */
|
std::map<int16_t, std::set<int16_t>> m_positions; /* indexed by Z, contains X coords */
|
||||||
ColorMap m_colorMap;
|
ColorMap m_colorMap;
|
||||||
BitmapThing m_readPixels;
|
BitmapThing m_readPixels;
|
||||||
|
@ -12,8 +12,12 @@ public:
|
|||||||
std::vector<BlockPos> getBlockPos(BlockPos min, BlockPos max) override;
|
std::vector<BlockPos> getBlockPos(BlockPos min, BlockPos max) override;
|
||||||
void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
|
void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
|
||||||
int16_t min_y, int16_t max_y) override;
|
int16_t min_y, int16_t max_y) override;
|
||||||
|
void getBlocksByPos(BlockList &blocks,
|
||||||
|
const std::vector<BlockPos> &positions) override;
|
||||||
~DBLevelDB() override;
|
~DBLevelDB() override;
|
||||||
|
|
||||||
|
bool preferRangeQueries() const override { return false; }
|
||||||
|
|
||||||
private:
|
private:
|
||||||
using pos2d = std::pair<int16_t, int16_t>;
|
using pos2d = std::pair<int16_t, int16_t>;
|
||||||
|
|
||||||
|
@ -10,8 +10,12 @@ public:
|
|||||||
std::vector<BlockPos> getBlockPos(BlockPos min, BlockPos max) override;
|
std::vector<BlockPos> getBlockPos(BlockPos min, BlockPos max) override;
|
||||||
void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
|
void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
|
||||||
int16_t min_y, int16_t max_y) override;
|
int16_t min_y, int16_t max_y) override;
|
||||||
|
void getBlocksByPos(BlockList &blocks,
|
||||||
|
const std::vector<BlockPos> &positions) override;
|
||||||
~DBPostgreSQL() override;
|
~DBPostgreSQL() override;
|
||||||
|
|
||||||
|
bool preferRangeQueries() const override { return true; }
|
||||||
|
|
||||||
protected:
|
protected:
|
||||||
PGresult *checkResults(PGresult *res, bool clear = true);
|
PGresult *checkResults(PGresult *res, bool clear = true);
|
||||||
void prepareStatement(const std::string &name, const std::string &sql);
|
void prepareStatement(const std::string &name, const std::string &sql);
|
||||||
|
@ -4,6 +4,7 @@
|
|||||||
#include "db.h"
|
#include "db.h"
|
||||||
#include <unordered_map>
|
#include <unordered_map>
|
||||||
#include <utility>
|
#include <utility>
|
||||||
|
#include <functional>
|
||||||
#include <hiredis/hiredis.h>
|
#include <hiredis/hiredis.h>
|
||||||
|
|
||||||
class DBRedis : public DB {
|
class DBRedis : public DB {
|
||||||
@ -12,14 +13,19 @@ public:
|
|||||||
std::vector<BlockPos> getBlockPos(BlockPos min, BlockPos max) override;
|
std::vector<BlockPos> getBlockPos(BlockPos min, BlockPos max) override;
|
||||||
void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
|
void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
|
||||||
int16_t min_y, int16_t max_y) override;
|
int16_t min_y, int16_t max_y) override;
|
||||||
|
void getBlocksByPos(BlockList &blocks,
|
||||||
|
const std::vector<BlockPos> &positions) override;
|
||||||
~DBRedis() override;
|
~DBRedis() override;
|
||||||
|
|
||||||
|
bool preferRangeQueries() const override { return false; }
|
||||||
|
|
||||||
private:
|
private:
|
||||||
using pos2d = std::pair<int16_t, int16_t>;
|
using pos2d = std::pair<int16_t, int16_t>;
|
||||||
static std::string replyTypeStr(int type);
|
static const char *replyTypeStr(int type);
|
||||||
|
|
||||||
void loadPosCache();
|
void loadPosCache();
|
||||||
void HMGET(const std::vector<BlockPos> &positions, std::vector<ustring> *result);
|
void HMGET(const std::vector<BlockPos> &positions,
|
||||||
|
std::function<void(std::size_t, ustring)> result);
|
||||||
|
|
||||||
// indexed by Z, contains all (x,y) position pairs
|
// indexed by Z, contains all (x,y) position pairs
|
||||||
std::unordered_map<int16_t, std::vector<pos2d>> posCache;
|
std::unordered_map<int16_t, std::vector<pos2d>> posCache;
|
||||||
|
@ -11,8 +11,12 @@ public:
|
|||||||
std::vector<BlockPos> getBlockPos(BlockPos min, BlockPos max) override;
|
std::vector<BlockPos> getBlockPos(BlockPos min, BlockPos max) override;
|
||||||
void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
|
void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
|
||||||
int16_t min_y, int16_t max_y) override;
|
int16_t min_y, int16_t max_y) override;
|
||||||
|
void getBlocksByPos(BlockList &blocks,
|
||||||
|
const std::vector<BlockPos> &positions) override;
|
||||||
~DBSQLite3() override;
|
~DBSQLite3() override;
|
||||||
|
|
||||||
|
bool preferRangeQueries() const override { return false; }
|
||||||
|
|
||||||
private:
|
private:
|
||||||
inline void getPosRange(int64_t &min, int64_t &max, int16_t zPos,
|
inline void getPosRange(int64_t &min, int64_t &max, int16_t zPos,
|
||||||
int16_t zPos2) const;
|
int16_t zPos2) const;
|
||||||
@ -23,6 +27,7 @@ private:
|
|||||||
sqlite3_stmt *stmt_get_block_pos;
|
sqlite3_stmt *stmt_get_block_pos;
|
||||||
sqlite3_stmt *stmt_get_block_pos_z;
|
sqlite3_stmt *stmt_get_block_pos_z;
|
||||||
sqlite3_stmt *stmt_get_blocks_z;
|
sqlite3_stmt *stmt_get_blocks_z;
|
||||||
|
sqlite3_stmt *stmt_get_block_exact;
|
||||||
|
|
||||||
int16_t blockCachedZ = -10000;
|
int16_t blockCachedZ = -10000;
|
||||||
std::unordered_map<int16_t, BlockList> blockCache; // indexed by X
|
std::unordered_map<int16_t, BlockList> blockCache; // indexed by X
|
||||||
|
14
include/db.h
14
include/db.h
@ -52,11 +52,21 @@ public:
|
|||||||
* so that min.x <= x < max.x, ...
|
* so that min.x <= x < max.x, ...
|
||||||
*/
|
*/
|
||||||
virtual std::vector<BlockPos> getBlockPos(BlockPos min, BlockPos max) = 0;
|
virtual std::vector<BlockPos> getBlockPos(BlockPos min, BlockPos max) = 0;
|
||||||
/* Return all blocks in column given by x and z
|
/* Read all blocks in column given by x and z
|
||||||
* and inside the given Y range (min_y <= y < max_y)
|
* and inside the given Y range (min_y <= y < max_y) into list
|
||||||
*/
|
*/
|
||||||
virtual void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
|
virtual void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
|
||||||
int16_t min_y, int16_t max_y) = 0;
|
int16_t min_y, int16_t max_y) = 0;
|
||||||
|
/* Read blocks at given positions into list
|
||||||
|
*/
|
||||||
|
virtual void getBlocksByPos(BlockList &blocks,
|
||||||
|
const std::vector<BlockPos> &positions) = 0;
|
||||||
|
/* Can this database efficiently do range queries?
|
||||||
|
* (for large data sets, more efficient that brute force)
|
||||||
|
*/
|
||||||
|
virtual bool preferRangeQueries() const = 0;
|
||||||
|
|
||||||
|
|
||||||
virtual ~DB() {}
|
virtual ~DB() {}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
15
mapper.cpp
15
mapper.cpp
@ -33,6 +33,7 @@ static void usage()
|
|||||||
" --zoom <zoomlevel>\n"
|
" --zoom <zoomlevel>\n"
|
||||||
" --colors <colors.txt>\n"
|
" --colors <colors.txt>\n"
|
||||||
" --scales [t][b][l][r]\n"
|
" --scales [t][b][l][r]\n"
|
||||||
|
" --exhaustive never|y|full|auto\n"
|
||||||
"Color format: '#000000'\n";
|
"Color format: '#000000'\n";
|
||||||
std::cout << usage_text;
|
std::cout << usage_text;
|
||||||
}
|
}
|
||||||
@ -90,6 +91,7 @@ int main(int argc, char *argv[])
|
|||||||
{"colors", required_argument, 0, 'C'},
|
{"colors", required_argument, 0, 'C'},
|
||||||
{"scales", required_argument, 0, 'f'},
|
{"scales", required_argument, 0, 'f'},
|
||||||
{"noemptyimage", no_argument, 0, 'n'},
|
{"noemptyimage", no_argument, 0, 'n'},
|
||||||
|
{"exhaustive", required_argument, 0, 'j'},
|
||||||
{0, 0, 0, 0}
|
{0, 0, 0, 0}
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -201,6 +203,19 @@ int main(int argc, char *argv[])
|
|||||||
case 'n':
|
case 'n':
|
||||||
generator.setDontWriteEmpty(true);
|
generator.setDontWriteEmpty(true);
|
||||||
break;
|
break;
|
||||||
|
case 'j': {
|
||||||
|
int mode;
|
||||||
|
if (!strcmp(optarg, "never"))
|
||||||
|
mode = EXH_NEVER;
|
||||||
|
else if (!strcmp(optarg, "y"))
|
||||||
|
mode = EXH_Y;
|
||||||
|
else if (!strcmp(optarg, "full"))
|
||||||
|
mode = EXH_FULL;
|
||||||
|
else
|
||||||
|
mode = EXH_AUTO;
|
||||||
|
generator.setExhaustiveSearch(mode);
|
||||||
|
}
|
||||||
|
break;
|
||||||
default:
|
default:
|
||||||
exit(1);
|
exit(1);
|
||||||
}
|
}
|
||||||
|
@ -70,11 +70,11 @@ Don't draw nodes above this y value, e.g. "--max-y 75"
|
|||||||
|
|
||||||
.TP
|
.TP
|
||||||
.BR \-\-backend " " \fIbackend\fR
|
.BR \-\-backend " " \fIbackend\fR
|
||||||
Use specific map backend; supported: *sqlite3*, *leveldb*, *redis*, *postgresql*, e.g. "--backend leveldb"
|
Use specific map backend; supported: \fIsqlite3\fP, \fIleveldb\fP, \fIredis\fP, \fIpostgresql\fP, e.g. "--backend leveldb"
|
||||||
|
|
||||||
.TP
|
.TP
|
||||||
.BR \-\-geometry " " \fIgeometry\fR
|
.BR \-\-geometry " " \fIgeometry\fR
|
||||||
Limit area to specific geometry (*x:y+w+h* where x and y specify the lower left corner), e.g. "--geometry -800:-800+1600+1600"
|
Limit area to specific geometry (\fIx:y+w+h\fP where x and y specify the lower left corner), e.g. "--geometry -800:-800+1600+1600"
|
||||||
|
|
||||||
.TP
|
.TP
|
||||||
.BR \-\-extent " " \fIextent\fR
|
.BR \-\-extent " " \fIextent\fR
|
||||||
@ -90,7 +90,13 @@ Forcefully set path to colors.txt file (it's autodetected otherwise), e.g. "--co
|
|||||||
|
|
||||||
.TP
|
.TP
|
||||||
.BR \-\-scales " " \fIedges\fR
|
.BR \-\-scales " " \fIedges\fR
|
||||||
Draw scales on specified image edges (letters *t b l r* meaning top, bottom, left and right), e.g. "--scales tbr"
|
Draw scales on specified image edges (letters \fIt b l r\fP meaning top, bottom, left and right), e.g. "--scales tbr"
|
||||||
|
|
||||||
|
.TP
|
||||||
|
.BR \-\-exhaustive " \fImode\fR
|
||||||
|
Select if database should be traversed exhaustively or using range queries, available: \fInever\fP, \fIy\fP, \fIfull\fP, \fIauto\fP
|
||||||
|
|
||||||
|
Defaults to \fIauto\fP. You shouldn't need to change this, but doing so can improve rendering times on large maps.
|
||||||
|
|
||||||
.SH MORE INFORMATION
|
.SH MORE INFORMATION
|
||||||
Website: https://github.com/minetest/minetestmapper
|
Website: https://github.com/minetest/minetestmapper
|
||||||
|
Loading…
Reference in New Issue
Block a user