From 5b264fd4431c6af910544ad7b3b1cb621b51ea6c Mon Sep 17 00:00:00 2001 From: sfan5 Date: Fri, 27 Mar 2020 16:12:26 +0100 Subject: [PATCH] Rewrite DB class to allow backends to fully optimize block fetches --- TileGenerator.cpp | 85 +++++++++++++++------------- db-leveldb.cpp | 48 ++++++++++++---- db-postgresql.cpp | 73 ++++++++++++++---------- db-redis.cpp | 55 +++++++++++------- db-sqlite3.cpp | 121 +++++++++++++++++++++++++++++++--------- include/TileGenerator.h | 21 ++++--- include/db-leveldb.h | 16 ++++-- include/db-postgresql.h | 12 ++-- include/db-redis.h | 14 +++-- include/db-sqlite3.h | 17 +++++- include/db.h | 17 ++++-- 11 files changed, 325 insertions(+), 154 deletions(-) diff --git a/TileGenerator.cpp b/TileGenerator.cpp index ed4b58e..a2ad59a 100644 --- a/TileGenerator.cpp +++ b/TileGenerator.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -89,8 +90,8 @@ TileGenerator::TileGenerator(): m_xMax(INT_MIN), m_zMin(INT_MAX), m_zMax(INT_MIN), - m_yMin(-30000), - m_yMax(30000), + m_yMin(INT16_MIN), + m_yMax(INT16_MAX), m_geomX(-2048), m_geomY(-2048), m_geomX2(2048), @@ -184,6 +185,7 @@ void TileGenerator::setBackend(std::string backend) void TileGenerator::setGeometry(int x, int y, int w, int h) { + assert(w > 0 && h > 0); m_geomX = round_multiple_nosign(x, 16) / 16; m_geomY = round_multiple_nosign(y, 16) / 16; m_geomX2 = round_multiple_nosign(x + w, 16) / 16; @@ -193,11 +195,15 @@ void TileGenerator::setGeometry(int x, int y, int w, int h) void TileGenerator::setMinY(int y) { m_yMin = y; + if (m_yMin > m_yMax) + std::swap(m_yMin, m_yMax); } void TileGenerator::setMaxY(int y) { m_yMax = y; + if (m_yMin > m_yMax) + std::swap(m_yMin, m_yMax); } void TileGenerator::parseColorsFile(const std::string &fileName) @@ -244,7 +250,7 @@ void TileGenerator::generate(const std::string &input, const std::string &output openDb(input_path); loadBlocks(); - if (m_dontWriteEmpty && ! m_positions.size()) + if (m_dontWriteEmpty && m_positions.empty()) { closeDatabase(); return; @@ -268,7 +274,7 @@ void TileGenerator::generate(const std::string &input, const std::string &output void TileGenerator::parseColorsStream(std::istream &in) { - char line[128]; + char line[512]; while (in.good()) { in.getline(line, sizeof(line)); @@ -281,11 +287,11 @@ void TileGenerator::parseColorsStream(std::istream &in) if(strlen(line) == 0) continue; - char name[64 + 1] = {0}; + char name[128 + 1] = {0}; unsigned int r, g, b, a, t; a = 255; t = 0; - int items = sscanf(line, "%64s %u %u %u %u %u", name, &r, &g, &b, &a, &t); + int items = sscanf(line, "%128s %u %u %u %u %u", name, &r, &g, &b, &a, &t); if(items < 4) { std::cerr << "Failed to parse color entry '" << line << "'" << std::endl; continue; @@ -333,15 +339,17 @@ void TileGenerator::closeDatabase() void TileGenerator::loadBlocks() { - std::vector vec = m_db->getBlockPos(); - for (std::vector::iterator it = vec.begin(); it != vec.end(); ++it) { - BlockPos pos = *it; - // Check that it's in geometry (from --geometry option) - if (pos.x < m_geomX || pos.x >= m_geomX2 || pos.z < m_geomY || pos.z >= m_geomY2) - continue; - // Check that it's between --min-y and --max-y - if (pos.y * 16 < m_yMin || pos.y * 16 > m_yMax) - continue; + const int16_t yMax = m_yMax / 16 + 1; + std::vector vec = m_db->getBlockPos( + BlockPos(m_geomX, m_yMin / 16, m_geomY), + BlockPos(m_geomX2, yMax, m_geomY2) + ); + + for (auto pos : vec) { + assert(pos.x >= m_geomX && pos.x < m_geomX2); + assert(pos.y >= m_yMin / 16 && pos.y < yMax); + assert(pos.z >= m_geomY && pos.z < m_geomY2); + // Adjust minimum and maximum positions to the nearest block if (pos.x < m_xMin) m_xMin = pos.x; @@ -352,10 +360,17 @@ void TileGenerator::loadBlocks() m_zMin = pos.z; if (pos.z > m_zMax) m_zMax = pos.z; - m_positions.push_back(std::make_pair(pos.x, pos.z)); + + m_positions[pos.z].emplace(pos.x); } - m_positions.sort(); - m_positions.unique(); + +#ifndef NDEBUG + int count = 0; + for (const auto &it : m_positions) + count += it.second.size(); + std::cout << "Loaded " << count + << " positions (across Z: " << m_positions.size() << ") for rendering" << std::endl; +#endif } void TileGenerator::createImage() @@ -405,13 +420,12 @@ void TileGenerator::createImage() void TileGenerator::renderMap() { BlockDecoder blk; - std::list zlist = getZValueList(); - for (int16_t zPos : zlist) { - std::map blocks; - m_db->getBlocksOnZ(blocks, zPos); - for (const auto position : m_positions) { - if (position.second != zPos) - continue; + const int16_t yMax = m_yMax / 16 + 1; + + for (auto it = m_positions.rbegin(); it != m_positions.rend(); ++it) { + int16_t zPos = it->first; + for (auto it2 = it->second.rbegin(); it2 != it->second.rend(); ++it2) { + int16_t xPos = *it2; m_readPixels.reset(); m_readInfo.reset(); @@ -423,11 +437,13 @@ void TileGenerator::renderMap() } } - int16_t xPos = position.first; - blocks[xPos].sort(); - const BlockList &blockStack = blocks[xPos]; + BlockList blockStack; + m_db->getBlocksOnXZ(blockStack, xPos, zPos, m_yMin / 16, yMax); + blockStack.sort(); for (const auto &it : blockStack) { - const BlockPos &pos = it.first; + const BlockPos pos = it.first; + assert(pos.x == xPos && pos.z == zPos); + assert(pos.y >= m_yMin / 16 && pos.y < yMax); blk.reset(); blk.decode(it.second); @@ -651,17 +667,6 @@ void TileGenerator::renderPlayers(const std::string &inputPath) } } -inline std::list TileGenerator::getZValueList() const -{ - std::list zlist; - for (const auto position : m_positions) - zlist.push_back(position.second); - zlist.sort(); - zlist.unique(); - zlist.reverse(); - return zlist; -} - void TileGenerator::writeImage(const std::string &output) { m_image->save(output); diff --git a/db-leveldb.cpp b/db-leveldb.cpp index cbee57d..b7b2362 100644 --- a/db-leveldb.cpp +++ b/db-leveldb.cpp @@ -11,7 +11,6 @@ static inline int64_t stoi64(const std::string &s) return t; } - static inline std::string i64tos(int64_t i) { std::ostringstream os; @@ -19,6 +18,7 @@ static inline std::string i64tos(int64_t i) return os.str(); } + DBLevelDB::DBLevelDB(const std::string &mapdir) { leveldb::Options options; @@ -28,6 +28,9 @@ DBLevelDB::DBLevelDB(const std::string &mapdir) throw std::runtime_error(std::string("Failed to open Database: ") + status.ToString()); } + /* LevelDB is a dumb key-value store, so the only optimization we can do + * is to cache the block positions that exist in the db. + */ loadPosCache(); } @@ -38,9 +41,21 @@ DBLevelDB::~DBLevelDB() } -std::vector DBLevelDB::getBlockPos() +std::vector DBLevelDB::getBlockPos(BlockPos min, BlockPos max) { - return posCache; + std::vector res; + for (const auto &it : posCache) { + if (it.first < min.z || it.first >= max.z) + continue; + for (auto pos2 : it.second) { + if (pos2.first < min.x || pos2.first >= max.x) + continue; + if (pos2.second < min.y || pos2.second >= max.y) + continue; + res.emplace_back(pos2.first, pos2.second, it.first); + } + } + return res; } @@ -49,26 +64,35 @@ void DBLevelDB::loadPosCache() leveldb::Iterator * it = db->NewIterator(leveldb::ReadOptions()); for (it->SeekToFirst(); it->Valid(); it->Next()) { int64_t posHash = stoi64(it->key().ToString()); - posCache.push_back(decodeBlockPos(posHash)); + BlockPos pos = decodeBlockPos(posHash); + + posCache[pos.z].emplace_back(pos.x, pos.y); } delete it; } -void DBLevelDB::getBlocksOnZ(std::map &blocks, int16_t zPos) +void DBLevelDB::getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z, + int16_t min_y, int16_t max_y) { std::string datastr; leveldb::Status status; - for (const auto &it : posCache) { - if (it.z != zPos) { + auto it = posCache.find(z); + if (it == posCache.cend()) + return; + for (auto pos2 : it->second) { + if (pos2.first != x) continue; - } - status = db->Get(leveldb::ReadOptions(), i64tos(encodeBlockPos(it)), &datastr); + if (pos2.second < min_y || pos2.second >= max_y) + continue; + + BlockPos pos(x, pos2.second, z); + status = db->Get(leveldb::ReadOptions(), i64tos(encodeBlockPos(pos)), &datastr); if (status.ok()) { - Block b(it, ustring((const unsigned char *) datastr.data(), datastr.size())); - blocks[b.first.x].push_back(b); + blocks.emplace_back( + pos, ustring((unsigned char *) datastr.data(), datastr.size()) + ); } } } - diff --git a/db-postgresql.cpp b/db-postgresql.cpp index dc727c8..730961a 100644 --- a/db-postgresql.cpp +++ b/db-postgresql.cpp @@ -27,11 +27,16 @@ DBPostgreSQL::DBPostgreSQL(const std::string &mapdir) prepareStatement( "get_block_pos", - "SELECT posX, posY, posZ FROM blocks" + "SELECT posX::int4, posY::int4, posZ::int4 FROM blocks WHERE" + " (posX BETWEEN $1::int4 AND $2::int4) AND" + " (posY BETWEEN $3::int4 AND $4::int4) AND" + " (posZ BETWEEN $5::int4 AND $6::int4)" ); prepareStatement( - "get_blocks_z", - "SELECT posX, posY, data FROM blocks WHERE posZ = $1::int4" + "get_blocks", + "SELECT posY::int4, data FROM blocks WHERE" + " posX = $1::int4 AND posZ = $2::int4" + " AND (posY BETWEEN $3::int4 AND $4::int4)" ); checkResults(PQexec(db, "START TRANSACTION;")); @@ -49,19 +54,31 @@ DBPostgreSQL::~DBPostgreSQL() PQfinish(db); } -std::vector DBPostgreSQL::getBlockPos() +std::vector DBPostgreSQL::getBlockPos(BlockPos min, BlockPos max) { - std::vector positions; + int32_t const x1 = htonl(min.x); + int32_t const x2 = htonl(max.x - 1); + int32_t const y1 = htonl(min.y); + int32_t const y2 = htonl(max.y - 1); + int32_t const z1 = htonl(min.z); + int32_t const z2 = htonl(max.z - 1); + + const void *args[] = { &x1, &x2, &y1, &y2, &z1, &z2 }; + const int argLen[] = { 4, 4, 4, 4, 4, 4 }; + const int argFmt[] = { 1, 1, 1, 1, 1, 1 }; PGresult *results = execPrepared( - "get_block_pos", 0, - NULL, NULL, NULL, false, false + "get_block_pos", ARRLEN(args), args, + argLen, argFmt, false ); int numrows = PQntuples(results); + std::vector positions; + positions.reserve(numrows); + for (int row = 0; row < numrows; ++row) - positions.push_back(pg_to_blockpos(results, row, 0)); + positions.emplace_back(pg_to_blockpos(results, row, 0)); PQclear(results); @@ -69,16 +86,20 @@ std::vector DBPostgreSQL::getBlockPos() } -void DBPostgreSQL::getBlocksOnZ(std::map &blocks, int16_t zPos) +void DBPostgreSQL::getBlocksOnXZ(BlockList &blocks, int16_t xPos, int16_t zPos, + int16_t min_y, int16_t max_y) { + int32_t const x = htonl(xPos); int32_t const z = htonl(zPos); + int32_t const y1 = htonl(min_y); + int32_t const y2 = htonl(max_y - 1); - const void *args[] = { &z }; - const int argLen[] = { sizeof(z) }; - const int argFmt[] = { 1 }; + const void *args[] = { &x, &z, &y1, &y2 }; + const int argLen[] = { 4, 4, 4, 4 }; + const int argFmt[] = { 1, 1, 1, 1 }; PGresult *results = execPrepared( - "get_blocks_z", ARRLEN(args), args, + "get_blocks", ARRLEN(args), args, argLen, argFmt, false ); @@ -86,19 +107,18 @@ void DBPostgreSQL::getBlocksOnZ(std::map &blocks, int16_t zP for (int row = 0; row < numrows; ++row) { BlockPos position; - position.x = pg_binary_to_int(results, row, 0); - position.y = pg_binary_to_int(results, row, 1); + position.x = xPos; + position.y = pg_binary_to_int(results, row, 0); position.z = zPos; - Block const b( + blocks.emplace_back( position, ustring( reinterpret_cast( - PQgetvalue(results, row, 2) + PQgetvalue(results, row, 1) ), - PQgetlength(results, row, 2) + PQgetlength(results, row, 1) ) ); - blocks[position.x].push_back(b); } PQclear(results); @@ -138,20 +158,15 @@ PGresult *DBPostgreSQL::execPrepared( const char *stmtName, const int paramsNumber, const void **params, const int *paramsLengths, const int *paramsFormats, - bool clear, bool nobinary + bool clear ) { return checkResults(PQexecPrepared(db, stmtName, paramsNumber, (const char* const*) params, paramsLengths, paramsFormats, - nobinary ? 1 : 0), clear + 1 /* binary output */), clear ); } -int DBPostgreSQL::pg_to_int(PGresult *res, int row, int col) -{ - return atoi(PQgetvalue(res, row, col)); -} - int DBPostgreSQL::pg_binary_to_int(PGresult *res, int row, int col) { int32_t* raw = reinterpret_cast(PQgetvalue(res, row, col)); @@ -161,8 +176,8 @@ int DBPostgreSQL::pg_binary_to_int(PGresult *res, int row, int col) BlockPos DBPostgreSQL::pg_to_blockpos(PGresult *res, int row, int col) { BlockPos result; - result.x = pg_to_int(res, row, col); - result.y = pg_to_int(res, row, col + 1); - result.z = pg_to_int(res, row, col + 2); + result.x = pg_binary_to_int(res, row, col); + result.y = pg_binary_to_int(res, row, col + 1); + result.z = pg_binary_to_int(res, row, col + 2); return result; } diff --git a/db-redis.cpp b/db-redis.cpp index 53a5660..8ba3f97 100644 --- a/db-redis.cpp +++ b/db-redis.cpp @@ -51,6 +51,9 @@ DBRedis::DBRedis(const std::string &mapdir) throw std::runtime_error(err); } + /* Redis is just a key-value store, so the only optimization we can do + * is to cache the block positions that exist in the db. + */ loadPosCache(); } @@ -61,9 +64,21 @@ DBRedis::~DBRedis() } -std::vector DBRedis::getBlockPos() +std::vector DBRedis::getBlockPos(BlockPos min, BlockPos max) { - return posCache; + std::vector res; + for (const auto &it : posCache) { + if (it.first < min.z || it.first >= max.z) + continue; + for (auto pos2 : it.second) { + if (pos2.first < min.x || pos2.first >= max.x) + continue; + if (pos2.second < min.y || pos2.second >= max.y) + continue; + res.emplace_back(pos2.first, pos2.second, it.first); + } + } + return res; } @@ -98,7 +113,8 @@ void DBRedis::loadPosCache() for(size_t i = 0; i < reply->elements; i++) { if(reply->element[i]->type != REDIS_REPLY_STRING) REPLY_TYPE_ERR(reply->element[i], "HKEYS subreply"); - posCache.push_back(decodeBlockPos(stoi64(reply->element[i]->str))); + BlockPos pos = decodeBlockPos(stoi64(reply->element[i]->str)); + posCache[pos.z].emplace_back(pos.x, pos.y); } freeReplyObject(reply); @@ -150,7 +166,7 @@ void DBRedis::HMGET(const std::vector &positions, std::vector freeReplyObject(reply); throw std::runtime_error("HMGET empty string"); } - result->push_back(ustring((const unsigned char *) subreply->str, subreply->len)); + result->emplace_back((const unsigned char *) subreply->str, subreply->len); } freeReplyObject(reply); remaining -= batch_size; @@ -158,22 +174,23 @@ void DBRedis::HMGET(const std::vector &positions, std::vector } -void DBRedis::getBlocksOnZ(std::map &blocks, int16_t zPos) +void DBRedis::getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z, + int16_t min_y, int16_t max_y) { - std::vector z_positions; - for (std::vector::const_iterator it = posCache.begin(); it != posCache.end(); ++it) { - if (it->z != zPos) { - continue; - } - z_positions.push_back(*it); - } - std::vector z_blocks; - HMGET(z_positions, &z_blocks); + auto it = posCache.find(z); + if (it == posCache.cend()) + return; - std::vector::const_iterator z_block = z_blocks.begin(); - for (std::vector::const_iterator pos = z_positions.begin(); - pos != z_positions.end(); - ++pos, ++z_block) { - blocks[pos->x].push_back(Block(*pos, *z_block)); + std::vector positions; + for (auto pos2 : it->second) { + if (pos2.first == x && pos2.second >= min_y && pos2.second < max_y) + positions.emplace_back(x, pos2.second, z); } + + std::vector db_blocks; + HMGET(positions, &db_blocks); + + auto block = db_blocks.cbegin(); + for (auto pos = positions.cbegin(); pos != positions.cend(); ++pos, ++block) + blocks.emplace_back(*pos, *block); } diff --git a/db-sqlite3.cpp b/db-sqlite3.cpp index eb87d1a..87bcdec 100644 --- a/db-sqlite3.cpp +++ b/db-sqlite3.cpp @@ -1,6 +1,8 @@ #include #include // for usleep #include +#include +#include #include "db-sqlite3.h" #include "types.h" @@ -11,7 +13,6 @@ } #define SQLOK(f) SQLRES(f, SQLITE_OK) - DBSQLite3::DBSQLite3(const std::string &mapdir) { int result; @@ -27,6 +28,10 @@ DBSQLite3::DBSQLite3(const std::string &mapdir) SQLOK(prepare_v2(db, "SELECT pos FROM blocks", -1, &stmt_get_block_pos, NULL)) + + SQLOK(prepare_v2(db, + "SELECT pos FROM blocks WHERE pos BETWEEN ? AND ?", + -1, &stmt_get_block_pos_z, NULL)) } @@ -34,59 +39,125 @@ DBSQLite3::~DBSQLite3() { sqlite3_finalize(stmt_get_blocks_z); sqlite3_finalize(stmt_get_block_pos); + sqlite3_finalize(stmt_get_block_pos_z); if (sqlite3_close(db) != SQLITE_OK) { std::cerr << "Error closing SQLite database." << std::endl; }; } -std::vector DBSQLite3::getBlockPos() + +inline void DBSQLite3::getPosRange(int64_t &min, int64_t &max, int16_t zPos, + int16_t zPos2) const +{ + /* The range of block positions is [-2048, 2047], which turns into [0, 4095] + * when casted to unsigned. This didn't actually help me understand the + * numbers below, but I wanted to write it down. + */ + + // Magic numbers! + min = encodeBlockPos(BlockPos(0, -2048, zPos)); + max = encodeBlockPos(BlockPos(0, 2048, zPos2)) - 1; +} + + +std::vector DBSQLite3::getBlockPos(BlockPos min, BlockPos max) { int result; + sqlite3_stmt *stmt; + + if(min.z <= -2048 && max.z >= 2048) { + stmt = stmt_get_block_pos; + } else { + stmt = stmt_get_block_pos_z; + int64_t minPos, maxPos; + if (min.z < -2048) + min.z = -2048; + if (max.z > 2048) + max.z = 2048; + getPosRange(minPos, maxPos, min.z, max.z - 1); + SQLOK(bind_int64(stmt, 1, minPos)) + SQLOK(bind_int64(stmt, 2, maxPos)) + } + std::vector positions; - while ((result = sqlite3_step(stmt_get_block_pos)) != SQLITE_DONE) { - if (result == SQLITE_ROW) { - int64_t posHash = sqlite3_column_int64(stmt_get_block_pos, 0); - positions.push_back(decodeBlockPos(posHash)); - } else if (result == SQLITE_BUSY) { // Wait some time and try again + while ((result = sqlite3_step(stmt)) != SQLITE_DONE) { + if (result == SQLITE_BUSY) { // Wait some time and try again usleep(10000); - } else { + } else if (result != SQLITE_ROW) { throw std::runtime_error(sqlite3_errmsg(db)); } + + int64_t posHash = sqlite3_column_int64(stmt, 0); + BlockPos pos = decodeBlockPos(posHash); + if(pos.x >= min.x && pos.x < max.x && pos.y >= min.y && pos.y < max.y) + positions.emplace_back(pos); } - SQLOK(reset(stmt_get_block_pos)); + SQLOK(reset(stmt)); return positions; } -void DBSQLite3::getBlocksOnZ(std::map &blocks, int16_t zPos) +void DBSQLite3::loadBlockCache(int16_t zPos) { int result; + blockCache.clear(); - // Magic numbers! - int64_t minPos = encodeBlockPos(BlockPos(0, -2048, zPos)); - int64_t maxPos = encodeBlockPos(BlockPos(0, 2048, zPos)) - 1; + int64_t minPos, maxPos; + getPosRange(minPos, maxPos, zPos, zPos); SQLOK(bind_int64(stmt_get_blocks_z, 1, minPos)); SQLOK(bind_int64(stmt_get_blocks_z, 2, maxPos)); while ((result = sqlite3_step(stmt_get_blocks_z)) != SQLITE_DONE) { - if (result == SQLITE_ROW) { - int64_t posHash = sqlite3_column_int64(stmt_get_blocks_z, 0); - const unsigned char *data = reinterpret_cast( - sqlite3_column_blob(stmt_get_blocks_z, 1)); - size_t size = sqlite3_column_bytes(stmt_get_blocks_z, 1); - Block b(decodeBlockPos(posHash), ustring(data, size)); - blocks[b.first.x].push_back(b); - } else if (result == SQLITE_BUSY) { // Wait some time and try again + if (result == SQLITE_BUSY) { // Wait some time and try again usleep(10000); - } else { + } else if (result != SQLITE_ROW) { throw std::runtime_error(sqlite3_errmsg(db)); } + + int64_t posHash = sqlite3_column_int64(stmt_get_blocks_z, 0); + BlockPos pos = decodeBlockPos(posHash); + const unsigned char *data = reinterpret_cast( + sqlite3_column_blob(stmt_get_blocks_z, 1)); + size_t size = sqlite3_column_bytes(stmt_get_blocks_z, 1); + blockCache[pos.x].emplace_back(pos, ustring(data, size)); } - SQLOK(reset(stmt_get_blocks_z)); + SQLOK(reset(stmt_get_blocks_z)) } -#undef SQLRES -#undef SQLOK +void DBSQLite3::getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z, + int16_t min_y, int16_t max_y) +{ + /* Cache the blocks on the given Z coordinate between calls, this only + * works due to order in which the TileGenerator asks for blocks. */ + if (z != blockCachedZ) { + loadBlockCache(z); + blockCachedZ = z; + } + + auto it = blockCache.find(x); + if (it == blockCache.end()) + return; + + if (it->second.empty()) { + /* We have swapped this list before, this is not supposed to happen + * because it's bad for performance. But rather than silently breaking + * do the right thing and load the blocks again. */ +#ifndef NDEBUG + std::cout << "Warning: suboptimal access pattern for sqlite3 backend" << std::endl; +#endif + loadBlockCache(z); + } + // Swap lists to avoid copying contents + blocks.clear(); + std::swap(blocks, it->second); + + for (auto it = blocks.begin(); it != blocks.end(); ) { + if (it->first.y < min_y || it->first.y >= max_y) + it = blocks.erase(it); + else + it++; + } +} diff --git a/include/TileGenerator.h b/include/TileGenerator.h index dce1b89..60e0957 100644 --- a/include/TileGenerator.h +++ b/include/TileGenerator.h @@ -2,7 +2,8 @@ #define TILEGENERATOR_HEADER #include -#include +#include +#include #include #include #include @@ -64,7 +65,8 @@ public: void setBgColor(const std::string &bgColor); void setScaleColor(const std::string &scaleColor); void setOriginColor(const std::string &originColor); - void setPlayerColor(const std::string &playerColor); Color parseColor(const std::string &color); + void setPlayerColor(const std::string &playerColor); + Color parseColor(const std::string &color); void setDrawOrigin(bool drawOrigin); void setDrawPlayers(bool drawPlayers); void setDrawScale(bool drawScale); @@ -88,7 +90,6 @@ private: void loadBlocks(); void createImage(); void renderMap(); - std::list getZValueList() const; void renderMapBlock(const BlockDecoder &blk, const BlockPos &pos); void renderMapBlockBottom(const BlockPos &pos); void renderShading(int zPos); @@ -118,19 +119,23 @@ private: DB *m_db; Image *m_image; PixelAttributes m_blockPixelAttributes; + /* smallest/largest seen X or Z block coordinate */ int m_xMin; int m_xMax; int m_zMin; int m_zMax; + /* Y limits for rendered area (node units) */ int m_yMin; int m_yMax; - int m_geomX; - int m_geomY; - int m_geomX2; - int m_geomY2; + /* limits for rendered area (block units) */ + int16_t m_geomX; + int16_t m_geomY; /* Y in terms of rendered image, Z in the world */ + int16_t m_geomX2; + int16_t m_geomY2; + /* */ int m_mapWidth; int m_mapHeight; - std::list> m_positions; + std::map> m_positions; /* indexed by Z, contains X coords */ ColorMap m_colorMap; BitmapThing m_readPixels; BitmapThing m_readInfo; diff --git a/include/db-leveldb.h b/include/db-leveldb.h index e2a0e10..606f4cd 100644 --- a/include/db-leveldb.h +++ b/include/db-leveldb.h @@ -2,19 +2,25 @@ #define DB_LEVELDB_HEADER #include "db.h" +#include +#include #include class DBLevelDB : public DB { public: DBLevelDB(const std::string &mapdir); - virtual std::vector getBlockPos(); - virtual void getBlocksOnZ(std::map &blocks, int16_t zPos); - virtual ~DBLevelDB(); + std::vector getBlockPos(BlockPos min, BlockPos max) override; + void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z, + int16_t min_y, int16_t max_y) override; + ~DBLevelDB() override; + private: + using pos2d = std::pair; + void loadPosCache(); - std::vector posCache; - + // indexed by Z, contains all (x,y) position pairs + std::unordered_map> posCache; leveldb::DB *db; }; diff --git a/include/db-postgresql.h b/include/db-postgresql.h index 8692694..33e1881 100644 --- a/include/db-postgresql.h +++ b/include/db-postgresql.h @@ -7,9 +7,11 @@ class DBPostgreSQL : public DB { public: DBPostgreSQL(const std::string &mapdir); - virtual std::vector getBlockPos(); - virtual void getBlocksOnZ(std::map &blocks, int16_t zPos); - virtual ~DBPostgreSQL(); + std::vector getBlockPos(BlockPos min, BlockPos max) override; + void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z, + int16_t min_y, int16_t max_y) override; + ~DBPostgreSQL() override; + protected: PGresult *checkResults(PGresult *res, bool clear = true); void prepareStatement(const std::string &name, const std::string &sql); @@ -17,11 +19,11 @@ protected: const char *stmtName, const int paramsNumber, const void **params, const int *paramsLengths = NULL, const int *paramsFormats = NULL, - bool clear = true, bool nobinary = true + bool clear = true ); - int pg_to_int(PGresult *res, int row, int col); int pg_binary_to_int(PGresult *res, int row, int col); BlockPos pg_to_blockpos(PGresult *res, int row, int col); + private: PGconn *db; }; diff --git a/include/db-redis.h b/include/db-redis.h index b665abf..47c37bb 100644 --- a/include/db-redis.h +++ b/include/db-redis.h @@ -2,21 +2,27 @@ #define DB_REDIS_HEADER #include "db.h" +#include +#include #include class DBRedis : public DB { public: DBRedis(const std::string &mapdir); - virtual std::vector getBlockPos(); - virtual void getBlocksOnZ(std::map &blocks, int16_t zPos); - virtual ~DBRedis(); + std::vector getBlockPos(BlockPos min, BlockPos max) override; + void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z, + int16_t min_y, int16_t max_y) override; + ~DBRedis() override; + private: + using pos2d = std::pair; static std::string replyTypeStr(int type); void loadPosCache(); void HMGET(const std::vector &positions, std::vector *result); - std::vector posCache; + // indexed by Z, contains all (x,y) position pairs + std::unordered_map> posCache; redisContext *ctx; std::string hash; diff --git a/include/db-sqlite3.h b/include/db-sqlite3.h index 80fe07b..63b7fb1 100644 --- a/include/db-sqlite3.h +++ b/include/db-sqlite3.h @@ -2,19 +2,30 @@ #define _DB_SQLITE3_H #include "db.h" +#include #include class DBSQLite3 : public DB { public: DBSQLite3(const std::string &mapdir); - virtual std::vector getBlockPos(); - virtual void getBlocksOnZ(std::map &blocks, int16_t zPos); - virtual ~DBSQLite3(); + std::vector getBlockPos(BlockPos min, BlockPos max) override; + void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z, + int16_t min_y, int16_t max_y) override; + ~DBSQLite3() override; + private: + inline void getPosRange(int64_t &min, int64_t &max, int16_t zPos, + int16_t zPos2) const; + void loadBlockCache(int16_t zPos); + sqlite3 *db; sqlite3_stmt *stmt_get_block_pos; + sqlite3_stmt *stmt_get_block_pos_z; sqlite3_stmt *stmt_get_blocks_z; + + int16_t blockCachedZ = -10000; + std::unordered_map blockCache; // indexed by X }; #endif // _DB_SQLITE3_H diff --git a/include/db.h b/include/db.h index 6ed5aae..ae83723 100644 --- a/include/db.h +++ b/include/db.h @@ -5,7 +5,6 @@ #include #include #include -#include #include #include "types.h" @@ -17,6 +16,8 @@ struct BlockPos { BlockPos() : x(0), y(0), z(0) {} BlockPos(int16_t x, int16_t y, int16_t z) : x(x), y(y), z(z) {} + + // Implements the inverse ordering so that (2,2,2) < (1,1,1) bool operator < (const BlockPos &p) const { if (z > p.z) @@ -42,13 +43,21 @@ typedef std::list BlockList; class DB { protected: + // Helpers that implement the hashed positions used by most backends inline int64_t encodeBlockPos(const BlockPos pos) const; inline BlockPos decodeBlockPos(int64_t hash) const; public: - virtual std::vector getBlockPos() = 0; - virtual void getBlocksOnZ(std::map &blocks, int16_t zPos) = 0; - virtual ~DB() {}; + /* Return all block positions inside the range given by min and max, + * so that min.x <= x < max.x, ... + */ + virtual std::vector getBlockPos(BlockPos min, BlockPos max) = 0; + /* Return all blocks in column given by x and z + * and inside the given Y range (min_y <= y < max_y) + */ + virtual void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z, + int16_t min_y, int16_t max_y) = 0; + virtual ~DB() {} };