mirror of
https://github.com/minetest/minetestmapper.git
synced 2024-11-27 18:03:54 +01:00
Rewrite DB class to allow backends to fully optimize block fetches
This commit is contained in:
sfan5
parent
ecc2b31f78
commit
5b264fd443
@ -1,6 +1,7 @@
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <climits>
|
||||
#include <cassert>
|
||||
#include <fstream>
|
||||
#include <iostream>
|
||||
#include <sstream>
|
||||
@ -89,8 +90,8 @@ TileGenerator::TileGenerator():
|
||||
m_xMax(INT_MIN),
|
||||
m_zMin(INT_MAX),
|
||||
m_zMax(INT_MIN),
|
||||
m_yMin(-30000),
|
||||
m_yMax(30000),
|
||||
m_yMin(INT16_MIN),
|
||||
m_yMax(INT16_MAX),
|
||||
m_geomX(-2048),
|
||||
m_geomY(-2048),
|
||||
m_geomX2(2048),
|
||||
@ -184,6 +185,7 @@ void TileGenerator::setBackend(std::string backend)
|
||||
|
||||
void TileGenerator::setGeometry(int x, int y, int w, int h)
|
||||
{
|
||||
assert(w > 0 && h > 0);
|
||||
m_geomX = round_multiple_nosign(x, 16) / 16;
|
||||
m_geomY = round_multiple_nosign(y, 16) / 16;
|
||||
m_geomX2 = round_multiple_nosign(x + w, 16) / 16;
|
||||
@ -193,11 +195,15 @@ void TileGenerator::setGeometry(int x, int y, int w, int h)
|
||||
void TileGenerator::setMinY(int y)
|
||||
{
|
||||
m_yMin = y;
|
||||
if (m_yMin > m_yMax)
|
||||
std::swap(m_yMin, m_yMax);
|
||||
}
|
||||
|
||||
void TileGenerator::setMaxY(int y)
|
||||
{
|
||||
m_yMax = y;
|
||||
if (m_yMin > m_yMax)
|
||||
std::swap(m_yMin, m_yMax);
|
||||
}
|
||||
|
||||
void TileGenerator::parseColorsFile(const std::string &fileName)
|
||||
@ -244,7 +250,7 @@ void TileGenerator::generate(const std::string &input, const std::string &output
|
||||
openDb(input_path);
|
||||
loadBlocks();
|
||||
|
||||
if (m_dontWriteEmpty && ! m_positions.size())
|
||||
if (m_dontWriteEmpty && m_positions.empty())
|
||||
{
|
||||
closeDatabase();
|
||||
return;
|
||||
@ -268,7 +274,7 @@ void TileGenerator::generate(const std::string &input, const std::string &output
|
||||
|
||||
void TileGenerator::parseColorsStream(std::istream &in)
|
||||
{
|
||||
char line[128];
|
||||
char line[512];
|
||||
while (in.good()) {
|
||||
in.getline(line, sizeof(line));
|
||||
|
||||
@ -281,11 +287,11 @@ void TileGenerator::parseColorsStream(std::istream &in)
|
||||
if(strlen(line) == 0)
|
||||
continue;
|
||||
|
||||
char name[64 + 1] = {0};
|
||||
char name[128 + 1] = {0};
|
||||
unsigned int r, g, b, a, t;
|
||||
a = 255;
|
||||
t = 0;
|
||||
int items = sscanf(line, "%64s %u %u %u %u %u", name, &r, &g, &b, &a, &t);
|
||||
int items = sscanf(line, "%128s %u %u %u %u %u", name, &r, &g, &b, &a, &t);
|
||||
if(items < 4) {
|
||||
std::cerr << "Failed to parse color entry '" << line << "'" << std::endl;
|
||||
continue;
|
||||
@ -333,15 +339,17 @@ void TileGenerator::closeDatabase()
|
||||
|
||||
void TileGenerator::loadBlocks()
|
||||
{
|
||||
std::vector<BlockPos> vec = m_db->getBlockPos();
|
||||
for (std::vector<BlockPos>::iterator it = vec.begin(); it != vec.end(); ++it) {
|
||||
BlockPos pos = *it;
|
||||
// Check that it's in geometry (from --geometry option)
|
||||
if (pos.x < m_geomX || pos.x >= m_geomX2 || pos.z < m_geomY || pos.z >= m_geomY2)
|
||||
continue;
|
||||
// Check that it's between --min-y and --max-y
|
||||
if (pos.y * 16 < m_yMin || pos.y * 16 > m_yMax)
|
||||
continue;
|
||||
const int16_t yMax = m_yMax / 16 + 1;
|
||||
std::vector<BlockPos> vec = m_db->getBlockPos(
|
||||
BlockPos(m_geomX, m_yMin / 16, m_geomY),
|
||||
BlockPos(m_geomX2, yMax, m_geomY2)
|
||||
);
|
||||
|
||||
for (auto pos : vec) {
|
||||
assert(pos.x >= m_geomX && pos.x < m_geomX2);
|
||||
assert(pos.y >= m_yMin / 16 && pos.y < yMax);
|
||||
assert(pos.z >= m_geomY && pos.z < m_geomY2);
|
||||
|
||||
// Adjust minimum and maximum positions to the nearest block
|
||||
if (pos.x < m_xMin)
|
||||
m_xMin = pos.x;
|
||||
@ -352,10 +360,17 @@ void TileGenerator::loadBlocks()
|
||||
m_zMin = pos.z;
|
||||
if (pos.z > m_zMax)
|
||||
m_zMax = pos.z;
|
||||
m_positions.push_back(std::make_pair(pos.x, pos.z));
|
||||
|
||||
m_positions[pos.z].emplace(pos.x);
|
||||
}
|
||||
m_positions.sort();
|
||||
m_positions.unique();
|
||||
|
||||
#ifndef NDEBUG
|
||||
int count = 0;
|
||||
for (const auto &it : m_positions)
|
||||
count += it.second.size();
|
||||
std::cout << "Loaded " << count
|
||||
<< " positions (across Z: " << m_positions.size() << ") for rendering" << std::endl;
|
||||
#endif
|
||||
}
|
||||
|
||||
void TileGenerator::createImage()
|
||||
@ -405,13 +420,12 @@ void TileGenerator::createImage()
|
||||
void TileGenerator::renderMap()
|
||||
{
|
||||
BlockDecoder blk;
|
||||
std::list<int16_t> zlist = getZValueList();
|
||||
for (int16_t zPos : zlist) {
|
||||
std::map<int16_t, BlockList> blocks;
|
||||
m_db->getBlocksOnZ(blocks, zPos);
|
||||
for (const auto position : m_positions) {
|
||||
if (position.second != zPos)
|
||||
continue;
|
||||
const int16_t yMax = m_yMax / 16 + 1;
|
||||
|
||||
for (auto it = m_positions.rbegin(); it != m_positions.rend(); ++it) {
|
||||
int16_t zPos = it->first;
|
||||
for (auto it2 = it->second.rbegin(); it2 != it->second.rend(); ++it2) {
|
||||
int16_t xPos = *it2;
|
||||
|
||||
m_readPixels.reset();
|
||||
m_readInfo.reset();
|
||||
@ -423,11 +437,13 @@ void TileGenerator::renderMap()
|
||||
}
|
||||
}
|
||||
|
||||
int16_t xPos = position.first;
|
||||
blocks[xPos].sort();
|
||||
const BlockList &blockStack = blocks[xPos];
|
||||
BlockList blockStack;
|
||||
m_db->getBlocksOnXZ(blockStack, xPos, zPos, m_yMin / 16, yMax);
|
||||
blockStack.sort();
|
||||
for (const auto &it : blockStack) {
|
||||
const BlockPos &pos = it.first;
|
||||
const BlockPos pos = it.first;
|
||||
assert(pos.x == xPos && pos.z == zPos);
|
||||
assert(pos.y >= m_yMin / 16 && pos.y < yMax);
|
||||
|
||||
blk.reset();
|
||||
blk.decode(it.second);
|
||||
@ -651,17 +667,6 @@ void TileGenerator::renderPlayers(const std::string &inputPath)
|
||||
}
|
||||
}
|
||||
|
||||
inline std::list<int16_t> TileGenerator::getZValueList() const
|
||||
{
|
||||
std::list<int16_t> zlist;
|
||||
for (const auto position : m_positions)
|
||||
zlist.push_back(position.second);
|
||||
zlist.sort();
|
||||
zlist.unique();
|
||||
zlist.reverse();
|
||||
return zlist;
|
||||
}
|
||||
|
||||
void TileGenerator::writeImage(const std::string &output)
|
||||
{
|
||||
m_image->save(output);
|
||||
|
||||
@ -11,7 +11,6 @@ static inline int64_t stoi64(const std::string &s)
|
||||
return t;
|
||||
}
|
||||
|
||||
|
||||
static inline std::string i64tos(int64_t i)
|
||||
{
|
||||
std::ostringstream os;
|
||||
@ -19,6 +18,7 @@ static inline std::string i64tos(int64_t i)
|
||||
return os.str();
|
||||
}
|
||||
|
||||
|
||||
DBLevelDB::DBLevelDB(const std::string &mapdir)
|
||||
{
|
||||
leveldb::Options options;
|
||||
@ -28,6 +28,9 @@ DBLevelDB::DBLevelDB(const std::string &mapdir)
|
||||
throw std::runtime_error(std::string("Failed to open Database: ") + status.ToString());
|
||||
}
|
||||
|
||||
/* LevelDB is a dumb key-value store, so the only optimization we can do
|
||||
* is to cache the block positions that exist in the db.
|
||||
*/
|
||||
loadPosCache();
|
||||
}
|
||||
|
||||
@ -38,9 +41,21 @@ DBLevelDB::~DBLevelDB()
|
||||
}
|
||||
|
||||
|
||||
std::vector<BlockPos> DBLevelDB::getBlockPos()
|
||||
std::vector<BlockPos> DBLevelDB::getBlockPos(BlockPos min, BlockPos max)
|
||||
{
|
||||
return posCache;
|
||||
std::vector<BlockPos> res;
|
||||
for (const auto &it : posCache) {
|
||||
if (it.first < min.z || it.first >= max.z)
|
||||
continue;
|
||||
for (auto pos2 : it.second) {
|
||||
if (pos2.first < min.x || pos2.first >= max.x)
|
||||
continue;
|
||||
if (pos2.second < min.y || pos2.second >= max.y)
|
||||
continue;
|
||||
res.emplace_back(pos2.first, pos2.second, it.first);
|
||||
}
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
|
||||
@ -49,26 +64,35 @@ void DBLevelDB::loadPosCache()
|
||||
leveldb::Iterator * it = db->NewIterator(leveldb::ReadOptions());
|
||||
for (it->SeekToFirst(); it->Valid(); it->Next()) {
|
||||
int64_t posHash = stoi64(it->key().ToString());
|
||||
posCache.push_back(decodeBlockPos(posHash));
|
||||
BlockPos pos = decodeBlockPos(posHash);
|
||||
|
||||
posCache[pos.z].emplace_back(pos.x, pos.y);
|
||||
}
|
||||
delete it;
|
||||
}
|
||||
|
||||
|
||||
void DBLevelDB::getBlocksOnZ(std::map<int16_t, BlockList> &blocks, int16_t zPos)
|
||||
void DBLevelDB::getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
|
||||
int16_t min_y, int16_t max_y)
|
||||
{
|
||||
std::string datastr;
|
||||
leveldb::Status status;
|
||||
|
||||
for (const auto &it : posCache) {
|
||||
if (it.z != zPos) {
|
||||
auto it = posCache.find(z);
|
||||
if (it == posCache.cend())
|
||||
return;
|
||||
for (auto pos2 : it->second) {
|
||||
if (pos2.first != x)
|
||||
continue;
|
||||
}
|
||||
status = db->Get(leveldb::ReadOptions(), i64tos(encodeBlockPos(it)), &datastr);
|
||||
if (pos2.second < min_y || pos2.second >= max_y)
|
||||
continue;
|
||||
|
||||
BlockPos pos(x, pos2.second, z);
|
||||
status = db->Get(leveldb::ReadOptions(), i64tos(encodeBlockPos(pos)), &datastr);
|
||||
if (status.ok()) {
|
||||
Block b(it, ustring((const unsigned char *) datastr.data(), datastr.size()));
|
||||
blocks[b.first.x].push_back(b);
|
||||
blocks.emplace_back(
|
||||
pos, ustring((unsigned char *) datastr.data(), datastr.size())
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -27,11 +27,16 @@ DBPostgreSQL::DBPostgreSQL(const std::string &mapdir)
|
||||
|
||||
prepareStatement(
|
||||
"get_block_pos",
|
||||
"SELECT posX, posY, posZ FROM blocks"
|
||||
"SELECT posX::int4, posY::int4, posZ::int4 FROM blocks WHERE"
|
||||
" (posX BETWEEN $1::int4 AND $2::int4) AND"
|
||||
" (posY BETWEEN $3::int4 AND $4::int4) AND"
|
||||
" (posZ BETWEEN $5::int4 AND $6::int4)"
|
||||
);
|
||||
prepareStatement(
|
||||
"get_blocks_z",
|
||||
"SELECT posX, posY, data FROM blocks WHERE posZ = $1::int4"
|
||||
"get_blocks",
|
||||
"SELECT posY::int4, data FROM blocks WHERE"
|
||||
" posX = $1::int4 AND posZ = $2::int4"
|
||||
" AND (posY BETWEEN $3::int4 AND $4::int4)"
|
||||
);
|
||||
|
||||
checkResults(PQexec(db, "START TRANSACTION;"));
|
||||
@ -49,19 +54,31 @@ DBPostgreSQL::~DBPostgreSQL()
|
||||
PQfinish(db);
|
||||
}
|
||||
|
||||
std::vector<BlockPos> DBPostgreSQL::getBlockPos()
|
||||
std::vector<BlockPos> DBPostgreSQL::getBlockPos(BlockPos min, BlockPos max)
|
||||
{
|
||||
std::vector<BlockPos> positions;
|
||||
int32_t const x1 = htonl(min.x);
|
||||
int32_t const x2 = htonl(max.x - 1);
|
||||
int32_t const y1 = htonl(min.y);
|
||||
int32_t const y2 = htonl(max.y - 1);
|
||||
int32_t const z1 = htonl(min.z);
|
||||
int32_t const z2 = htonl(max.z - 1);
|
||||
|
||||
const void *args[] = { &x1, &x2, &y1, &y2, &z1, &z2 };
|
||||
const int argLen[] = { 4, 4, 4, 4, 4, 4 };
|
||||
const int argFmt[] = { 1, 1, 1, 1, 1, 1 };
|
||||
|
||||
PGresult *results = execPrepared(
|
||||
"get_block_pos", 0,
|
||||
NULL, NULL, NULL, false, false
|
||||
"get_block_pos", ARRLEN(args), args,
|
||||
argLen, argFmt, false
|
||||
);
|
||||
|
||||
int numrows = PQntuples(results);
|
||||
|
||||
std::vector<BlockPos> positions;
|
||||
positions.reserve(numrows);
|
||||
|
||||
for (int row = 0; row < numrows; ++row)
|
||||
positions.push_back(pg_to_blockpos(results, row, 0));
|
||||
positions.emplace_back(pg_to_blockpos(results, row, 0));
|
||||
|
||||
PQclear(results);
|
||||
|
||||
@ -69,16 +86,20 @@ std::vector<BlockPos> DBPostgreSQL::getBlockPos()
|
||||
}
|
||||
|
||||
|
||||
void DBPostgreSQL::getBlocksOnZ(std::map<int16_t, BlockList> &blocks, int16_t zPos)
|
||||
void DBPostgreSQL::getBlocksOnXZ(BlockList &blocks, int16_t xPos, int16_t zPos,
|
||||
int16_t min_y, int16_t max_y)
|
||||
{
|
||||
int32_t const x = htonl(xPos);
|
||||
int32_t const z = htonl(zPos);
|
||||
int32_t const y1 = htonl(min_y);
|
||||
int32_t const y2 = htonl(max_y - 1);
|
||||
|
||||
const void *args[] = { &z };
|
||||
const int argLen[] = { sizeof(z) };
|
||||
const int argFmt[] = { 1 };
|
||||
const void *args[] = { &x, &z, &y1, &y2 };
|
||||
const int argLen[] = { 4, 4, 4, 4 };
|
||||
const int argFmt[] = { 1, 1, 1, 1 };
|
||||
|
||||
PGresult *results = execPrepared(
|
||||
"get_blocks_z", ARRLEN(args), args,
|
||||
"get_blocks", ARRLEN(args), args,
|
||||
argLen, argFmt, false
|
||||
);
|
||||
|
||||
@ -86,19 +107,18 @@ void DBPostgreSQL::getBlocksOnZ(std::map<int16_t, BlockList> &blocks, int16_t zP
|
||||
|
||||
for (int row = 0; row < numrows; ++row) {
|
||||
BlockPos position;
|
||||
position.x = pg_binary_to_int(results, row, 0);
|
||||
position.y = pg_binary_to_int(results, row, 1);
|
||||
position.x = xPos;
|
||||
position.y = pg_binary_to_int(results, row, 0);
|
||||
position.z = zPos;
|
||||
Block const b(
|
||||
blocks.emplace_back(
|
||||
position,
|
||||
ustring(
|
||||
reinterpret_cast<unsigned char*>(
|
||||
PQgetvalue(results, row, 2)
|
||||
PQgetvalue(results, row, 1)
|
||||
),
|
||||
PQgetlength(results, row, 2)
|
||||
PQgetlength(results, row, 1)
|
||||
)
|
||||
);
|
||||
blocks[position.x].push_back(b);
|
||||
}
|
||||
|
||||
PQclear(results);
|
||||
@ -138,20 +158,15 @@ PGresult *DBPostgreSQL::execPrepared(
|
||||
const char *stmtName, const int paramsNumber,
|
||||
const void **params,
|
||||
const int *paramsLengths, const int *paramsFormats,
|
||||
bool clear, bool nobinary
|
||||
bool clear
|
||||
)
|
||||
{
|
||||
return checkResults(PQexecPrepared(db, stmtName, paramsNumber,
|
||||
(const char* const*) params, paramsLengths, paramsFormats,
|
||||
nobinary ? 1 : 0), clear
|
||||
1 /* binary output */), clear
|
||||
);
|
||||
}
|
||||
|
||||
int DBPostgreSQL::pg_to_int(PGresult *res, int row, int col)
|
||||
{
|
||||
return atoi(PQgetvalue(res, row, col));
|
||||
}
|
||||
|
||||
int DBPostgreSQL::pg_binary_to_int(PGresult *res, int row, int col)
|
||||
{
|
||||
int32_t* raw = reinterpret_cast<int32_t*>(PQgetvalue(res, row, col));
|
||||
@ -161,8 +176,8 @@ int DBPostgreSQL::pg_binary_to_int(PGresult *res, int row, int col)
|
||||
BlockPos DBPostgreSQL::pg_to_blockpos(PGresult *res, int row, int col)
|
||||
{
|
||||
BlockPos result;
|
||||
result.x = pg_to_int(res, row, col);
|
||||
result.y = pg_to_int(res, row, col + 1);
|
||||
result.z = pg_to_int(res, row, col + 2);
|
||||
result.x = pg_binary_to_int(res, row, col);
|
||||
result.y = pg_binary_to_int(res, row, col + 1);
|
||||
result.z = pg_binary_to_int(res, row, col + 2);
|
||||
return result;
|
||||
}
|
||||
|
||||
55
db-redis.cpp
55
db-redis.cpp
@ -51,6 +51,9 @@ DBRedis::DBRedis(const std::string &mapdir)
|
||||
throw std::runtime_error(err);
|
||||
}
|
||||
|
||||
/* Redis is just a key-value store, so the only optimization we can do
|
||||
* is to cache the block positions that exist in the db.
|
||||
*/
|
||||
loadPosCache();
|
||||
}
|
||||
|
||||
@ -61,9 +64,21 @@ DBRedis::~DBRedis()
|
||||
}
|
||||
|
||||
|
||||
std::vector<BlockPos> DBRedis::getBlockPos()
|
||||
std::vector<BlockPos> DBRedis::getBlockPos(BlockPos min, BlockPos max)
|
||||
{
|
||||
return posCache;
|
||||
std::vector<BlockPos> res;
|
||||
for (const auto &it : posCache) {
|
||||
if (it.first < min.z || it.first >= max.z)
|
||||
continue;
|
||||
for (auto pos2 : it.second) {
|
||||
if (pos2.first < min.x || pos2.first >= max.x)
|
||||
continue;
|
||||
if (pos2.second < min.y || pos2.second >= max.y)
|
||||
continue;
|
||||
res.emplace_back(pos2.first, pos2.second, it.first);
|
||||
}
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
|
||||
@ -98,7 +113,8 @@ void DBRedis::loadPosCache()
|
||||
for(size_t i = 0; i < reply->elements; i++) {
|
||||
if(reply->element[i]->type != REDIS_REPLY_STRING)
|
||||
REPLY_TYPE_ERR(reply->element[i], "HKEYS subreply");
|
||||
posCache.push_back(decodeBlockPos(stoi64(reply->element[i]->str)));
|
||||
BlockPos pos = decodeBlockPos(stoi64(reply->element[i]->str));
|
||||
posCache[pos.z].emplace_back(pos.x, pos.y);
|
||||
}
|
||||
|
||||
freeReplyObject(reply);
|
||||
@ -150,7 +166,7 @@ void DBRedis::HMGET(const std::vector<BlockPos> &positions, std::vector<ustring>
|
||||
freeReplyObject(reply);
|
||||
throw std::runtime_error("HMGET empty string");
|
||||
}
|
||||
result->push_back(ustring((const unsigned char *) subreply->str, subreply->len));
|
||||
result->emplace_back((const unsigned char *) subreply->str, subreply->len);
|
||||
}
|
||||
freeReplyObject(reply);
|
||||
remaining -= batch_size;
|
||||
@ -158,22 +174,23 @@ void DBRedis::HMGET(const std::vector<BlockPos> &positions, std::vector<ustring>
|
||||
}
|
||||
|
||||
|
||||
void DBRedis::getBlocksOnZ(std::map<int16_t, BlockList> &blocks, int16_t zPos)
|
||||
void DBRedis::getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
|
||||
int16_t min_y, int16_t max_y)
|
||||
{
|
||||
std::vector<BlockPos> z_positions;
|
||||
for (std::vector<BlockPos>::const_iterator it = posCache.begin(); it != posCache.end(); ++it) {
|
||||
if (it->z != zPos) {
|
||||
continue;
|
||||
}
|
||||
z_positions.push_back(*it);
|
||||
}
|
||||
std::vector<ustring> z_blocks;
|
||||
HMGET(z_positions, &z_blocks);
|
||||
auto it = posCache.find(z);
|
||||
if (it == posCache.cend())
|
||||
return;
|
||||
|
||||
std::vector<ustring>::const_iterator z_block = z_blocks.begin();
|
||||
for (std::vector<BlockPos>::const_iterator pos = z_positions.begin();
|
||||
pos != z_positions.end();
|
||||
++pos, ++z_block) {
|
||||
blocks[pos->x].push_back(Block(*pos, *z_block));
|
||||
std::vector<BlockPos> positions;
|
||||
for (auto pos2 : it->second) {
|
||||
if (pos2.first == x && pos2.second >= min_y && pos2.second < max_y)
|
||||
positions.emplace_back(x, pos2.second, z);
|
||||
}
|
||||
|
||||
std::vector<ustring> db_blocks;
|
||||
HMGET(positions, &db_blocks);
|
||||
|
||||
auto block = db_blocks.cbegin();
|
||||
for (auto pos = positions.cbegin(); pos != positions.cend(); ++pos, ++block)
|
||||
blocks.emplace_back(*pos, *block);
|
||||
}
|
||||
|
||||
121
db-sqlite3.cpp
121
db-sqlite3.cpp
@ -1,6 +1,8 @@
|
||||
#include <stdexcept>
|
||||
#include <unistd.h> // for usleep
|
||||
#include <iostream>
|
||||
#include <algorithm>
|
||||
#include <time.h>
|
||||
#include "db-sqlite3.h"
|
||||
#include "types.h"
|
||||
|
||||
@ -11,7 +13,6 @@
|
||||
}
|
||||
#define SQLOK(f) SQLRES(f, SQLITE_OK)
|
||||
|
||||
|
||||
DBSQLite3::DBSQLite3(const std::string &mapdir)
|
||||
{
|
||||
int result;
|
||||
@ -27,6 +28,10 @@ DBSQLite3::DBSQLite3(const std::string &mapdir)
|
||||
SQLOK(prepare_v2(db,
|
||||
"SELECT pos FROM blocks",
|
||||
-1, &stmt_get_block_pos, NULL))
|
||||
|
||||
SQLOK(prepare_v2(db,
|
||||
"SELECT pos FROM blocks WHERE pos BETWEEN ? AND ?",
|
||||
-1, &stmt_get_block_pos_z, NULL))
|
||||
}
|
||||
|
||||
|
||||
@ -34,59 +39,125 @@ DBSQLite3::~DBSQLite3()
|
||||
{
|
||||
sqlite3_finalize(stmt_get_blocks_z);
|
||||
sqlite3_finalize(stmt_get_block_pos);
|
||||
sqlite3_finalize(stmt_get_block_pos_z);
|
||||
|
||||
if (sqlite3_close(db) != SQLITE_OK) {
|
||||
std::cerr << "Error closing SQLite database." << std::endl;
|
||||
};
|
||||
}
|
||||
|
||||
std::vector<BlockPos> DBSQLite3::getBlockPos()
|
||||
|
||||
inline void DBSQLite3::getPosRange(int64_t &min, int64_t &max, int16_t zPos,
|
||||
int16_t zPos2) const
|
||||
{
|
||||
/* The range of block positions is [-2048, 2047], which turns into [0, 4095]
|
||||
* when casted to unsigned. This didn't actually help me understand the
|
||||
* numbers below, but I wanted to write it down.
|
||||
*/
|
||||
|
||||
// Magic numbers!
|
||||
min = encodeBlockPos(BlockPos(0, -2048, zPos));
|
||||
max = encodeBlockPos(BlockPos(0, 2048, zPos2)) - 1;
|
||||
}
|
||||
|
||||
|
||||
std::vector<BlockPos> DBSQLite3::getBlockPos(BlockPos min, BlockPos max)
|
||||
{
|
||||
int result;
|
||||
std::vector<BlockPos> positions;
|
||||
while ((result = sqlite3_step(stmt_get_block_pos)) != SQLITE_DONE) {
|
||||
if (result == SQLITE_ROW) {
|
||||
int64_t posHash = sqlite3_column_int64(stmt_get_block_pos, 0);
|
||||
positions.push_back(decodeBlockPos(posHash));
|
||||
} else if (result == SQLITE_BUSY) { // Wait some time and try again
|
||||
usleep(10000);
|
||||
sqlite3_stmt *stmt;
|
||||
|
||||
if(min.z <= -2048 && max.z >= 2048) {
|
||||
stmt = stmt_get_block_pos;
|
||||
} else {
|
||||
stmt = stmt_get_block_pos_z;
|
||||
int64_t minPos, maxPos;
|
||||
if (min.z < -2048)
|
||||
min.z = -2048;
|
||||
if (max.z > 2048)
|
||||
max.z = 2048;
|
||||
getPosRange(minPos, maxPos, min.z, max.z - 1);
|
||||
SQLOK(bind_int64(stmt, 1, minPos))
|
||||
SQLOK(bind_int64(stmt, 2, maxPos))
|
||||
}
|
||||
|
||||
std::vector<BlockPos> positions;
|
||||
while ((result = sqlite3_step(stmt)) != SQLITE_DONE) {
|
||||
if (result == SQLITE_BUSY) { // Wait some time and try again
|
||||
usleep(10000);
|
||||
} else if (result != SQLITE_ROW) {
|
||||
throw std::runtime_error(sqlite3_errmsg(db));
|
||||
}
|
||||
|
||||
int64_t posHash = sqlite3_column_int64(stmt, 0);
|
||||
BlockPos pos = decodeBlockPos(posHash);
|
||||
if(pos.x >= min.x && pos.x < max.x && pos.y >= min.y && pos.y < max.y)
|
||||
positions.emplace_back(pos);
|
||||
}
|
||||
SQLOK(reset(stmt_get_block_pos));
|
||||
SQLOK(reset(stmt));
|
||||
return positions;
|
||||
}
|
||||
|
||||
|
||||
void DBSQLite3::getBlocksOnZ(std::map<int16_t, BlockList> &blocks, int16_t zPos)
|
||||
void DBSQLite3::loadBlockCache(int16_t zPos)
|
||||
{
|
||||
int result;
|
||||
blockCache.clear();
|
||||
|
||||
// Magic numbers!
|
||||
int64_t minPos = encodeBlockPos(BlockPos(0, -2048, zPos));
|
||||
int64_t maxPos = encodeBlockPos(BlockPos(0, 2048, zPos)) - 1;
|
||||
int64_t minPos, maxPos;
|
||||
getPosRange(minPos, maxPos, zPos, zPos);
|
||||
|
||||
SQLOK(bind_int64(stmt_get_blocks_z, 1, minPos));
|
||||
SQLOK(bind_int64(stmt_get_blocks_z, 2, maxPos));
|
||||
|
||||
while ((result = sqlite3_step(stmt_get_blocks_z)) != SQLITE_DONE) {
|
||||
if (result == SQLITE_ROW) {
|
||||
if (result == SQLITE_BUSY) { // Wait some time and try again
|
||||
usleep(10000);
|
||||
} else if (result != SQLITE_ROW) {
|
||||
throw std::runtime_error(sqlite3_errmsg(db));
|
||||
}
|
||||
|
||||
int64_t posHash = sqlite3_column_int64(stmt_get_blocks_z, 0);
|
||||
BlockPos pos = decodeBlockPos(posHash);
|
||||
const unsigned char *data = reinterpret_cast<const unsigned char *>(
|
||||
sqlite3_column_blob(stmt_get_blocks_z, 1));
|
||||
size_t size = sqlite3_column_bytes(stmt_get_blocks_z, 1);
|
||||
Block b(decodeBlockPos(posHash), ustring(data, size));
|
||||
blocks[b.first.x].push_back(b);
|
||||
} else if (result == SQLITE_BUSY) { // Wait some time and try again
|
||||
usleep(10000);
|
||||
} else {
|
||||
throw std::runtime_error(sqlite3_errmsg(db));
|
||||
blockCache[pos.x].emplace_back(pos, ustring(data, size));
|
||||
}
|
||||
}
|
||||
SQLOK(reset(stmt_get_blocks_z));
|
||||
SQLOK(reset(stmt_get_blocks_z))
|
||||
}
|
||||
|
||||
#undef SQLRES
|
||||
#undef SQLOK
|
||||
|
||||
void DBSQLite3::getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
|
||||
int16_t min_y, int16_t max_y)
|
||||
{
|
||||
/* Cache the blocks on the given Z coordinate between calls, this only
|
||||
* works due to order in which the TileGenerator asks for blocks. */
|
||||
if (z != blockCachedZ) {
|
||||
loadBlockCache(z);
|
||||
blockCachedZ = z;
|
||||
}
|
||||
|
||||
auto it = blockCache.find(x);
|
||||
if (it == blockCache.end())
|
||||
return;
|
||||
|
||||
if (it->second.empty()) {
|
||||
/* We have swapped this list before, this is not supposed to happen
|
||||
* because it's bad for performance. But rather than silently breaking
|
||||
* do the right thing and load the blocks again. */
|
||||
#ifndef NDEBUG
|
||||
std::cout << "Warning: suboptimal access pattern for sqlite3 backend" << std::endl;
|
||||
#endif
|
||||
loadBlockCache(z);
|
||||
}
|
||||
// Swap lists to avoid copying contents
|
||||
blocks.clear();
|
||||
std::swap(blocks, it->second);
|
||||
|
||||
for (auto it = blocks.begin(); it != blocks.end(); ) {
|
||||
if (it->first.y < min_y || it->first.y >= max_y)
|
||||
it = blocks.erase(it);
|
||||
else
|
||||
it++;
|
||||
}
|
||||
}
|
||||
|
||||
@ -2,7 +2,8 @@
|
||||
#define TILEGENERATOR_HEADER
|
||||
|
||||
#include <iosfwd>
|
||||
#include <list>
|
||||
#include <map>
|
||||
#include <set>
|
||||
#include <config.h>
|
||||
#include <unordered_map>
|
||||
#include <unordered_set>
|
||||
@ -64,7 +65,8 @@ public:
|
||||
void setBgColor(const std::string &bgColor);
|
||||
void setScaleColor(const std::string &scaleColor);
|
||||
void setOriginColor(const std::string &originColor);
|
||||
void setPlayerColor(const std::string &playerColor); Color parseColor(const std::string &color);
|
||||
void setPlayerColor(const std::string &playerColor);
|
||||
Color parseColor(const std::string &color);
|
||||
void setDrawOrigin(bool drawOrigin);
|
||||
void setDrawPlayers(bool drawPlayers);
|
||||
void setDrawScale(bool drawScale);
|
||||
@ -88,7 +90,6 @@ private:
|
||||
void loadBlocks();
|
||||
void createImage();
|
||||
void renderMap();
|
||||
std::list<int16_t> getZValueList() const;
|
||||
void renderMapBlock(const BlockDecoder &blk, const BlockPos &pos);
|
||||
void renderMapBlockBottom(const BlockPos &pos);
|
||||
void renderShading(int zPos);
|
||||
@ -118,19 +119,23 @@ private:
|
||||
DB *m_db;
|
||||
Image *m_image;
|
||||
PixelAttributes m_blockPixelAttributes;
|
||||
/* smallest/largest seen X or Z block coordinate */
|
||||
int m_xMin;
|
||||
int m_xMax;
|
||||
int m_zMin;
|
||||
int m_zMax;
|
||||
/* Y limits for rendered area (node units) */
|
||||
int m_yMin;
|
||||
int m_yMax;
|
||||
int m_geomX;
|
||||
int m_geomY;
|
||||
int m_geomX2;
|
||||
int m_geomY2;
|
||||
/* limits for rendered area (block units) */
|
||||
int16_t m_geomX;
|
||||
int16_t m_geomY; /* Y in terms of rendered image, Z in the world */
|
||||
int16_t m_geomX2;
|
||||
int16_t m_geomY2;
|
||||
/* */
|
||||
int m_mapWidth;
|
||||
int m_mapHeight;
|
||||
std::list<std::pair<int16_t, int16_t>> m_positions;
|
||||
std::map<int16_t, std::set<int16_t>> m_positions; /* indexed by Z, contains X coords */
|
||||
ColorMap m_colorMap;
|
||||
BitmapThing m_readPixels;
|
||||
BitmapThing m_readInfo;
|
||||
|
||||
@ -2,19 +2,25 @@
|
||||
#define DB_LEVELDB_HEADER
|
||||
|
||||
#include "db.h"
|
||||
#include <unordered_map>
|
||||
#include <utility>
|
||||
#include <leveldb/db.h>
|
||||
|
||||
class DBLevelDB : public DB {
|
||||
public:
|
||||
DBLevelDB(const std::string &mapdir);
|
||||
virtual std::vector<BlockPos> getBlockPos();
|
||||
virtual void getBlocksOnZ(std::map<int16_t, BlockList> &blocks, int16_t zPos);
|
||||
virtual ~DBLevelDB();
|
||||
std::vector<BlockPos> getBlockPos(BlockPos min, BlockPos max) override;
|
||||
void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
|
||||
int16_t min_y, int16_t max_y) override;
|
||||
~DBLevelDB() override;
|
||||
|
||||
private:
|
||||
using pos2d = std::pair<int16_t, int16_t>;
|
||||
|
||||
void loadPosCache();
|
||||
|
||||
std::vector<BlockPos> posCache;
|
||||
|
||||
// indexed by Z, contains all (x,y) position pairs
|
||||
std::unordered_map<int16_t, std::vector<pos2d>> posCache;
|
||||
leveldb::DB *db;
|
||||
};
|
||||
|
||||
|
||||
@ -7,9 +7,11 @@
|
||||
class DBPostgreSQL : public DB {
|
||||
public:
|
||||
DBPostgreSQL(const std::string &mapdir);
|
||||
virtual std::vector<BlockPos> getBlockPos();
|
||||
virtual void getBlocksOnZ(std::map<int16_t, BlockList> &blocks, int16_t zPos);
|
||||
virtual ~DBPostgreSQL();
|
||||
std::vector<BlockPos> getBlockPos(BlockPos min, BlockPos max) override;
|
||||
void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
|
||||
int16_t min_y, int16_t max_y) override;
|
||||
~DBPostgreSQL() override;
|
||||
|
||||
protected:
|
||||
PGresult *checkResults(PGresult *res, bool clear = true);
|
||||
void prepareStatement(const std::string &name, const std::string &sql);
|
||||
@ -17,11 +19,11 @@ protected:
|
||||
const char *stmtName, const int paramsNumber,
|
||||
const void **params,
|
||||
const int *paramsLengths = NULL, const int *paramsFormats = NULL,
|
||||
bool clear = true, bool nobinary = true
|
||||
bool clear = true
|
||||
);
|
||||
int pg_to_int(PGresult *res, int row, int col);
|
||||
int pg_binary_to_int(PGresult *res, int row, int col);
|
||||
BlockPos pg_to_blockpos(PGresult *res, int row, int col);
|
||||
|
||||
private:
|
||||
PGconn *db;
|
||||
};
|
||||
|
||||
@ -2,21 +2,27 @@
|
||||
#define DB_REDIS_HEADER
|
||||
|
||||
#include "db.h"
|
||||
#include <unordered_map>
|
||||
#include <utility>
|
||||
#include <hiredis/hiredis.h>
|
||||
|
||||
class DBRedis : public DB {
|
||||
public:
|
||||
DBRedis(const std::string &mapdir);
|
||||
virtual std::vector<BlockPos> getBlockPos();
|
||||
virtual void getBlocksOnZ(std::map<int16_t, BlockList> &blocks, int16_t zPos);
|
||||
virtual ~DBRedis();
|
||||
std::vector<BlockPos> getBlockPos(BlockPos min, BlockPos max) override;
|
||||
void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
|
||||
int16_t min_y, int16_t max_y) override;
|
||||
~DBRedis() override;
|
||||
|
||||
private:
|
||||
using pos2d = std::pair<int16_t, int16_t>;
|
||||
static std::string replyTypeStr(int type);
|
||||
|
||||
void loadPosCache();
|
||||
void HMGET(const std::vector<BlockPos> &positions, std::vector<ustring> *result);
|
||||
|
||||
std::vector<BlockPos> posCache;
|
||||
// indexed by Z, contains all (x,y) position pairs
|
||||
std::unordered_map<int16_t, std::vector<pos2d>> posCache;
|
||||
|
||||
redisContext *ctx;
|
||||
std::string hash;
|
||||
|
||||
@ -2,19 +2,30 @@
|
||||
#define _DB_SQLITE3_H
|
||||
|
||||
#include "db.h"
|
||||
#include <unordered_map>
|
||||
#include <sqlite3.h>
|
||||
|
||||
class DBSQLite3 : public DB {
|
||||
public:
|
||||
DBSQLite3(const std::string &mapdir);
|
||||
virtual std::vector<BlockPos> getBlockPos();
|
||||
virtual void getBlocksOnZ(std::map<int16_t, BlockList> &blocks, int16_t zPos);
|
||||
virtual ~DBSQLite3();
|
||||
std::vector<BlockPos> getBlockPos(BlockPos min, BlockPos max) override;
|
||||
void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
|
||||
int16_t min_y, int16_t max_y) override;
|
||||
~DBSQLite3() override;
|
||||
|
||||
private:
|
||||
inline void getPosRange(int64_t &min, int64_t &max, int16_t zPos,
|
||||
int16_t zPos2) const;
|
||||
void loadBlockCache(int16_t zPos);
|
||||
|
||||
sqlite3 *db;
|
||||
|
||||
sqlite3_stmt *stmt_get_block_pos;
|
||||
sqlite3_stmt *stmt_get_block_pos_z;
|
||||
sqlite3_stmt *stmt_get_blocks_z;
|
||||
|
||||
int16_t blockCachedZ = -10000;
|
||||
std::unordered_map<int16_t, BlockList> blockCache; // indexed by X
|
||||
};
|
||||
|
||||
#endif // _DB_SQLITE3_H
|
||||
|
||||
17
include/db.h
17
include/db.h
@ -5,7 +5,6 @@
|
||||
#include <map>
|
||||
#include <list>
|
||||
#include <vector>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include "types.h"
|
||||
|
||||
@ -17,6 +16,8 @@ struct BlockPos {
|
||||
|
||||
BlockPos() : x(0), y(0), z(0) {}
|
||||
BlockPos(int16_t x, int16_t y, int16_t z) : x(x), y(y), z(z) {}
|
||||
|
||||
// Implements the inverse ordering so that (2,2,2) < (1,1,1)
|
||||
bool operator < (const BlockPos &p) const
|
||||
{
|
||||
if (z > p.z)
|
||||
@ -42,13 +43,21 @@ typedef std::list<Block> BlockList;
|
||||
|
||||
class DB {
|
||||
protected:
|
||||
// Helpers that implement the hashed positions used by most backends
|
||||
inline int64_t encodeBlockPos(const BlockPos pos) const;
|
||||
inline BlockPos decodeBlockPos(int64_t hash) const;
|
||||
|
||||
public:
|
||||
virtual std::vector<BlockPos> getBlockPos() = 0;
|
||||
virtual void getBlocksOnZ(std::map<int16_t, BlockList> &blocks, int16_t zPos) = 0;
|
||||
virtual ~DB() {};
|
||||
/* Return all block positions inside the range given by min and max,
|
||||
* so that min.x <= x < max.x, ...
|
||||
*/
|
||||
virtual std::vector<BlockPos> getBlockPos(BlockPos min, BlockPos max) = 0;
|
||||
/* Return all blocks in column given by x and z
|
||||
* and inside the given Y range (min_y <= y < max_y)
|
||||
*/
|
||||
virtual void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
|
||||
int16_t min_y, int16_t max_y) = 0;
|
||||
virtual ~DB() {}
|
||||
};
|
||||
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user