From 22d3e401f84daac0979c60b0744a66e0bd323131 Mon Sep 17 00:00:00 2001 From: ShadowNinja Date: Sat, 19 Apr 2014 02:13:34 -0400 Subject: [PATCH] Don't use a temporary table for block positions This also moves database functions and data structures to db.h --- TileGenerator.cpp | 70 ++++----------------------- TileGenerator.h | 54 +++------------------ ZlibDecompressor.cpp | 6 +-- ZlibDecompressor.h | 4 +- db-leveldb.cpp | 70 +++++++++++++-------------- db-leveldb.h | 15 +++--- db-sqlite3.cpp | 66 ++++++++++++++------------ db-sqlite3.h | 6 +-- db.h | 109 +++++++++++++++++++++++++++++++++++++++---- types.h | 5 ++ 10 files changed, 206 insertions(+), 199 deletions(-) create mode 100644 types.h diff --git a/TileGenerator.cpp b/TileGenerator.cpp index a8d0b6a..d346a81 100644 --- a/TileGenerator.cpp +++ b/TileGenerator.cpp @@ -1,12 +1,3 @@ -/* - * ===================================================================== - * Version: 1.0 - * Created: 23.08.2012 12:35:53 - * Author: Miroslav Bendík - * Company: LinuxOS.sk - * ===================================================================== - */ - #include #include #include @@ -29,26 +20,6 @@ using namespace std; -static inline int64_t pythonmodulo(int64_t i, int64_t mod) -{ - if (i >= 0) { - return i % mod; - } - else { - return mod - ((-i) % mod); - } -} - -static inline int unsignedToSigned(long i, long max_positive) -{ - if (i < max_positive) { - return i; - } - else { - return i - 2l * max_positive; - } -} - static inline uint16_t readU16(const unsigned char *data) { return data[0] << 8 | data[1]; @@ -334,9 +305,9 @@ void TileGenerator::openDb(const std::string &input) void TileGenerator::loadBlocks() { - std::vector vec = m_db->getBlockPos(); - for(std::vector::iterator it = vec.begin(); it != vec.end(); ++it) { - BlockPos pos = decodeBlockPos(*it); + std::vector vec = m_db->getBlockPos(); + for (std::vector::iterator it = vec.begin(); it != vec.end(); ++it) { + BlockPos pos = *it; // Check that it's in geometry (from --geometry option) if (pos.x < m_geomX || pos.x > m_geomX2 || pos.z < m_geomY || pos.z > m_geomY2) { continue; @@ -364,17 +335,6 @@ void TileGenerator::loadBlocks() m_positions.unique(); } -inline BlockPos TileGenerator::decodeBlockPos(int64_t blockId) const -{ - BlockPos pos; - pos.x = unsignedToSigned(pythonmodulo(blockId, 4096), 2048); - blockId = (blockId - pos.x) / 4096; - pos.y = unsignedToSigned(pythonmodulo(blockId, 4096), 2048); - blockId = (blockId - pos.y) / 4096; - pos.z = unsignedToSigned(pythonmodulo(blockId, 4096), 2048); - return pos; -} - void TileGenerator::createImage() { m_mapWidth = (m_xMax - m_xMin + 1) * 16; @@ -385,27 +345,13 @@ void TileGenerator::createImage() gdImageFilledRectangle(m_image, 0, 0, m_mapWidth + m_border - 1, m_mapHeight + m_border -1, color2int(m_bgColor)); } -std::map TileGenerator::getBlocksOnZ(int zPos) -{ - DBBlockList in = m_db->getBlocksOnZ(zPos); - std::map out; - for(DBBlockList::const_iterator it = in.begin(); it != in.end(); ++it) { - Block b = Block(decodeBlockPos(it->first), it->second); - if(out.find(b.first.x) == out.end()) { - BlockList bl; - out[b.first.x] = bl; - } - out[b.first.x].push_back(b); - } - return out; -} - void TileGenerator::renderMap() { std::list zlist = getZValueList(); for (std::list::iterator zPosition = zlist.begin(); zPosition != zlist.end(); ++zPosition) { int zPos = *zPosition; - std::map blocks = getBlocksOnZ(zPos); + std::map blocks; + m_db->getBlocksOnZ(blocks, zPos); for (std::list >::const_iterator position = m_positions.begin(); position != m_positions.end(); ++position) { if (position->second != zPos) { continue; @@ -436,8 +382,8 @@ void TileGenerator::renderMap() ZlibDecompressor decompressor(data, length); decompressor.setSeekPos(dataOffset); - ZlibDecompressor::string mapData = decompressor.decompress(); - ZlibDecompressor::string mapMetadata = decompressor.decompress(); + ustring mapData = decompressor.decompress(); + ustring mapMetadata = decompressor.decompress(); dataOffset = decompressor.seekPos(); // Skip unused data @@ -519,7 +465,7 @@ void TileGenerator::renderMap() } } -inline void TileGenerator::renderMapBlock(const unsigned_string &mapBlock, const BlockPos &pos, int version) +inline void TileGenerator::renderMapBlock(const ustring &mapBlock, const BlockPos &pos, int version) { int xBegin = (pos.x - m_xMin) * 16; int zBegin = (m_zMax - pos.z) * 16; diff --git a/TileGenerator.h b/TileGenerator.h index 1cea693..04a5947 100644 --- a/TileGenerator.h +++ b/TileGenerator.h @@ -1,14 +1,5 @@ -/* - * ===================================================================== - * Version: 1.0 - * Created: 23.08.2012 12:35:59 - * Author: Miroslav Bendík - * Company: LinuxOS.sk - * ===================================================================== - */ - -#ifndef TILEGENERATOR_H_JJNUCARH -#define TILEGENERATOR_H_JJNUCARH +#ifndef TILEGENERATOR_HEADER +#define TILEGENERATOR_HEADER #include #include @@ -19,6 +10,7 @@ #include #include "PixelAttributes.h" #include "db.h" +#include "types.h" struct Color { Color(): r(0xFF), g(0xFF), b(0xFF), a(0) {}; @@ -42,42 +34,10 @@ struct ColorEntry { }; -struct BlockPos { - int x; - int y; - int z; - bool operator<(const BlockPos& p) const - { - if (z > p.z) { - return true; - } - if (z < p.z) { - return false; - } - if (y > p.y) { - return true; - } - if (y < p.y) { - return false; - } - if (x > p.x) { - return true; - } - if (x < p.x) { - return false; - } - return false; - } -}; - - class TileGenerator { private: - typedef std::basic_string unsigned_string; typedef std::map ColorMap; - typedef std::pair Block; - typedef std::list BlockList; public: TileGenerator(); @@ -102,12 +62,10 @@ private: void parseColorsStream(std::istream &in); void openDb(const std::string &input); void loadBlocks(); - BlockPos decodeBlockPos(int64_t blockId) const; void createImage(); void renderMap(); std::list getZValueList() const; - std::map getBlocksOnZ(int zPos); - void renderMapBlock(const unsigned_string &mapBlock, const BlockPos &pos, int version); + void renderMapBlock(const ustring &mapBlock, const BlockPos &pos, int version); void renderShading(int zPos); void renderScale(); void renderOrigin(); @@ -153,7 +111,7 @@ private: int m_blockAirId; int m_blockIgnoreId; -}; /* ----- end of class TileGenerator ----- */ +}; // class TileGenerator -#endif /* end of include guard: TILEGENERATOR_H_JJNUCARH */ +#endif // TILEGENERATOR_HEADER diff --git a/ZlibDecompressor.cpp b/ZlibDecompressor.cpp index b676f2a..69ee3f6 100644 --- a/ZlibDecompressor.cpp +++ b/ZlibDecompressor.cpp @@ -32,12 +32,12 @@ std::size_t ZlibDecompressor::seekPos() const return m_seekPos; } -ZlibDecompressor::string ZlibDecompressor::decompress() +ustring ZlibDecompressor::decompress() { const unsigned char *data = m_data + m_seekPos; const std::size_t size = m_size - m_seekPos; - string buffer; + ustring buffer; const size_t BUFSIZE = 128 * 1024; uint8_t temp_buffer[BUFSIZE]; @@ -58,7 +58,7 @@ ZlibDecompressor::string ZlibDecompressor::decompress() strm.avail_out = BUFSIZE; strm.next_out = temp_buffer; ret = inflate(&strm, Z_NO_FLUSH); - buffer += string(reinterpret_cast(temp_buffer), BUFSIZE - strm.avail_out); + buffer += ustring(reinterpret_cast(temp_buffer), BUFSIZE - strm.avail_out); } while (ret == Z_OK); if (ret != Z_STREAM_END) { throw DecompressError(); diff --git a/ZlibDecompressor.h b/ZlibDecompressor.h index 5b842bd..677328e 100644 --- a/ZlibDecompressor.h +++ b/ZlibDecompressor.h @@ -12,12 +12,12 @@ #include #include +#include "types.h" class ZlibDecompressor { public: - typedef std::basic_string string; class DecompressError { }; @@ -25,7 +25,7 @@ public: ~ZlibDecompressor(); void setSeekPos(std::size_t seekPos); std::size_t seekPos() const; - string decompress(); + ustring decompress(); private: const unsigned char *m_data; diff --git a/db-leveldb.cpp b/db-leveldb.cpp index 18d105c..ce2cfc6 100644 --- a/db-leveldb.cpp +++ b/db-leveldb.cpp @@ -1,74 +1,74 @@ -#include "db-leveldb.h" #include #include +#include "db-leveldb.h" +#include "types.h" -inline int64_t stoi64(const std::string &s) { +static inline int64_t stoi64(const std::string &s) +{ std::stringstream tmp(s); - long long t; + int64_t t; tmp >> t; return t; } -inline std::string i64tos(int64_t i) { - std::ostringstream o; - o< DBLevelDB::getBlockPos() { - loadPosCache(); + +std::vector DBLevelDB::getBlockPos() +{ return posCache; } -void DBLevelDB::loadPosCache() { - if (posCacheLoaded) { - return; - } +void DBLevelDB::loadPosCache() +{ leveldb::Iterator * it = db->NewIterator(leveldb::ReadOptions()); for (it->SeekToFirst(); it->Valid(); it->Next()) { - posCache.push_back(stoi64(it->key().ToString())); + int64_t posHash = stoi64(it->key().ToString()); + posCache.push_back(decodeBlockPos(posHash)); } delete it; - posCacheLoaded = true; } -DBBlockList DBLevelDB::getBlocksOnZ(int zPos) { - DBBlockList blocks; + +void DBLevelDB::getBlocksOnZ(std::map &blocks, int16_t zPos) +{ std::string datastr; leveldb::Status status; - int64_t psMin = (zPos * 16777216L) - 0x800000; - int64_t psMax = (zPos * 16777216L) + 0x7fffff; - - for (std::vector::iterator it = posCache.begin(); it != posCache.end(); ++it) { - int64_t i = *it; - if (i < psMin || i > psMax) { + for (std::vector::iterator it = posCache.begin(); it != posCache.end(); ++it) { + if (it->z != zPos) { continue; } - status = db->Get(leveldb::ReadOptions(), i64tos(i), &datastr); + status = db->Get(leveldb::ReadOptions(), i64tos(encodeBlockPos(*it)), &datastr); if (status.ok()) { - blocks.push_back( - DBBlock(i, - std::basic_string((const unsigned char*) datastr.data(), datastr.size()) - ) - ); + Block b(*it, ustring((const unsigned char *) datastr.data(), datastr.size())); + blocks[b.first.x].push_back(b); } } - - return blocks; } diff --git a/db-leveldb.h b/db-leveldb.h index bf0002f..8128ee9 100644 --- a/db-leveldb.h +++ b/db-leveldb.h @@ -1,5 +1,5 @@ -#ifndef _DB_LEVELDB_H -#define _DB_LEVELDB_H +#ifndef DB_LEVELDB_HEADER +#define DB_LEVELDB_HEADER #include "db.h" #include @@ -7,16 +7,15 @@ class DBLevelDB : public DB { public: DBLevelDB(const std::string &mapdir); - virtual std::vector getBlockPos(); - virtual DBBlockList getBlocksOnZ(int zPos); + virtual std::vector getBlockPos(); + virtual void getBlocksOnZ(std::map &blocks, int16_t zPos); ~DBLevelDB(); private: void loadPosCache(); - leveldb::DB *db; + std::vector posCache; - bool posCacheLoaded; - std::vector posCache; + leveldb::DB *db; }; -#endif // _DB_LEVELDB_H +#endif // DB_LEVELDB_HEADER diff --git a/db-sqlite3.cpp b/db-sqlite3.cpp index 966f450..f75a998 100644 --- a/db-sqlite3.cpp +++ b/db-sqlite3.cpp @@ -1,6 +1,7 @@ -#include "db-sqlite3.h" #include #include // for usleep +#include "db-sqlite3.h" +#include "types.h" #define SQLRES(f, good) \ result = (sqlite3_##f);\ @@ -9,36 +10,42 @@ } #define SQLOK(f) SQLRES(f, SQLITE_OK) -DBSQLite3::DBSQLite3(const std::string &mapdir) { + +DBSQLite3::DBSQLite3(const std::string &mapdir) +{ int result; std::string db_name = mapdir + "map.sqlite"; - SQLOK(open_v2(db_name.c_str(), &db, SQLITE_OPEN_READONLY | SQLITE_OPEN_PRIVATECACHE, 0)) + SQLOK(open_v2(db_name.c_str(), &db, SQLITE_OPEN_READONLY | + SQLITE_OPEN_PRIVATECACHE, 0)) SQLOK(prepare_v2(db, - "SELECT pos, data FROM blocks WHERE (pos >= ? AND pos <= ?)", - -1, &stmt_get_blocks, NULL)) + "SELECT pos, data FROM blocks WHERE pos BETWEEN ? AND ?", + -1, &stmt_get_blocks_z, NULL)) SQLOK(prepare_v2(db, "SELECT pos FROM blocks", -1, &stmt_get_block_pos, NULL)) } -DBSQLite3::~DBSQLite3() { + +DBSQLite3::~DBSQLite3() +{ int result; - SQLOK(finalize(stmt_get_blocks)); + SQLOK(finalize(stmt_get_blocks_z)); SQLOK(finalize(stmt_get_block_pos)); SQLOK(close(db)); } -std::vector DBSQLite3::getBlockPos() { - std::vector vec; +std::vector DBSQLite3::getBlockPos() +{ int result; + std::vector positions; while ((result = sqlite3_step(stmt_get_block_pos)) != SQLITE_DONE) { if (result == SQLITE_ROW) { - int64_t blockpos = sqlite3_column_int64(stmt_get_block_pos, 0); - vec.push_back(blockpos); + int64_t posHash = sqlite3_column_int64(stmt_get_block_pos, 0); + positions.push_back(decodeBlockPos(posHash)); } else if (result == SQLITE_BUSY) { // Wait some time and try again usleep(10000); } else { @@ -46,35 +53,36 @@ std::vector DBSQLite3::getBlockPos() { } } SQLOK(reset(stmt_get_block_pos)); - return vec; + return positions; } -DBBlockList DBSQLite3::getBlocksOnZ(int zPos) + +void DBSQLite3::getBlocksOnZ(std::map &blocks, int16_t zPos) { - DBBlockList blocks; - - int64_t psMin = (static_cast(zPos) * 16777216L) - 0x800000; - int64_t psMax = (static_cast(zPos) * 16777216L) + 0x7fffff; - - sqlite3_bind_int64(stmt_get_blocks, 1, psMin); - sqlite3_bind_int64(stmt_get_blocks, 2, psMax); - int result; - while ((result = sqlite3_step(stmt_get_blocks)) != SQLITE_DONE) { + + // Magic numbers! + int64_t minPos = (zPos * 0x1000000) - 0x800000; + int64_t maxPos = (zPos * 0x1000000) + 0x7FFFFF; + + SQLOK(bind_int64(stmt_get_blocks_z, 1, minPos)); + SQLOK(bind_int64(stmt_get_blocks_z, 2, maxPos)); + + while ((result = sqlite3_step(stmt_get_blocks_z)) != SQLITE_DONE) { if (result == SQLITE_ROW) { - int64_t blocknum = sqlite3_column_int64(stmt_get_blocks, 0); - const unsigned char *data = reinterpret_cast(sqlite3_column_blob(stmt_get_blocks, 1)); - int size = sqlite3_column_bytes(stmt_get_blocks, 1); - blocks.push_back(DBBlock(blocknum, std::basic_string(data, size))); + int64_t posHash = sqlite3_column_int64(stmt_get_blocks_z, 0); + const unsigned char *data = reinterpret_cast( + sqlite3_column_blob(stmt_get_blocks_z, 1)); + size_t size = sqlite3_column_bytes(stmt_get_blocks_z, 1); + Block b(decodeBlockPos(posHash), ustring(data, size)); + blocks[b.first.x].push_back(b); } else if (result == SQLITE_BUSY) { // Wait some time and try again usleep(10000); } else { throw std::runtime_error(sqlite3_errmsg(db)); } } - SQLOK(reset(stmt_get_blocks)); - - return blocks; + SQLOK(reset(stmt_get_blocks_z)); } #undef SQLRES diff --git a/db-sqlite3.h b/db-sqlite3.h index 38b447a..013035b 100644 --- a/db-sqlite3.h +++ b/db-sqlite3.h @@ -7,14 +7,14 @@ class DBSQLite3 : public DB { public: DBSQLite3(const std::string &mapdir); - virtual std::vector getBlockPos(); - virtual DBBlockList getBlocksOnZ(int zPos); + virtual std::vector getBlockPos(); + virtual void getBlocksOnZ(std::map &blocks, int16_t zPos); ~DBSQLite3(); private: sqlite3 *db; sqlite3_stmt *stmt_get_block_pos; - sqlite3_stmt *stmt_get_blocks; + sqlite3_stmt *stmt_get_blocks_z; }; #endif // _DB_SQLITE3_H diff --git a/db.h b/db.h index 329e409..40770b0 100644 --- a/db.h +++ b/db.h @@ -1,19 +1,110 @@ -#ifndef _DB_H -#define _DB_H +#ifndef DB_HEADER +#define DB_HEADER #include -#include +#include #include +#include #include #include +#include "types.h" -typedef std::pair > DBBlock; -typedef std::list DBBlockList; -class DB { +class BlockPos { public: - virtual std::vector getBlockPos() = 0; - virtual DBBlockList getBlocksOnZ(int zPos) = 0; + int16_t x; + int16_t y; + int16_t z; + + bool operator < (const BlockPos &p) const + { + if (z > p.z) { + return true; + } + if (z < p.z) { + return false; + } + if (y > p.y) { + return true; + } + if (y < p.y) { + return false; + } + if (x > p.x) { + return true; + } + if (x < p.x) { + return false; + } + return false; + } }; -#endif // _DB_H + +typedef std::pair Block; +typedef std::list BlockList; + + +class DB { +protected: + inline int64_t encodeBlockPos(const BlockPos pos) const; + inline BlockPos decodeBlockPos(int64_t hash) const; + +public: + virtual std::vector getBlockPos() = 0; + virtual void getBlocksOnZ(std::map &blocks, int16_t zPos) = 0; +}; + + + +/**************** + * Black magic! * + **************** + * The position hashing is seriously messed up, + * and is a lot more complicated than it looks. + */ + +static inline int16_t unsigned_to_signed(uint16_t i, uint16_t max_positive) +{ + if (i < max_positive) { + return i; + } else { + return i - (max_positive * 2); + } +} + + +// Modulo of a negative number does not work consistently in C +static inline int64_t pythonmodulo(int64_t i, int64_t mod) +{ + if (i >= 0) { + return i % mod; + } + return mod - ((-i) % mod); +} + + +inline int64_t DB::encodeBlockPos(const BlockPos pos) const +{ + return (uint64_t) pos.z * 0x1000000 + + (uint64_t) pos.y * 0x1000 + + (uint64_t) pos.x; +} + + +inline BlockPos DB::decodeBlockPos(int64_t hash) const +{ + BlockPos pos; + pos.x = unsigned_to_signed(pythonmodulo(hash, 4096), 2048); + hash = (hash - pos.x) / 4096; + pos.y = unsigned_to_signed(pythonmodulo(hash, 4096), 2048); + hash = (hash - pos.y) / 4096; + pos.z = unsigned_to_signed(pythonmodulo(hash, 4096), 2048); + return pos; +} + +/******************* + * End black magic * + *******************/ + +#endif // DB_HEADER diff --git a/types.h b/types.h new file mode 100644 index 0000000..c92cc8d --- /dev/null +++ b/types.h @@ -0,0 +1,5 @@ + +#include + +typedef std::basic_string ustring; +