Don't use a temporary table for block positions

This also moves database functions and data structures to db.h

TileGenerator.cpp

@@ -1,12 +1,3 @@
-/*
- * =====================================================================
- *        Version:  1.0
- *        Created:  23.08.2012 12:35:53
- *         Author:  Miroslav Bendík
- *        Company:  LinuxOS.sk
- * =====================================================================
- */
-
 #include <cstdio>
 #include <cstdlib>
 #include <climits>
@@ -29,26 +20,6 @@
 
 using namespace std;
 
-static inline int64_t pythonmodulo(int64_t i, int64_t mod)
-{
-	if (i >= 0) {
-		return i % mod;
-	}
-	else {
-		return mod - ((-i) % mod);
-	}
-}
-
-static inline int unsignedToSigned(long i, long max_positive)
-{
-	if (i < max_positive) {
-		return i;
-	}
-	else {
-		return i - 2l * max_positive;
-	}
-}
-
 static inline uint16_t readU16(const unsigned char *data)
 {
 	return data[0] << 8 | data[1];
@@ -334,9 +305,9 @@ void TileGenerator::openDb(const std::string &input)
 
 void TileGenerator::loadBlocks()
 {
-	std::vector<int64_t> vec = m_db->getBlockPos();
+	std::vector<BlockPos> vec = m_db->getBlockPos();
-	for(std::vector<int64_t>::iterator it = vec.begin(); it != vec.end(); ++it) {
+	for (std::vector<BlockPos>::iterator it = vec.begin(); it != vec.end(); ++it) {
-		BlockPos pos = decodeBlockPos(*it);
+		BlockPos pos = *it;
 		// Check that it's in geometry (from --geometry option)
 		if (pos.x < m_geomX || pos.x > m_geomX2 || pos.z < m_geomY || pos.z > m_geomY2) {
 			continue;
@@ -364,17 +335,6 @@ void TileGenerator::loadBlocks()
 	m_positions.unique();
 }
 
-inline BlockPos TileGenerator::decodeBlockPos(int64_t blockId) const
-{
-	BlockPos pos;
-	pos.x = unsignedToSigned(pythonmodulo(blockId, 4096), 2048);
-	blockId = (blockId - pos.x) / 4096;
-	pos.y = unsignedToSigned(pythonmodulo(blockId, 4096), 2048);
-	blockId = (blockId - pos.y) / 4096;
-	pos.z = unsignedToSigned(pythonmodulo(blockId, 4096), 2048);
-	return pos;
-}
-
 void TileGenerator::createImage()
 {
 	m_mapWidth = (m_xMax - m_xMin + 1) * 16;
@@ -385,27 +345,13 @@ void TileGenerator::createImage()
 	gdImageFilledRectangle(m_image, 0, 0, m_mapWidth + m_border - 1, m_mapHeight + m_border -1, color2int(m_bgColor));
 }
 
-std::map<int, TileGenerator::BlockList> TileGenerator::getBlocksOnZ(int zPos)
-{
-	DBBlockList in = m_db->getBlocksOnZ(zPos);
-	std::map<int, BlockList> out;
-	for(DBBlockList::const_iterator it = in.begin(); it != in.end(); ++it) {
-		Block b = Block(decodeBlockPos(it->first), it->second);
-		if(out.find(b.first.x) == out.end()) {
-			BlockList bl;
-			out[b.first.x] = bl;
-		}
-		out[b.first.x].push_back(b);
-	}
-	return out;
-}
-
 void TileGenerator::renderMap()
 {
 	std::list<int> zlist = getZValueList();
 	for (std::list<int>::iterator zPosition = zlist.begin(); zPosition != zlist.end(); ++zPosition) {
 		int zPos = *zPosition;
-		std::map<int, BlockList> blocks = getBlocksOnZ(zPos);
+		std::map<int16_t, BlockList> blocks;
+		m_db->getBlocksOnZ(blocks, zPos);
 		for (std::list<std::pair<int, int> >::const_iterator position = m_positions.begin(); position != m_positions.end(); ++position) {
 			if (position->second != zPos) {
 				continue;
@@ -436,8 +382,8 @@ void TileGenerator::renderMap()
 
 				ZlibDecompressor decompressor(data, length);
 				decompressor.setSeekPos(dataOffset);
-				ZlibDecompressor::string mapData = decompressor.decompress();
+				ustring mapData = decompressor.decompress();
-				ZlibDecompressor::string mapMetadata = decompressor.decompress();
+				ustring mapMetadata = decompressor.decompress();
 				dataOffset = decompressor.seekPos();
 
 				// Skip unused data
@@ -519,7 +465,7 @@ void TileGenerator::renderMap()
 	}
 }
 
-inline void TileGenerator::renderMapBlock(const unsigned_string &mapBlock, const BlockPos &pos, int version)
+inline void TileGenerator::renderMapBlock(const ustring &mapBlock, const BlockPos &pos, int version)
 {
 	int xBegin = (pos.x - m_xMin) * 16;
 	int zBegin = (m_zMax - pos.z) * 16;

TileGenerator.h

@@ -1,14 +1,5 @@
-/*
+#ifndef TILEGENERATOR_HEADER
- * =====================================================================
+#define TILEGENERATOR_HEADER
- *        Version:  1.0
- *        Created:  23.08.2012 12:35:59
- *         Author:  Miroslav Bendík
- *        Company:  LinuxOS.sk
- * =====================================================================
- */
-
-#ifndef TILEGENERATOR_H_JJNUCARH
-#define TILEGENERATOR_H_JJNUCARH
 
 #include <gd.h>
 #include <iosfwd>
@@ -19,6 +10,7 @@
 #include <string>
 #include "PixelAttributes.h"
 #include "db.h"
+#include "types.h"
 
 struct Color {
 	Color(): r(0xFF), g(0xFF), b(0xFF), a(0) {};
@@ -42,42 +34,10 @@ struct ColorEntry {
 };
 
 
-struct BlockPos {
-	int x;
-	int y;
-	int z;
-	bool operator<(const BlockPos& p) const
-	{
-		if (z > p.z) {
-			return true;
-		}
-		if (z < p.z) {
-			return false;
-		}
-		if (y > p.y) {
-			return true;
-		}
-		if (y < p.y) {
-			return false;
-		}
-		if (x > p.x) {
-			return true;
-		}
-		if (x < p.x) {
-			return false;
-		}
-		return false;
-	}
-};
-
-
 class TileGenerator
 {
 private:
-	typedef std::basic_string<unsigned char> unsigned_string;
 	typedef std::map<std::string, ColorEntry> ColorMap;
-	typedef std::pair<BlockPos, unsigned_string> Block;
-	typedef std::list<Block> BlockList;
 
 public:
 	TileGenerator();
@@ -102,12 +62,10 @@ private:
 	void parseColorsStream(std::istream &in);
 	void openDb(const std::string &input);
 	void loadBlocks();
-	BlockPos decodeBlockPos(int64_t blockId) const;
 	void createImage();
 	void renderMap();
 	std::list<int> getZValueList() const;
-	std::map<int, BlockList> getBlocksOnZ(int zPos);
+	void renderMapBlock(const ustring &mapBlock, const BlockPos &pos, int version);
-	void renderMapBlock(const unsigned_string &mapBlock, const BlockPos &pos, int version);
 	void renderShading(int zPos);
 	void renderScale();
 	void renderOrigin();
@@ -153,7 +111,7 @@ private:
 
 	int m_blockAirId;
 	int m_blockIgnoreId;
-}; /* -----  end of class TileGenerator  ----- */
+}; // class TileGenerator
 
-#endif /* end of include guard: TILEGENERATOR_H_JJNUCARH */
+#endif // TILEGENERATOR_HEADER
 

ZlibDecompressor.cpp

@@ -32,12 +32,12 @@ std::size_t ZlibDecompressor::seekPos() const
 	return m_seekPos;
 }
 
-ZlibDecompressor::string ZlibDecompressor::decompress()
+ustring ZlibDecompressor::decompress()
 {
 	const unsigned char *data = m_data + m_seekPos;
 	const std::size_t size = m_size - m_seekPos;
 
-	string buffer;
+	ustring buffer;
 	const size_t BUFSIZE = 128 * 1024;
 	uint8_t temp_buffer[BUFSIZE];
 
@@ -58,7 +58,7 @@ ZlibDecompressor::string ZlibDecompressor::decompress()
 		strm.avail_out = BUFSIZE;
 		strm.next_out = temp_buffer;
 		ret = inflate(&strm, Z_NO_FLUSH);
-		buffer += string(reinterpret_cast<unsigned char *>(temp_buffer), BUFSIZE - strm.avail_out);
+		buffer += ustring(reinterpret_cast<unsigned char *>(temp_buffer), BUFSIZE - strm.avail_out);
 	} while (ret == Z_OK);
 	if (ret != Z_STREAM_END) {
 		throw DecompressError();

ZlibDecompressor.h

@@ -12,12 +12,12 @@
 
 #include <cstdlib>
 #include <string>
+#include "types.h"
 
 
 class ZlibDecompressor
 {
 public:
-	typedef std::basic_string<unsigned char> string;
 	class DecompressError {
 	};
 
@@ -25,7 +25,7 @@ public:
 	~ZlibDecompressor();
 	void setSeekPos(std::size_t seekPos);
 	std::size_t seekPos() const;
-	string decompress();
+	ustring decompress();
 
 private:
 	const unsigned char *m_data;

db-leveldb.cpp

@@ -1,74 +1,74 @@
-#include "db-leveldb.h"
 #include <stdexcept>
 #include <sstream>
+#include "db-leveldb.h"
+#include "types.h"
 
-inline int64_t stoi64(const std::string &s) {
+static inline int64_t stoi64(const std::string &s)
+{
 	std::stringstream tmp(s);
-	long long t;
+	int64_t t;
 	tmp >> t;
 	return t;
 }
 
-inline std::string i64tos(int64_t i) {
+
-	std::ostringstream o;
+static inline std::string i64tos(int64_t i)
-	o<<i;
+{
-	return o.str();
+	std::ostringstream os;
+	os << i;
+	return os.str();
 }
 
-DBLevelDB::DBLevelDB(const std::string &mapdir) {
+DBLevelDB::DBLevelDB(const std::string &mapdir)
+{
 	leveldb::Options options;
-	posCacheLoaded = false;
 	options.create_if_missing = false;
 	leveldb::Status status = leveldb::DB::Open(options, mapdir + "map.db", &db);
-	if(!status.ok())
+	if (!status.ok()) {
 		throw std::runtime_error("Failed to open Database");
+	}
+
+	loadPosCache();
 }
 
-DBLevelDB::~DBLevelDB() {
+
+DBLevelDB::~DBLevelDB()
+{
 	delete db;
 }
 
-std::vector<int64_t> DBLevelDB::getBlockPos() {
+
-	loadPosCache();
+std::vector<BlockPos> DBLevelDB::getBlockPos()
+{
 	return posCache;
 }
 
-void DBLevelDB::loadPosCache() {
-	if (posCacheLoaded) {
-		return;
-	}
 
+void DBLevelDB::loadPosCache()
+{
 	leveldb::Iterator * it = db->NewIterator(leveldb::ReadOptions());
 	for (it->SeekToFirst(); it->Valid(); it->Next()) {
-		posCache.push_back(stoi64(it->key().ToString()));
+		int64_t posHash = stoi64(it->key().ToString());
+		posCache.push_back(decodeBlockPos(posHash));
 	}
 	delete it;
-	posCacheLoaded = true;
 }
 
-DBBlockList DBLevelDB::getBlocksOnZ(int zPos) {
+
-	DBBlockList blocks;
+void DBLevelDB::getBlocksOnZ(std::map<int16_t, BlockList> &blocks, int16_t zPos)
+{
 	std::string datastr;
 	leveldb::Status status;
 
-	int64_t psMin = (zPos * 16777216L) - 0x800000;
+	for (std::vector<BlockPos>::iterator it = posCache.begin(); it != posCache.end(); ++it) {
-	int64_t psMax = (zPos * 16777216L) + 0x7fffff;
+		if (it->z != zPos) {
-
-	for (std::vector<int64_t>::iterator it = posCache.begin(); it != posCache.end(); ++it) {
-		int64_t i = *it;
-		if (i < psMin || i > psMax) {
 			continue;
 		}
-		status = db->Get(leveldb::ReadOptions(), i64tos(i), &datastr);
+		status = db->Get(leveldb::ReadOptions(), i64tos(encodeBlockPos(*it)), &datastr);
 		if (status.ok()) {
-			blocks.push_back(
+			Block b(*it, ustring((const unsigned char *) datastr.data(), datastr.size()));
-				DBBlock(i,
+			blocks[b.first.x].push_back(b);
-					std::basic_string<unsigned char>((const unsigned char*) datastr.data(), datastr.size())
-				)
-			);
 		}
 	}
-
-	return blocks;
 }
 

db-leveldb.h

@@ -1,5 +1,5 @@
-#ifndef _DB_LEVELDB_H
+#ifndef DB_LEVELDB_HEADER
-#define _DB_LEVELDB_H
+#define DB_LEVELDB_HEADER
 
 #include "db.h"
 #include <leveldb/db.h>
@@ -7,16 +7,15 @@
 class DBLevelDB : public DB {
 public:
 	DBLevelDB(const std::string &mapdir);
-	virtual std::vector<int64_t> getBlockPos();
+	virtual std::vector<BlockPos> getBlockPos();
-	virtual DBBlockList getBlocksOnZ(int zPos);
+	virtual void getBlocksOnZ(std::map<int16_t, BlockList> &blocks, int16_t zPos);
 	~DBLevelDB();
 private:
 	void loadPosCache();
 
-	leveldb::DB *db;
+	std::vector<BlockPos> posCache;
 
-	bool posCacheLoaded;
+	leveldb::DB *db;
-	std::vector<int64_t> posCache;
 };
 
-#endif // _DB_LEVELDB_H
+#endif // DB_LEVELDB_HEADER

db-sqlite3.cpp

@@ -1,6 +1,7 @@
-#include "db-sqlite3.h"
 #include <stdexcept>
 #include <unistd.h> // for usleep
+#include "db-sqlite3.h"
+#include "types.h"
 
 #define SQLRES(f, good) \
 	result = (sqlite3_##f);\
@@ -9,36 +10,42 @@
 	}
 #define SQLOK(f) SQLRES(f, SQLITE_OK)
 
-DBSQLite3::DBSQLite3(const std::string &mapdir) {
+
+DBSQLite3::DBSQLite3(const std::string &mapdir)
+{
 	int result;
 	std::string db_name = mapdir + "map.sqlite";
 
-	SQLOK(open_v2(db_name.c_str(), &db, SQLITE_OPEN_READONLY | SQLITE_OPEN_PRIVATECACHE, 0))
+	SQLOK(open_v2(db_name.c_str(), &db, SQLITE_OPEN_READONLY |
+			SQLITE_OPEN_PRIVATECACHE, 0))
 
 	SQLOK(prepare_v2(db,
-			"SELECT pos, data FROM blocks WHERE (pos >= ? AND pos <= ?)",
+			"SELECT pos, data FROM blocks WHERE pos BETWEEN ? AND ?",
-		-1, &stmt_get_blocks, NULL))
+		-1, &stmt_get_blocks_z, NULL))
 
 	SQLOK(prepare_v2(db,
 			"SELECT pos FROM blocks",
 		-1, &stmt_get_block_pos, NULL))
 }
 
-DBSQLite3::~DBSQLite3() {
+
+DBSQLite3::~DBSQLite3()
+{
 	int result;
-	SQLOK(finalize(stmt_get_blocks));
+	SQLOK(finalize(stmt_get_blocks_z));
 	SQLOK(finalize(stmt_get_block_pos));
 
 	SQLOK(close(db));
 }
 
-std::vector<int64_t> DBSQLite3::getBlockPos() {
+std::vector<BlockPos> DBSQLite3::getBlockPos()
-	std::vector<int64_t> vec;
+{
 	int result;
+	std::vector<BlockPos> positions;
 	while ((result = sqlite3_step(stmt_get_block_pos)) != SQLITE_DONE) {
 		if (result == SQLITE_ROW) {
-			int64_t blockpos = sqlite3_column_int64(stmt_get_block_pos, 0);
+			int64_t posHash = sqlite3_column_int64(stmt_get_block_pos, 0);
-			vec.push_back(blockpos);
+			positions.push_back(decodeBlockPos(posHash));
 		} else if (result == SQLITE_BUSY) { // Wait some time and try again
 			usleep(10000);
 		} else {
@@ -46,35 +53,36 @@ std::vector<int64_t> DBSQLite3::getBlockPos() {
 		}
 	}
 	SQLOK(reset(stmt_get_block_pos));
-	return vec;
+	return positions;
 }
 
-DBBlockList DBSQLite3::getBlocksOnZ(int zPos)
+
+void DBSQLite3::getBlocksOnZ(std::map<int16_t, BlockList> &blocks, int16_t zPos)
 {
-	DBBlockList blocks;
-
-	int64_t psMin = (static_cast<sqlite3_int64>(zPos) * 16777216L) - 0x800000;
-	int64_t psMax = (static_cast<sqlite3_int64>(zPos) * 16777216L) + 0x7fffff;
-
-	sqlite3_bind_int64(stmt_get_blocks, 1, psMin);
-	sqlite3_bind_int64(stmt_get_blocks, 2, psMax);
-
 	int result;
-	while ((result = sqlite3_step(stmt_get_blocks)) != SQLITE_DONE) {
+
+	// Magic numbers!
+	int64_t minPos = (zPos * 0x1000000) - 0x800000;
+	int64_t maxPos = (zPos * 0x1000000) + 0x7FFFFF;
+
+	SQLOK(bind_int64(stmt_get_blocks_z, 1, minPos));
+	SQLOK(bind_int64(stmt_get_blocks_z, 2, maxPos));
+
+	while ((result = sqlite3_step(stmt_get_blocks_z)) != SQLITE_DONE) {
 		if (result == SQLITE_ROW) {
-			int64_t blocknum = sqlite3_column_int64(stmt_get_blocks, 0);
+			int64_t posHash = sqlite3_column_int64(stmt_get_blocks_z, 0);
-			const unsigned char *data = reinterpret_cast<const unsigned char *>(sqlite3_column_blob(stmt_get_blocks, 1));
+			const unsigned char *data = reinterpret_cast<const unsigned char *>(
-			int size = sqlite3_column_bytes(stmt_get_blocks, 1);
+					sqlite3_column_blob(stmt_get_blocks_z, 1));
-			blocks.push_back(DBBlock(blocknum, std::basic_string<unsigned char>(data, size)));
+			size_t size = sqlite3_column_bytes(stmt_get_blocks_z, 1);
+			Block b(decodeBlockPos(posHash), ustring(data, size));
+			blocks[b.first.x].push_back(b);
 		} else if (result == SQLITE_BUSY) { // Wait some time and try again
 			usleep(10000);
 		} else {
 			throw std::runtime_error(sqlite3_errmsg(db));
 		}
 	}
-	SQLOK(reset(stmt_get_blocks));
+	SQLOK(reset(stmt_get_blocks_z));
-
-	return blocks;
 }
 
 #undef SQLRES

db-sqlite3.h

@@ -7,14 +7,14 @@
 class DBSQLite3 : public DB {
 public:
 	DBSQLite3(const std::string &mapdir);
-	virtual std::vector<int64_t> getBlockPos();
+	virtual std::vector<BlockPos> getBlockPos();
-	virtual DBBlockList getBlocksOnZ(int zPos);
+	virtual void getBlocksOnZ(std::map<int16_t, BlockList> &blocks, int16_t zPos);
 	~DBSQLite3();
 private:
 	sqlite3 *db;
 
 	sqlite3_stmt *stmt_get_block_pos;
-	sqlite3_stmt *stmt_get_blocks;
+	sqlite3_stmt *stmt_get_blocks_z;
 };
 
 #endif // _DB_SQLITE3_H

db.h

@@ -1,19 +1,110 @@
-#ifndef _DB_H
+#ifndef DB_HEADER
-#define _DB_H
+#define DB_HEADER
 
 #include <stdint.h>
-#include <vector>
+#include <map>
 #include <list>
+#include <vector>
 #include <string>
 #include <utility>
+#include "types.h"
 
-typedef std::pair<int64_t, std::basic_string<unsigned char> > DBBlock;
-typedef std::list<DBBlock> DBBlockList;
 
-class DB {
+class BlockPos {
 public:
-	virtual std::vector<int64_t> getBlockPos() = 0;
+	int16_t x;
-	virtual DBBlockList getBlocksOnZ(int zPos) = 0;
+	int16_t y;
+	int16_t z;
+
+	bool operator < (const BlockPos &p) const
+	{
+		if (z > p.z) {
+			return true;
+		}
+		if (z < p.z) {
+			return false;
+		}
+		if (y > p.y) {
+			return true;
+		}
+		if (y < p.y) {
+			return false;
+		}
+		if (x > p.x) {
+			return true;
+		}
+		if (x < p.x) {
+			return false;
+		}
+		return false;
+	}
 };
 
-#endif // _DB_H
+
+typedef std::pair<BlockPos, ustring> Block;
+typedef std::list<Block> BlockList;
+
+
+class DB {
+protected:
+	inline int64_t  encodeBlockPos(const BlockPos pos) const;
+	inline BlockPos decodeBlockPos(int64_t hash) const;
+
+public:
+	virtual std::vector<BlockPos> getBlockPos() = 0;
+	virtual void getBlocksOnZ(std::map<int16_t, BlockList> &blocks, int16_t zPos) = 0;
+};
+
+
+
+/****************
+ * Black magic! *
+ ****************
+ * The position hashing is seriously messed up,
+ * and is a lot more complicated than it looks.
+ */
+
+static inline int16_t unsigned_to_signed(uint16_t i, uint16_t max_positive)
+{
+	if (i < max_positive) {
+		return i;
+	} else {
+		return i - (max_positive * 2);
+	}
+}
+
+
+// Modulo of a negative number does not work consistently in C
+static inline int64_t pythonmodulo(int64_t i, int64_t mod)
+{
+	if (i >= 0) {
+		return i % mod;
+	}
+	return mod - ((-i) % mod);
+}
+
+
+inline int64_t DB::encodeBlockPos(const BlockPos pos) const
+{
+	return (uint64_t) pos.z * 0x1000000 +
+		(uint64_t) pos.y * 0x1000 +
+		(uint64_t) pos.x;
+}
+
+
+inline BlockPos DB::decodeBlockPos(int64_t hash) const
+{
+	BlockPos pos;
+	pos.x = unsigned_to_signed(pythonmodulo(hash, 4096), 2048);
+	hash = (hash - pos.x) / 4096;
+	pos.y = unsigned_to_signed(pythonmodulo(hash, 4096), 2048);
+	hash = (hash - pos.y) / 4096;
+	pos.z = unsigned_to_signed(pythonmodulo(hash, 4096), 2048);
+	return pos;
+}
+
+/*******************
+ * End black magic *
+ *******************/
+
+#endif // DB_HEADER

types.h

@@ -0,0 +1,5 @@
+
+#include <string>
+
+typedef std::basic_string<unsigned char> ustring;
+