LevelDB support

.gitignore

@@ -6,5 +6,5 @@ CMakeCache.txt
 CMakeFiles/
 Makefile
 cmake_install.cmake
-
+cmake_config.h
 *~

CMakeLists.txt

@@ -28,6 +28,25 @@ else (LIBSQLITE3_INCLUDE_DIR)
 	message(FATAL_ERROR "Could not find sqlite3")
 endif (LIBSQLITE3_INCLUDE_DIR)
 
+set(USE_LEVELDB 0)
+
+OPTION(ENABLE_LEVELDB "Enable LevelDB backend")
+
+if(ENABLE_LEVELDB)
+	find_library(LEVELDB_LIBRARY leveldb)
+	find_path(LEVELDB_INCLUDE_DIR db.h PATH_SUFFIXES leveldb)
+	message (STATUS "LevelDB library: ${LEVELDB_LIBRARY}")
+	message (STATUS "LevelDB headers: ${LEVELDB_INCLUDE_DIR}")
+	if(LEVELDB_LIBRARY AND LEVELDB_INCLUDE_DIR)
+		set(USE_LEVELDB 1)
+		message(STATUS "LevelDB backend enabled")
+		include_directories(${LEVELDB_INCLUDE_DIR})
+	else(LEVELDB_LIBRARY AND LEVELDB_INCLUDE_DIR)
+		set(USE_LEVELDB 0)
+		message(STATUS "LevelDB not found!")
+	endif(LEVELDB_LIBRARY AND LEVELDB_INCLUDE_DIR)
+endif(ENABLE_LEVELDB)
+
 find_program(XXD_EXECUTABLE xxd)
 
 if (XXD_EXECUTABLE)
@@ -51,12 +70,11 @@ include_directories(
 	${LIBSQLITE3_INCLUDE_DIRS}
 )
 
-set(mapper_HDRS
+configure_file(
-	PixelAttributes.h
+	"${PROJECT_SOURCE_DIR}/cmake_config.h.in"
-	PlayerAttributes.h
+	"${PROJECT_BINARY_DIR}/cmake_config.h"
-	TileGenerator.h
-	ZlibDecompressor.h
 )
+add_definitions ( -DUSE_CMAKE_CONFIG_H )
 
 set(mapper_SRCS
 	PixelAttributes.cpp
@@ -64,16 +82,21 @@ set(mapper_SRCS
 	TileGenerator.cpp
 	ZlibDecompressor.cpp
 	mapper.cpp
+	db-sqlite3.cpp
 )
 
+if(USE_LEVELDB)
+	set(mapper_SRCS ${mapper_SRCS} db-leveldb.cpp)
+endif(USE_LEVELDB)
+
 add_executable(minetestmapper
-	${mapper_HDRS}
 	${mapper_SRCS}
 )
 
 target_link_libraries(
 	minetestmapper
 	${LIBSQLITE3_LIBRARIES}
+	${LEVELDB_LIBRARY}
 	gd
 	z
 )

README.rst

@@ -62,6 +62,9 @@ min-y:
 max-y:
     Don't draw nodes above this y value, `--max-y 75`
 
+backend:
+    Use specific map backend, supported: sqlite3, leveldb, `--backend leveldb`
+
 geometry:
     Limit area to specific geometry, `--geometry -800:-800+1600+1600`
 

TileGenerator.cpp

@@ -19,10 +19,14 @@
 #include "TileGenerator.h"
 #include "ZlibDecompressor.h"
 #include "colors.h"
+#include "db-sqlite3.h"
+#if USE_LEVELDB
+#include "db-leveldb.h"
+#endif
 
 using namespace std;
 
-static inline sqlite3_int64 pythonmodulo(sqlite3_int64 i, sqlite3_int64 mod)
+static inline int64_t pythonmodulo(int64_t i, int64_t mod)
 {
 	if (i >= 0) {
 		return i % mod;
@@ -96,7 +100,7 @@ TileGenerator::TileGenerator():
 	m_drawScale(false),
 	m_shading(true),
 	m_border(0),
-	m_db(0),
+	m_backend("sqlite3"),
 	m_image(0),
 	m_xMin(INT_MAX),
 	m_xMax(INT_MIN),
@@ -116,10 +120,6 @@ TileGenerator::TileGenerator():
 
 TileGenerator::~TileGenerator()
 {
-	if (m_db != 0) {
-		sqlite3_close(m_db);
-		m_db = 0;
-	}
 }
 
 void TileGenerator::setBgColor(const std::string &bgColor)
@@ -235,6 +235,11 @@ void TileGenerator::parseColorsFile(const std::string &fileName)
 	parseColorsStream(in);
 }
 
+void TileGenerator::setBackend(std::string backend)
+{
+	m_backend = backend;
+}
+
 void TileGenerator::generate(const std::string &input, const std::string &output)
 {
 	string input_path = input;
@@ -287,59 +292,49 @@ void TileGenerator::parseColorsStream(std::istream &in)
 
 void TileGenerator::openDb(const std::string &input)
 {
-	string db_name = input + "map.sqlite";
+	if(m_backend == "sqlite3")
-	if (sqlite3_open_v2(db_name.c_str(), &m_db, SQLITE_OPEN_READONLY | SQLITE_OPEN_PRIVATECACHE, 0) != SQLITE_OK) {
+		m_db = new DBSQLite3(input);
-		throw std::runtime_error(std::string(sqlite3_errmsg(m_db)) + ", Database file: " + db_name);
+#if USE_LEVELDB
-	}
+	if(m_backend == "leveldb")
+		m_db = new DBLevelDB(input);
+#endif
+	else
+		throw std::runtime_error(((std::string) "Unknown map backend: ") + m_backend);
 }
 
 void TileGenerator::loadBlocks()
 {
-	sqlite3_stmt *statement;
+	std::vector<int64_t> vec = m_db->getBlockPos();
-	string sql = "SELECT pos FROM blocks";
+	for(unsigned int i = 0; i < vec.size(); i++) {
-	if (sqlite3_prepare_v2(m_db, sql.c_str(), sql.length(), &statement, 0) == SQLITE_OK) {
+		BlockPos pos = decodeBlockPos(vec[i]);
-		int result = 0;
+		if (pos.x < m_geomX || pos.x >= m_geomX2 || pos.z < m_geomY || pos.z >= m_geomY2) {
-		while (true) {
+			continue;
-			result = sqlite3_step(statement);
-			if(result == SQLITE_ROW) {
-				sqlite3_int64 blocknum = sqlite3_column_int64(statement, 0);
-				BlockPos pos = decodeBlockPos(blocknum);
-				if (pos.x < m_geomX || pos.x >= m_geomX2 || pos.z < m_geomY || pos.z >= m_geomY2) {
-					continue;
-				}
-				if (pos.y < m_yMin) {
-					continue;
-				}
-				if (pos.y > m_yMax) {
-					continue;
-				}
-				if (pos.x < m_xMin) {
-					m_xMin = pos.x;
-				}
-				if (pos.x > m_xMax) {
-					m_xMax = pos.x;
-				}
-				if (pos.z < m_zMin) {
-					m_zMin = pos.z;
-				}
-				if (pos.z > m_zMax) {
-					m_zMax = pos.z;
-				}
-				m_positions.push_back(std::pair<int, int>(pos.x, pos.z));
-			}
-			else {
-				break;
-			}
 		}
+		if (pos.y < m_yMin) {
+			continue;
+		}
+		if (pos.y > m_yMax) {
+			continue;
+		}
+		if (pos.x < m_xMin) {
+			m_xMin = pos.x;
+		}
+		if (pos.x > m_xMax) {
+			m_xMax = pos.x;
+		}
+		if (pos.z < m_zMin) {
+			m_zMin = pos.z;
+		}
+		if (pos.z > m_zMax) {
+			m_zMax = pos.z;
+		}
+		m_positions.push_back(std::pair<int, int>(pos.x, pos.z));
 		m_positions.sort();
 		m_positions.unique();
 	}
-	else {
-		throw std::runtime_error("Failed to get list of MapBlocks");
-	}
 }
 
-inline BlockPos TileGenerator::decodeBlockPos(sqlite3_int64 blockId) const
+inline BlockPos TileGenerator::decodeBlockPos(int64_t blockId) const
 {
 	BlockPos pos;
 	pos.x = unsignedToSigned(pythonmodulo(blockId, 4096), 2048);
@@ -360,18 +355,27 @@ void TileGenerator::createImage()
 	gdImageFilledRectangle(m_image, 0, 0, m_mapWidth + m_border - 1, m_mapHeight + m_border -1, rgb2int(m_bgColor.r, m_bgColor.g, m_bgColor.b));
 }
 
+std::map<int, TileGenerator::BlockList> TileGenerator::getBlocksOnZ(int zPos)
+{
+	DBBlockList in = m_db->getBlocksOnZ(zPos);
+	std::map<int, BlockList> out;
+	for(DBBlockList::const_iterator it = in.begin(); it != in.end(); ++it) {
+		Block b = Block(decodeBlockPos(it->first), it->second);
+		if(out.find(b.first.x) == out.end()) {
+			BlockList bl;
+			out[b.first.x] = bl;
+		}
+		out[b.first.x].push_back(b);
+	}
+	return out;
+}
+
 void TileGenerator::renderMap()
 {
-	sqlite3_stmt *statement;
-	string sql = "SELECT pos, data FROM blocks WHERE (pos >= ? AND pos <= ?)";
-	if (sqlite3_prepare_v2(m_db, sql.c_str(), sql.length(), &statement, 0) != SQLITE_OK) {
-		throw std::runtime_error("Failed to get MapBlock");
-	}
-
 	std::list<int> zlist = getZValueList();
 	for (std::list<int>::iterator zPosition = zlist.begin(); zPosition != zlist.end(); ++zPosition) {
 		int zPos = *zPosition;
-		map<int, BlockList> blocks = getBlocksOnZ(zPos, statement);
+		std::map<int, BlockList> blocks = getBlocksOnZ(zPos);
 		for (std::list<std::pair<int, int> >::const_iterator position = m_positions.begin(); position != m_positions.end(); ++position) {
 			if (position->second != zPos) {
 				continue;
@@ -620,37 +624,6 @@ inline std::list<int> TileGenerator::getZValueList() const
 	return zlist;
 }
 
-std::map<int, TileGenerator::BlockList> TileGenerator::getBlocksOnZ(int zPos, sqlite3_stmt *statement) const
-{
-	map<int, BlockList> blocks;
-
-	sqlite3_int64 psMin;
-	sqlite3_int64 psMax;
-
-	psMin = (static_cast<sqlite3_int64>(zPos) * 16777216l) - 0x800000;
-	psMax = (static_cast<sqlite3_int64>(zPos) * 16777216l) + 0x7fffff;
-	sqlite3_bind_int64(statement, 1, psMin);
-	sqlite3_bind_int64(statement, 2, psMax);
-
-	int result = 0;
-	while (true) {
-		result = sqlite3_step(statement);
-		if(result == SQLITE_ROW) {
-			sqlite3_int64 blocknum = sqlite3_column_int64(statement, 0);
-			const unsigned char *data = reinterpret_cast<const unsigned char *>(sqlite3_column_blob(statement, 1));
-			int size = sqlite3_column_bytes(statement, 1);
-			BlockPos pos = decodeBlockPos(blocknum);
-			blocks[pos.x].push_back(Block(pos, unsigned_string(data, size)));
-		}
-		else {
-			break;
-		}
-	}
-	sqlite3_reset(statement);
-
-	return blocks;
-}
-
 void TileGenerator::writeImage(const std::string &output)
 {
 	FILE *out;

TileGenerator.h

@@ -15,10 +15,10 @@
 #include <list>
 #include <map>
 #include <set>
-#include <sqlite3.h>
 #include <stdint.h>
 #include <string>
 #include "PixelAttributes.h"
+#include "db.h"
 
 struct Color {
 	Color(): r(255), g(255), b(255) {};
@@ -80,17 +80,18 @@ public:
 	void setMinY(int y);
 	void setMaxY(int y);
 	void parseColorsFile(const std::string &fileName);
+	void setBackend(std::string backend);
 	void generate(const std::string &input, const std::string &output);
 
 private:
 	void parseColorsStream(std::istream &in);
 	void openDb(const std::string &input);
 	void loadBlocks();
-	BlockPos decodeBlockPos(sqlite3_int64 blockId) const;
+	BlockPos decodeBlockPos(int64_t blockId) const;
 	void createImage();
 	void renderMap();
 	std::list<int> getZValueList() const;
-	std::map<int, BlockList> getBlocksOnZ(int zPos, sqlite3_stmt *statement) const;
+	std::map<int, BlockList> getBlocksOnZ(int zPos);
 	void renderMapBlock(const unsigned_string &mapBlock, const BlockPos &pos, int version);
 	void renderShading(int zPos);
 	void renderScale();
@@ -111,8 +112,9 @@ private:
 	bool m_drawScale;
 	bool m_shading;
 	int m_border;
+	std::string m_backend;
 
-	sqlite3 *m_db;
+	DB *m_db;
 	gdImagePtr m_image;
 	PixelAttributes m_blockPixelAttributes;
 	int m_xMin;

cmake_config.h.in

@@ -0,0 +1,9 @@
+// Filled in by the build system
+
+#ifndef CMAKE_CONFIG_H
+#define CMAKE_CONFIG_H
+
+#define USE_LEVELDB @USE_LEVELDB@
+
+#endif
+

config.h

@@ -14,3 +14,9 @@
 #endif
 
 #define BLOCK_SIZE 16
+
+#ifdef USE_CMAKE_CONFIG_H
+#include "cmake_config.h"
+#else
+#define USE_LEVELDB 0
+#endif

db-leveldb.cpp

@@ -0,0 +1,64 @@
+#include "db-leveldb.h"
+#include <stdexcept>
+#include <sstream>
+
+inline int64_t stoi64(const std::string &s) {
+	std::stringstream tmp(s);
+	long long t;
+	tmp >> t;
+	return t;
+}
+
+inline std::string i64tos(int64_t i) {
+	std::ostringstream o;
+	o<<i;
+	return o.str();
+}
+
+DBLevelDB::DBLevelDB(const std::string &mapdir) {
+	leveldb::Options options;
+	options.create_if_missing = false;
+	leveldb::Status status = leveldb::DB::Open(options, mapdir + "map.db", &m_db);
+	if(!status.ok())
+		throw std::runtime_error("Failed to open Database");
+}
+
+DBLevelDB::~DBLevelDB() {
+	delete m_db;
+}
+
+std::vector<int64_t> DBLevelDB::getBlockPos() {
+	std::vector<int64_t> vec;
+	std::set<int64_t> s;
+	leveldb::Iterator* it = m_db->NewIterator(leveldb::ReadOptions());
+	for (it->SeekToFirst(); it->Valid(); it->Next()) {
+		vec.push_back(stoi64(it->key().ToString()));
+		s.insert(stoi64(it->key().ToString()));
+	}
+	delete it;
+	m_bpcache = s;
+	return vec;
+}
+
+DBBlockList DBLevelDB::getBlocksOnZ(int zPos)
+{
+	DBBlockList blocks;
+	std::string datastr;
+	leveldb::Status status;
+
+	int64_t psMin;
+	int64_t psMax;
+	psMin = (zPos * 16777216l) - 0x800000;
+	psMax = (zPos * 16777216l) + 0x7fffff;
+
+	for(int64_t i = psMin; i <= psMax; i++) { // FIXME: This is still very very inefficent (even with m_bpcache)
+		if(m_bpcache.find(i) == m_bpcache.end())
+			continue;
+		status = m_db->Get(leveldb::ReadOptions(), i64tos(i), &datastr);
+		if(status.ok())
+			blocks.push_back( DBBlock( i, std::basic_string<unsigned char>( (const unsigned char*) datastr.c_str(), datastr.size() ) ) );
+	}
+
+	return blocks;
+}
+

db-leveldb.h

@@ -0,0 +1,19 @@
+#ifndef _DB_LEVELDB_H
+#define _DB_LEVELDB_H
+
+#include "db.h"
+#include <leveldb/db.h>
+#include <set>
+
+class DBLevelDB : public DB {
+public:
+	DBLevelDB(const std::string &mapdir);
+	virtual std::vector<int64_t> getBlockPos();
+	virtual DBBlockList getBlocksOnZ(int zPos);
+	~DBLevelDB();
+private:
+	leveldb::DB *m_db;
+	std::set<int64_t> m_bpcache;
+};
+
+#endif // _DB_LEVELDB_H

db-sqlite3.cpp

@@ -0,0 +1,71 @@
+#include "db-sqlite3.h"
+#include <stdexcept>
+
+DBSQLite3::DBSQLite3(const std::string &mapdir) {
+	
+	std::string db_name = mapdir + "map.sqlite";
+	if (sqlite3_open_v2(db_name.c_str(), &m_db, SQLITE_OPEN_READONLY | SQLITE_OPEN_PRIVATECACHE, 0) != SQLITE_OK) {
+		throw std::runtime_error(std::string(sqlite3_errmsg(m_db)) + ", Database file: " + db_name);
+	}
+}
+
+DBSQLite3::~DBSQLite3() {
+	sqlite3_close_v2(m_db);
+}
+
+std::vector<int64_t> DBSQLite3::getBlockPos() {
+	std::vector<int64_t> vec;
+	sqlite3_stmt *statement;
+	std::string sql = "SELECT pos FROM blocks";
+	if (sqlite3_prepare_v2(m_db, sql.c_str(), sql.length(), &statement, 0) == SQLITE_OK) {
+		int result = 0;
+		while (true) {
+			result = sqlite3_step(statement);
+			if(result == SQLITE_ROW) {
+				sqlite3_int64 blocknum = sqlite3_column_int64(statement, 0);
+				vec.push_back(blocknum);
+			}
+			else
+				break;
+		}
+	} else {
+		throw std::runtime_error("Failed to get list of MapBlocks");
+	}
+	return vec;
+}
+
+DBBlockList DBSQLite3::getBlocksOnZ(int zPos)
+{
+	sqlite3_stmt *statement;
+	std::string sql = "SELECT pos, data FROM blocks WHERE (pos >= ? AND pos <= ?)";
+	if (sqlite3_prepare_v2(m_db, sql.c_str(), sql.length(), &statement, 0) != SQLITE_OK) {
+		throw std::runtime_error("Failed to prepare statement");
+	}
+	DBBlockList blocks;
+
+	sqlite3_int64 psMin;
+	sqlite3_int64 psMax;
+
+	psMin = (static_cast<sqlite3_int64>(zPos) * 16777216l) - 0x800000;
+	psMax = (static_cast<sqlite3_int64>(zPos) * 16777216l) + 0x7fffff;
+	sqlite3_bind_int64(statement, 1, psMin);
+	sqlite3_bind_int64(statement, 2, psMax);
+
+	int result = 0;
+	while (true) {
+		result = sqlite3_step(statement);
+		if(result == SQLITE_ROW) {
+			sqlite3_int64 blocknum = sqlite3_column_int64(statement, 0);
+			const unsigned char *data = reinterpret_cast<const unsigned char *>(sqlite3_column_blob(statement, 1));
+			int size = sqlite3_column_bytes(statement, 1);
+			blocks.push_back(DBBlock(blocknum, std::basic_string<unsigned char>(data, size)));
+		}
+		else {
+			break;
+		}
+	}
+	sqlite3_reset(statement);
+
+	return blocks;
+}
+

db-sqlite3.h

@@ -0,0 +1,17 @@
+#ifndef _DB_SQLITE3_H
+#define _DB_SQLITE3_H
+
+#include "db.h"
+#include <sqlite3.h>
+
+class DBSQLite3 : public DB {
+public:
+	DBSQLite3(const std::string &mapdir);
+	virtual std::vector<int64_t> getBlockPos();
+	virtual DBBlockList getBlocksOnZ(int zPos);
+	~DBSQLite3();
+private:
+	sqlite3 *m_db;
+};
+
+#endif // _DB_SQLITE3_H

db.h

@@ -0,0 +1,18 @@
+#ifndef _DB_H
+#define _DB_H
+
+#include <cstdint>
+#include <vector>
+#include <list>
+#include <string>
+
+typedef std::pair<int64_t, std::basic_string<unsigned char>> DBBlock;
+typedef std::list<DBBlock> DBBlockList;
+
+class DB {
+public:
+	virtual std::vector<int64_t> getBlockPos()=0;
+	virtual DBBlockList getBlocksOnZ(int zPos)=0;
+};
+
+#endif // _DB_H

mapper.cpp

@@ -30,6 +30,9 @@ void usage()
 			"  --drawplayers\n"
 			"  --draworigin\n"
 			"  --noshading\n"
+			"  --min-y <y>\n"
+			"  --max-y <y>\n"
+			"  --backend <sqlite3/leveldb>\n"
 			"  --geometry x:y+w+h\n"
 			"Color format: '#000000'\n";
 	std::cout << usage_text;
@@ -53,6 +56,7 @@ int main(int argc, char *argv[])
 		{"geometry", required_argument, 0, 'g'},
 		{"min-y", required_argument, 0, 'a'},
 		{"max-y", required_argument, 0, 'c'},
+		{"backend", required_argument, 0, 'd'},
 	};
 
 	string input;
@@ -135,6 +139,9 @@ int main(int argc, char *argv[])
 					generator.setGeometry(x, y, w, h);
 				}
 				break;
+			case 'd':
+				generator.setBackend(optarg);
+				break;
 			default:
 				exit(1);
 		}