Rewrite DB class to allow backends to fully optimize block fetches

TileGenerator.cpp

@@ -1,6 +1,7 @@
 #include <cstdio>
 #include <cstdlib>
 #include <climits>
+#include <cassert>
 #include <fstream>
 #include <iostream>
 #include <sstream>
@@ -89,8 +90,8 @@ TileGenerator::TileGenerator():
 	m_xMax(INT_MIN),
 	m_zMin(INT_MAX),
 	m_zMax(INT_MIN),
-	m_yMin(-30000),
+	m_yMin(INT16_MIN),
-	m_yMax(30000),
+	m_yMax(INT16_MAX),
 	m_geomX(-2048),
 	m_geomY(-2048),
 	m_geomX2(2048),
@@ -184,6 +185,7 @@ void TileGenerator::setBackend(std::string backend)
 
 void TileGenerator::setGeometry(int x, int y, int w, int h)
 {
+	assert(w > 0 && h > 0);
 	m_geomX  = round_multiple_nosign(x, 16) / 16;
 	m_geomY  = round_multiple_nosign(y, 16) / 16;
 	m_geomX2 = round_multiple_nosign(x + w, 16) / 16;
@@ -193,11 +195,15 @@ void TileGenerator::setGeometry(int x, int y, int w, int h)
 void TileGenerator::setMinY(int y)
 {
 	m_yMin = y;
+	if (m_yMin > m_yMax)
+		std::swap(m_yMin, m_yMax);
 }
 
 void TileGenerator::setMaxY(int y)
 {
 	m_yMax = y;
+	if (m_yMin > m_yMax)
+		std::swap(m_yMin, m_yMax);
 }
 
 void TileGenerator::parseColorsFile(const std::string &fileName)
@@ -244,7 +250,7 @@ void TileGenerator::generate(const std::string &input, const std::string &output
 	openDb(input_path);
 	loadBlocks();
 
-	if (m_dontWriteEmpty  && ! m_positions.size())
+	if (m_dontWriteEmpty && m_positions.empty())
 	{
 		closeDatabase();
 		return;
@@ -268,7 +274,7 @@ void TileGenerator::generate(const std::string &input, const std::string &output
 
 void TileGenerator::parseColorsStream(std::istream &in)
 {
-	char line[128];
+	char line[512];
 	while (in.good()) {
 		in.getline(line, sizeof(line));
 
@@ -281,11 +287,11 @@ void TileGenerator::parseColorsStream(std::istream &in)
 		if(strlen(line) == 0)
 			continue;
 
-		char name[64 + 1] = {0};
+		char name[128 + 1] = {0};
 		unsigned int r, g, b, a, t;
 		a = 255;
 		t = 0;
-		int items = sscanf(line, "%64s %u %u %u %u %u", name, &r, &g, &b, &a, &t);
+		int items = sscanf(line, "%128s %u %u %u %u %u", name, &r, &g, &b, &a, &t);
 		if(items < 4) {
 			std::cerr << "Failed to parse color entry '" << line << "'" << std::endl;
 			continue;
@@ -333,15 +339,17 @@ void TileGenerator::closeDatabase()
 
 void TileGenerator::loadBlocks()
 {
-	std::vector<BlockPos> vec = m_db->getBlockPos();
+	const int16_t yMax = m_yMax / 16 + 1;
-	for (std::vector<BlockPos>::iterator it = vec.begin(); it != vec.end(); ++it) {
+	std::vector<BlockPos> vec = m_db->getBlockPos(
-		BlockPos pos = *it;
+		BlockPos(m_geomX, m_yMin / 16, m_geomY),
-		// Check that it's in geometry (from --geometry option)
+		BlockPos(m_geomX2, yMax, m_geomY2)
-		if (pos.x < m_geomX || pos.x >= m_geomX2 || pos.z < m_geomY || pos.z >= m_geomY2)
+	);
-			continue;
+
-		// Check that it's between --min-y and --max-y
+	for (auto pos : vec) {
-		if (pos.y * 16 < m_yMin || pos.y * 16 > m_yMax)
+		assert(pos.x >= m_geomX && pos.x < m_geomX2);
-			continue;
+		assert(pos.y >= m_yMin / 16 && pos.y < yMax);
+		assert(pos.z >= m_geomY && pos.z < m_geomY2);
+
 		// Adjust minimum and maximum positions to the nearest block
 		if (pos.x < m_xMin)
 			m_xMin = pos.x;
@@ -352,10 +360,17 @@ void TileGenerator::loadBlocks()
 			m_zMin = pos.z;
 		if (pos.z > m_zMax)
 			m_zMax = pos.z;
-		m_positions.push_back(std::make_pair(pos.x, pos.z));
+
+		m_positions[pos.z].emplace(pos.x);
 	}
-	m_positions.sort();
+
-	m_positions.unique();
+#ifndef NDEBUG
+	int count = 0;
+	for (const auto &it : m_positions)
+		count += it.second.size();
+	std::cout << "Loaded " << count
+		<< " positions (across Z: " << m_positions.size() << ") for rendering" << std::endl;
+#endif
 }
 
 void TileGenerator::createImage()
@@ -405,13 +420,12 @@ void TileGenerator::createImage()
 void TileGenerator::renderMap()
 {
 	BlockDecoder blk;
-	std::list<int16_t> zlist = getZValueList();
+	const int16_t yMax = m_yMax / 16 + 1;
-	for (int16_t zPos : zlist) {
+
-		std::map<int16_t, BlockList> blocks;
+	for (auto it = m_positions.rbegin(); it != m_positions.rend(); ++it) {
-		m_db->getBlocksOnZ(blocks, zPos);
+		int16_t zPos = it->first;
-		for (const auto position : m_positions) {
+		for (auto it2 = it->second.rbegin(); it2 != it->second.rend(); ++it2) {
-			if (position.second != zPos)
+			int16_t xPos = *it2;
-				continue;
 
 			m_readPixels.reset();
 			m_readInfo.reset();
@@ -423,11 +437,13 @@ void TileGenerator::renderMap()
 				}
 			}
 
-			int16_t xPos = position.first;
+			BlockList blockStack;
-			blocks[xPos].sort();
+			m_db->getBlocksOnXZ(blockStack, xPos, zPos, m_yMin / 16, yMax);
-			const BlockList &blockStack = blocks[xPos];
+			blockStack.sort();
 			for (const auto &it : blockStack) {
-				const BlockPos &pos = it.first;
+				const BlockPos pos = it.first;
+				assert(pos.x == xPos && pos.z == zPos);
+				assert(pos.y >= m_yMin / 16 && pos.y < yMax);
 
 				blk.reset();
 				blk.decode(it.second);
@@ -651,17 +667,6 @@ void TileGenerator::renderPlayers(const std::string &inputPath)
 	}
 }
 
-inline std::list<int16_t> TileGenerator::getZValueList() const
-{
-	std::list<int16_t> zlist;
-	for (const auto position : m_positions)
-		zlist.push_back(position.second);
-	zlist.sort();
-	zlist.unique();
-	zlist.reverse();
-	return zlist;
-}
-
 void TileGenerator::writeImage(const std::string &output)
 {
 	m_image->save(output);

db-leveldb.cpp

@@ -11,7 +11,6 @@ static inline int64_t stoi64(const std::string &s)
 	return t;
 }
 
-
 static inline std::string i64tos(int64_t i)
 {
 	std::ostringstream os;
@@ -19,6 +18,7 @@ static inline std::string i64tos(int64_t i)
 	return os.str();
 }
 
+
 DBLevelDB::DBLevelDB(const std::string &mapdir)
 {
 	leveldb::Options options;
@@ -28,6 +28,9 @@ DBLevelDB::DBLevelDB(const std::string &mapdir)
 		throw std::runtime_error(std::string("Failed to open Database: ") + status.ToString());
 	}
 
+	/* LevelDB is a dumb key-value store, so the only optimization we can do
+	 * is to cache the block positions that exist in the db.
+	 */
 	loadPosCache();
 }
 
@@ -38,9 +41,21 @@ DBLevelDB::~DBLevelDB()
 }
 
 
-std::vector<BlockPos> DBLevelDB::getBlockPos()
+std::vector<BlockPos> DBLevelDB::getBlockPos(BlockPos min, BlockPos max)
 {
-	return posCache;
+	std::vector<BlockPos> res;
+	for (const auto &it : posCache) {
+		if (it.first < min.z || it.first >= max.z)
+			continue;
+		for (auto pos2 : it.second) {
+			if (pos2.first < min.x || pos2.first >= max.x)
+				continue;
+			if (pos2.second < min.y || pos2.second >= max.y)
+				continue;
+			res.emplace_back(pos2.first, pos2.second, it.first);
+		}
+	}
+	return res;
 }
 
 
@@ -49,26 +64,35 @@ void DBLevelDB::loadPosCache()
 	leveldb::Iterator * it = db->NewIterator(leveldb::ReadOptions());
 	for (it->SeekToFirst(); it->Valid(); it->Next()) {
 		int64_t posHash = stoi64(it->key().ToString());
-		posCache.push_back(decodeBlockPos(posHash));
+		BlockPos pos = decodeBlockPos(posHash);
+
+		posCache[pos.z].emplace_back(pos.x, pos.y);
 	}
 	delete it;
 }
 
 
-void DBLevelDB::getBlocksOnZ(std::map<int16_t, BlockList> &blocks, int16_t zPos)
+void DBLevelDB::getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
+		int16_t min_y, int16_t max_y)
 {
 	std::string datastr;
 	leveldb::Status status;
 
-	for (const auto &it : posCache) {
+	auto it = posCache.find(z);
-		if (it.z != zPos) {
+	if (it == posCache.cend())
+		return;
+	for (auto pos2 : it->second) {
+		if (pos2.first != x)
+			continue;
+		if (pos2.second < min_y || pos2.second >= max_y)
 			continue;
-		}
-		status = db->Get(leveldb::ReadOptions(), i64tos(encodeBlockPos(it)), &datastr);
-		if (status.ok()) {
-			Block b(it, ustring((const unsigned char *) datastr.data(), datastr.size()));
-			blocks[b.first.x].push_back(b);
-		}
-	}
-}
 
+		BlockPos pos(x, pos2.second, z);
+		status = db->Get(leveldb::ReadOptions(), i64tos(encodeBlockPos(pos)), &datastr);
+		if (status.ok()) {
+			blocks.emplace_back(
+				pos, ustring((unsigned char *) datastr.data(), datastr.size())
+			);
+		}
+	}
+}

db-postgresql.cpp

@@ -27,11 +27,16 @@ DBPostgreSQL::DBPostgreSQL(const std::string &mapdir)
 
 	prepareStatement(
 		"get_block_pos",
-		"SELECT posX, posY, posZ FROM blocks"
+		"SELECT posX::int4, posY::int4, posZ::int4 FROM blocks WHERE"
+		" (posX BETWEEN $1::int4 AND $2::int4) AND"
+		" (posY BETWEEN $3::int4 AND $4::int4) AND"
+		" (posZ BETWEEN $5::int4 AND $6::int4)"
 	);
 	prepareStatement(
-		"get_blocks_z",
+		"get_blocks",
-		"SELECT posX, posY, data FROM blocks WHERE posZ = $1::int4"
+		"SELECT posY::int4, data FROM blocks WHERE"
+		" posX = $1::int4 AND posZ = $2::int4"
+		" AND (posY BETWEEN $3::int4 AND $4::int4)"
 	);
 
 	checkResults(PQexec(db, "START TRANSACTION;"));
@@ -49,19 +54,31 @@ DBPostgreSQL::~DBPostgreSQL()
 	PQfinish(db);
 }
 
-std::vector<BlockPos> DBPostgreSQL::getBlockPos()
+std::vector<BlockPos> DBPostgreSQL::getBlockPos(BlockPos min, BlockPos max)
 {
-	std::vector<BlockPos> positions;
+	int32_t const x1 = htonl(min.x);
+	int32_t const x2 = htonl(max.x - 1);
+	int32_t const y1 = htonl(min.y);
+	int32_t const y2 = htonl(max.y - 1);
+	int32_t const z1 = htonl(min.z);
+	int32_t const z2 = htonl(max.z - 1);
+
+	const void *args[] = { &x1, &x2, &y1, &y2, &z1, &z2 };
+	const int argLen[] = { 4, 4, 4, 4, 4, 4 };
+	const int argFmt[] = { 1, 1, 1, 1, 1, 1 };
 
 	PGresult *results = execPrepared(
-		"get_block_pos", 0,
+		"get_block_pos", ARRLEN(args), args,
-		NULL, NULL, NULL, false, false
+		argLen, argFmt, false
 	);
 
 	int numrows = PQntuples(results);
 
+	std::vector<BlockPos> positions;
+	positions.reserve(numrows);
+
 	for (int row = 0; row < numrows; ++row)
-		positions.push_back(pg_to_blockpos(results, row, 0));
+		positions.emplace_back(pg_to_blockpos(results, row, 0));
 
 	PQclear(results);
 
@@ -69,16 +86,20 @@ std::vector<BlockPos> DBPostgreSQL::getBlockPos()
 }
 
 
-void DBPostgreSQL::getBlocksOnZ(std::map<int16_t, BlockList> &blocks, int16_t zPos)
+void DBPostgreSQL::getBlocksOnXZ(BlockList &blocks, int16_t xPos, int16_t zPos,
+		int16_t min_y, int16_t max_y)
 {
+	int32_t const x = htonl(xPos);
 	int32_t const z = htonl(zPos);
+	int32_t const y1 = htonl(min_y);
+	int32_t const y2 = htonl(max_y - 1);
 
-	const void *args[] = { &z };
+	const void *args[] = { &x, &z, &y1, &y2 };
-	const int argLen[] = { sizeof(z) };
+	const int argLen[] = { 4, 4, 4, 4 };
-	const int argFmt[] = { 1 };
+	const int argFmt[] = { 1, 1, 1, 1 };
 
 	PGresult *results = execPrepared(
-		"get_blocks_z", ARRLEN(args), args,
+		"get_blocks", ARRLEN(args), args,
 		argLen, argFmt, false
 	);
 
@@ -86,19 +107,18 @@ void DBPostgreSQL::getBlocksOnZ(std::map<int16_t, BlockList> &blocks, int16_t zP
 
 	for (int row = 0; row < numrows; ++row) {
 		BlockPos position;
-		position.x = pg_binary_to_int(results, row, 0);
+		position.x = xPos;
-		position.y = pg_binary_to_int(results, row, 1);
+		position.y = pg_binary_to_int(results, row, 0);
 		position.z = zPos;
-		Block const b(
+		blocks.emplace_back(
 			position,
 			ustring(
 				reinterpret_cast<unsigned char*>(
-					PQgetvalue(results, row, 2)
+					PQgetvalue(results, row, 1)
 				),
-				PQgetlength(results, row, 2)
+				PQgetlength(results, row, 1)
 			)
 		);
-		blocks[position.x].push_back(b);
 	}
 
 	PQclear(results);
@@ -138,20 +158,15 @@ PGresult *DBPostgreSQL::execPrepared(
 	const char *stmtName, const int paramsNumber,
 	const void **params,
 	const int *paramsLengths, const int *paramsFormats,
-	bool clear, bool nobinary
+	bool clear
 )
 {
 	return checkResults(PQexecPrepared(db, stmtName, paramsNumber,
 		(const char* const*) params, paramsLengths, paramsFormats,
-		nobinary ? 1 : 0), clear
+		1 /* binary output */), clear
 	);
 }
 
-int DBPostgreSQL::pg_to_int(PGresult *res, int row, int col)
-{
-	return atoi(PQgetvalue(res, row, col));
-}
-
 int DBPostgreSQL::pg_binary_to_int(PGresult *res, int row, int col)
 {
 	int32_t* raw = reinterpret_cast<int32_t*>(PQgetvalue(res, row, col));
@@ -161,8 +176,8 @@ int DBPostgreSQL::pg_binary_to_int(PGresult *res, int row, int col)
 BlockPos DBPostgreSQL::pg_to_blockpos(PGresult *res, int row, int col)
 {
 	BlockPos result;
-	result.x = pg_to_int(res, row, col);
+	result.x = pg_binary_to_int(res, row, col);
-	result.y = pg_to_int(res, row, col + 1);
+	result.y = pg_binary_to_int(res, row, col + 1);
-	result.z = pg_to_int(res, row, col + 2);
+	result.z = pg_binary_to_int(res, row, col + 2);
 	return result;
 }

db-redis.cpp

@@ -51,6 +51,9 @@ DBRedis::DBRedis(const std::string &mapdir)
 		throw std::runtime_error(err);
 	}
 
+	/* Redis is just a key-value store, so the only optimization we can do
+	 * is to cache the block positions that exist in the db.
+	 */
 	loadPosCache();
 }
 
@@ -61,9 +64,21 @@ DBRedis::~DBRedis()
 }
 
 
-std::vector<BlockPos> DBRedis::getBlockPos()
+std::vector<BlockPos> DBRedis::getBlockPos(BlockPos min, BlockPos max)
 {
-	return posCache;
+	std::vector<BlockPos> res;
+	for (const auto &it : posCache) {
+		if (it.first < min.z || it.first >= max.z)
+			continue;
+		for (auto pos2 : it.second) {
+			if (pos2.first < min.x || pos2.first >= max.x)
+				continue;
+			if (pos2.second < min.y || pos2.second >= max.y)
+				continue;
+			res.emplace_back(pos2.first, pos2.second, it.first);
+		}
+	}
+	return res;
 }
 
 
@@ -98,7 +113,8 @@ void DBRedis::loadPosCache()
 	for(size_t i = 0; i < reply->elements; i++) {
 		if(reply->element[i]->type != REDIS_REPLY_STRING)
 			REPLY_TYPE_ERR(reply->element[i], "HKEYS subreply");
-		posCache.push_back(decodeBlockPos(stoi64(reply->element[i]->str)));
+		BlockPos pos = decodeBlockPos(stoi64(reply->element[i]->str));
+		posCache[pos.z].emplace_back(pos.x, pos.y);
 	}
 
 	freeReplyObject(reply);
@@ -150,7 +166,7 @@ void DBRedis::HMGET(const std::vector<BlockPos> &positions, std::vector<ustring>
 				freeReplyObject(reply);
 				throw std::runtime_error("HMGET empty string");
 			}
-			result->push_back(ustring((const unsigned char *) subreply->str, subreply->len));
+			result->emplace_back((const unsigned char *) subreply->str, subreply->len);
 		}
 		freeReplyObject(reply);
 		remaining -= batch_size;
@@ -158,22 +174,23 @@ void DBRedis::HMGET(const std::vector<BlockPos> &positions, std::vector<ustring>
 }
 
 
-void DBRedis::getBlocksOnZ(std::map<int16_t, BlockList> &blocks, int16_t zPos)
+void DBRedis::getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
+		int16_t min_y, int16_t max_y)
 {
-	std::vector<BlockPos> z_positions;
+	auto it = posCache.find(z);
-	for (std::vector<BlockPos>::const_iterator it = posCache.begin(); it != posCache.end(); ++it) {
+	if (it == posCache.cend())
-		if (it->z != zPos) {
+		return;
-			continue;
-		}
-		z_positions.push_back(*it);
-	}
-	std::vector<ustring> z_blocks;
-	HMGET(z_positions, &z_blocks);
 
-	std::vector<ustring>::const_iterator z_block = z_blocks.begin();
+	std::vector<BlockPos> positions;
-	for (std::vector<BlockPos>::const_iterator pos = z_positions.begin();
+	for (auto pos2 : it->second) {
-			pos != z_positions.end();
+		if (pos2.first == x && pos2.second >= min_y && pos2.second < max_y)
-			++pos, ++z_block) {
+			positions.emplace_back(x, pos2.second, z);
-		blocks[pos->x].push_back(Block(*pos, *z_block));
 	}
+
+	std::vector<ustring> db_blocks;
+	HMGET(positions, &db_blocks);
+
+	auto block = db_blocks.cbegin();
+	for (auto pos = positions.cbegin(); pos != positions.cend(); ++pos, ++block)
+		blocks.emplace_back(*pos, *block);
 }

db-sqlite3.cpp

@@ -1,6 +1,8 @@
 #include <stdexcept>
 #include <unistd.h> // for usleep
 #include <iostream>
+#include <algorithm>
+#include <time.h>
 #include "db-sqlite3.h"
 #include "types.h"
 
@@ -11,7 +13,6 @@
 	}
 #define SQLOK(f) SQLRES(f, SQLITE_OK)
 
-
 DBSQLite3::DBSQLite3(const std::string &mapdir)
 {
 	int result;
@@ -27,6 +28,10 @@ DBSQLite3::DBSQLite3(const std::string &mapdir)
 	SQLOK(prepare_v2(db,
 			"SELECT pos FROM blocks",
 		-1, &stmt_get_block_pos, NULL))
+
+	SQLOK(prepare_v2(db,
+			"SELECT pos FROM blocks WHERE pos BETWEEN ? AND ?",
+		-1, &stmt_get_block_pos_z, NULL))
 }
 
 
@@ -34,59 +39,125 @@ DBSQLite3::~DBSQLite3()
 {
 	sqlite3_finalize(stmt_get_blocks_z);
 	sqlite3_finalize(stmt_get_block_pos);
+	sqlite3_finalize(stmt_get_block_pos_z);
 
 	if (sqlite3_close(db) != SQLITE_OK) {
 		std::cerr << "Error closing SQLite database." << std::endl;
 	};
 }
 
-std::vector<BlockPos> DBSQLite3::getBlockPos()
+
+inline void DBSQLite3::getPosRange(int64_t &min, int64_t &max, int16_t zPos,
+		int16_t zPos2) const
+{
+	/* The range of block positions is [-2048, 2047], which turns into [0, 4095]
+	 * when casted to unsigned. This didn't actually help me understand the
+	 * numbers below, but I wanted to write it down.
+	 */
+
+	// Magic numbers!
+	min = encodeBlockPos(BlockPos(0, -2048, zPos));
+	max = encodeBlockPos(BlockPos(0, 2048, zPos2)) - 1;
+}
+
+
+std::vector<BlockPos> DBSQLite3::getBlockPos(BlockPos min, BlockPos max)
 {
 	int result;
-	std::vector<BlockPos> positions;
+	sqlite3_stmt *stmt;
-	while ((result = sqlite3_step(stmt_get_block_pos)) != SQLITE_DONE) {
+
-		if (result == SQLITE_ROW) {
+	if(min.z <= -2048 && max.z >= 2048) {
-			int64_t posHash = sqlite3_column_int64(stmt_get_block_pos, 0);
+		stmt = stmt_get_block_pos;
-			positions.push_back(decodeBlockPos(posHash));
-		} else if (result == SQLITE_BUSY) { // Wait some time and try again
-			usleep(10000);
 	} else {
+		stmt = stmt_get_block_pos_z;
+		int64_t minPos, maxPos;
+		if (min.z < -2048)
+			min.z = -2048;
+		if (max.z > 2048)
+			max.z = 2048;
+		getPosRange(minPos, maxPos, min.z, max.z - 1);
+		SQLOK(bind_int64(stmt, 1, minPos))
+		SQLOK(bind_int64(stmt, 2, maxPos))
+	}
+
+	std::vector<BlockPos> positions;
+	while ((result = sqlite3_step(stmt)) != SQLITE_DONE) {
+		if (result == SQLITE_BUSY) { // Wait some time and try again
+			usleep(10000);
+		} else if (result != SQLITE_ROW) {
 			throw std::runtime_error(sqlite3_errmsg(db));
 		}
+
+		int64_t posHash = sqlite3_column_int64(stmt, 0);
+		BlockPos pos = decodeBlockPos(posHash);
+		if(pos.x >= min.x && pos.x < max.x && pos.y >= min.y && pos.y < max.y)
+			positions.emplace_back(pos);
 	}
-	SQLOK(reset(stmt_get_block_pos));
+	SQLOK(reset(stmt));
 	return positions;
 }
 
 
-void DBSQLite3::getBlocksOnZ(std::map<int16_t, BlockList> &blocks, int16_t zPos)
+void DBSQLite3::loadBlockCache(int16_t zPos)
 {
 	int result;
+	blockCache.clear();
 
-	// Magic numbers!
+	int64_t minPos, maxPos;
-	int64_t minPos = encodeBlockPos(BlockPos(0, -2048, zPos));
+	getPosRange(minPos, maxPos, zPos, zPos);
-	int64_t maxPos = encodeBlockPos(BlockPos(0, 2048, zPos)) - 1;
 
 	SQLOK(bind_int64(stmt_get_blocks_z, 1, minPos));
 	SQLOK(bind_int64(stmt_get_blocks_z, 2, maxPos));
 
 	while ((result = sqlite3_step(stmt_get_blocks_z)) != SQLITE_DONE) {
-		if (result == SQLITE_ROW) {
+		if (result == SQLITE_BUSY) { // Wait some time and try again
+			usleep(10000);
+		} else if (result != SQLITE_ROW) {
+			throw std::runtime_error(sqlite3_errmsg(db));
+		}
+
 		int64_t posHash = sqlite3_column_int64(stmt_get_blocks_z, 0);
+		BlockPos pos = decodeBlockPos(posHash);
 		const unsigned char *data = reinterpret_cast<const unsigned char *>(
 				sqlite3_column_blob(stmt_get_blocks_z, 1));
 		size_t size = sqlite3_column_bytes(stmt_get_blocks_z, 1);
-			Block b(decodeBlockPos(posHash), ustring(data, size));
+		blockCache[pos.x].emplace_back(pos, ustring(data, size));
-			blocks[b.first.x].push_back(b);
-		} else if (result == SQLITE_BUSY) { // Wait some time and try again
-			usleep(10000);
-		} else {
-			throw std::runtime_error(sqlite3_errmsg(db));
 	}
-	}
+	SQLOK(reset(stmt_get_blocks_z))
-	SQLOK(reset(stmt_get_blocks_z));
 }
 
-#undef SQLRES
-#undef SQLOK
 
+void DBSQLite3::getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
+		int16_t min_y, int16_t max_y)
+{
+	/* Cache the blocks on the given Z coordinate between calls, this only
+	 * works due to order in which the TileGenerator asks for blocks. */
+	if (z != blockCachedZ) {
+		loadBlockCache(z);
+		blockCachedZ = z;
+	}
+
+	auto it = blockCache.find(x);
+	if (it == blockCache.end())
+		return;
+
+	if (it->second.empty()) {
+		/* We have swapped this list before, this is not supposed to happen
+		 * because it's bad for performance. But rather than silently breaking
+		 * do the right thing and load the blocks again. */
+#ifndef NDEBUG
+		std::cout << "Warning: suboptimal access pattern for sqlite3 backend" << std::endl;
+#endif
+		loadBlockCache(z);
+	}
+	// Swap lists to avoid copying contents
+	blocks.clear();
+	std::swap(blocks, it->second);
+
+	for (auto it = blocks.begin(); it != blocks.end(); ) {
+		if (it->first.y < min_y || it->first.y >= max_y)
+			it = blocks.erase(it);
+		else
+			it++;
+	}
+}

include/TileGenerator.h

@@ -2,7 +2,8 @@
 #define TILEGENERATOR_HEADER
 
 #include <iosfwd>
-#include <list>
+#include <map>
+#include <set>
 #include <config.h>
 #include <unordered_map>
 #include <unordered_set>
@@ -64,7 +65,8 @@ public:
 	void setBgColor(const std::string &bgColor);
 	void setScaleColor(const std::string &scaleColor);
 	void setOriginColor(const std::string &originColor);
-	void setPlayerColor(const std::string &playerColor); Color parseColor(const std::string &color);
+	void setPlayerColor(const std::string &playerColor);
+	Color parseColor(const std::string &color);
 	void setDrawOrigin(bool drawOrigin);
 	void setDrawPlayers(bool drawPlayers);
 	void setDrawScale(bool drawScale);
@@ -88,7 +90,6 @@ private:
 	void loadBlocks();
 	void createImage();
 	void renderMap();
-	std::list<int16_t> getZValueList() const;
 	void renderMapBlock(const BlockDecoder &blk, const BlockPos &pos);
 	void renderMapBlockBottom(const BlockPos &pos);
 	void renderShading(int zPos);
@@ -118,19 +119,23 @@ private:
 	DB *m_db;
 	Image *m_image;
 	PixelAttributes m_blockPixelAttributes;
+	/* smallest/largest seen X or Z block coordinate */
 	int m_xMin;
 	int m_xMax;
 	int m_zMin;
 	int m_zMax;
+	/* Y limits for rendered area (node units) */
 	int m_yMin;
 	int m_yMax;
-	int m_geomX;
+	/* limits for rendered area (block units) */
-	int m_geomY;
+	int16_t m_geomX;
-	int m_geomX2;
+	int16_t m_geomY; /* Y in terms of rendered image, Z in the world */
-	int m_geomY2;
+	int16_t m_geomX2;
+	int16_t m_geomY2;
+	/* */
 	int m_mapWidth;
 	int m_mapHeight;
-	std::list<std::pair<int16_t, int16_t>> m_positions;
+	std::map<int16_t, std::set<int16_t>> m_positions; /* indexed by Z, contains X coords */
 	ColorMap m_colorMap;
 	BitmapThing m_readPixels;
 	BitmapThing m_readInfo;

include/db-leveldb.h

@@ -2,19 +2,25 @@
 #define DB_LEVELDB_HEADER
 
 #include "db.h"
+#include <unordered_map>
+#include <utility>
 #include <leveldb/db.h>
 
 class DBLevelDB : public DB {
 public:
 	DBLevelDB(const std::string &mapdir);
-	virtual std::vector<BlockPos> getBlockPos();
+	std::vector<BlockPos> getBlockPos(BlockPos min, BlockPos max) override;
-	virtual void getBlocksOnZ(std::map<int16_t, BlockList> &blocks, int16_t zPos);
+	void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
-	virtual ~DBLevelDB();
+			int16_t min_y, int16_t max_y) override;
+	~DBLevelDB() override;
+
 private:
+	using pos2d = std::pair<int16_t, int16_t>;
+
 	void loadPosCache();
 
-	std::vector<BlockPos> posCache;
+	// indexed by Z, contains all (x,y) position pairs
-
+	std::unordered_map<int16_t, std::vector<pos2d>> posCache;
 	leveldb::DB *db;
 };
 

include/db-postgresql.h

@@ -7,9 +7,11 @@
 class DBPostgreSQL : public DB {
 public:
 	DBPostgreSQL(const std::string &mapdir);
-	virtual std::vector<BlockPos> getBlockPos();
+	std::vector<BlockPos> getBlockPos(BlockPos min, BlockPos max) override;
-	virtual void getBlocksOnZ(std::map<int16_t, BlockList> &blocks, int16_t zPos);
+	void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
-	virtual ~DBPostgreSQL();
+		int16_t min_y, int16_t max_y) override;
+	~DBPostgreSQL() override;
+
 protected:
 	PGresult *checkResults(PGresult *res, bool clear = true);
 	void prepareStatement(const std::string &name, const std::string &sql);
@@ -17,11 +19,11 @@ protected:
 		const char *stmtName, const int paramsNumber,
 		const void **params,
 		const int *paramsLengths = NULL, const int *paramsFormats = NULL,
-		bool clear = true, bool nobinary = true
+		bool clear = true
 	);
-	int pg_to_int(PGresult *res, int row, int col);
 	int pg_binary_to_int(PGresult *res, int row, int col);
 	BlockPos pg_to_blockpos(PGresult *res, int row, int col);
+
 private:
 	PGconn *db;
 };

include/db-redis.h

@@ -2,21 +2,27 @@
 #define DB_REDIS_HEADER
 
 #include "db.h"
+#include <unordered_map>
+#include <utility>
 #include <hiredis/hiredis.h>
 
 class DBRedis : public DB {
 public:
 	DBRedis(const std::string &mapdir);
-	virtual std::vector<BlockPos> getBlockPos();
+	std::vector<BlockPos> getBlockPos(BlockPos min, BlockPos max) override;
-	virtual void getBlocksOnZ(std::map<int16_t, BlockList> &blocks, int16_t zPos);
+	void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
-	virtual ~DBRedis();
+		int16_t min_y, int16_t max_y) override;
+	~DBRedis() override;
+
 private:
+	using pos2d = std::pair<int16_t, int16_t>;
 	static std::string replyTypeStr(int type);
 
 	void loadPosCache();
 	void HMGET(const std::vector<BlockPos> &positions, std::vector<ustring> *result);
 
-	std::vector<BlockPos> posCache;
+	// indexed by Z, contains all (x,y) position pairs
+	std::unordered_map<int16_t, std::vector<pos2d>> posCache;
 
 	redisContext *ctx;
 	std::string hash;

include/db-sqlite3.h

@@ -2,19 +2,30 @@
 #define _DB_SQLITE3_H
 
 #include "db.h"
+#include <unordered_map>
 #include <sqlite3.h>
 
 class DBSQLite3 : public DB {
 public:
 	DBSQLite3(const std::string &mapdir);
-	virtual std::vector<BlockPos> getBlockPos();
+	std::vector<BlockPos> getBlockPos(BlockPos min, BlockPos max) override;
-	virtual void getBlocksOnZ(std::map<int16_t, BlockList> &blocks, int16_t zPos);
+	void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
-	virtual ~DBSQLite3();
+			int16_t min_y, int16_t max_y) override;
+	~DBSQLite3() override;
+
 private:
+	inline void getPosRange(int64_t &min, int64_t &max, int16_t zPos,
+			int16_t zPos2) const;
+	void loadBlockCache(int16_t zPos);
+
 	sqlite3 *db;
 
 	sqlite3_stmt *stmt_get_block_pos;
+	sqlite3_stmt *stmt_get_block_pos_z;
 	sqlite3_stmt *stmt_get_blocks_z;
+
+	int16_t blockCachedZ = -10000;
+	std::unordered_map<int16_t, BlockList> blockCache; // indexed by X
 };
 
 #endif // _DB_SQLITE3_H

include/db.h

@@ -5,7 +5,6 @@
 #include <map>
 #include <list>
 #include <vector>
-#include <string>
 #include <utility>
 #include "types.h"
 
@@ -17,6 +16,8 @@ struct BlockPos {
 
 	BlockPos() : x(0), y(0), z(0) {}
 	BlockPos(int16_t x, int16_t y, int16_t z) : x(x), y(y), z(z) {}
+
+	// Implements the inverse ordering so that (2,2,2) < (1,1,1)
 	bool operator < (const BlockPos &p) const
 	{
 		if (z > p.z)
@@ -42,13 +43,21 @@ typedef std::list<Block> BlockList;
 
 class DB {
 protected:
+	// Helpers that implement the hashed positions used by most backends
 	inline int64_t  encodeBlockPos(const BlockPos pos) const;
 	inline BlockPos decodeBlockPos(int64_t hash) const;
 
 public:
-	virtual std::vector<BlockPos> getBlockPos() = 0;
+	/* Return all block positions inside the range given by min and max,
-	virtual void getBlocksOnZ(std::map<int16_t, BlockList> &blocks, int16_t zPos) = 0;
+	 * so that min.x <= x < max.x, ...
-	virtual ~DB() {};
+	 */
+	virtual std::vector<BlockPos> getBlockPos(BlockPos min, BlockPos max) = 0;
+	/* Return all blocks in column given by x and z
+	 * and inside the given Y range (min_y <= y < max_y)
+	 */
+	virtual void getBlocksOnXZ(BlockList &blocks, int16_t x, int16_t z,
+			int16_t min_y, int16_t max_y) = 0;
+	virtual ~DB() {}
 };