Use a database for mod storage (#11763)

doc/minetest.6

@@ -112,6 +112,10 @@ leveldb, and files.
 Migrate from current players backend to another. Possible values are sqlite3,
 leveldb, postgresql, dummy, and files.
 .TP
+.B \-\-migrate-mod-storage <value>
+Migrate from current mod storage backend to another. Possible values are
+sqlite3, dummy, and files.
+.TP
 .B \-\-terminal
 Display an interactive terminal over ncurses during execution.
 

src/client/client.cpp

@@ -128,6 +128,11 @@ Client::Client(
 	// Add local player
 	m_env.setLocalPlayer(new LocalPlayer(this, playername));
 
+	// Make the mod storage database and begin the save for later
+	m_mod_storage_database =
+		new ModMetadataDatabaseSQLite3(porting::path_user + DIR_DELIM + "client");
+	m_mod_storage_database->beginSave();
+
 	if (g_settings->getBool("enable_minimap")) {
 		m_minimap = new Minimap(this);
 	}
@@ -305,6 +310,11 @@ Client::~Client()
 	m_minimap = nullptr;
 
 	delete m_media_downloader;
+
+	// Write the changes and delete
+	if (m_mod_storage_database)
+		m_mod_storage_database->endSave();
+	delete m_mod_storage_database;
 }
 
 void Client::connect(Address address, bool is_local_server)
@@ -641,19 +651,12 @@ void Client::step(float dtime)
 		}
 	}
 
+	// Write changes to the mod storage
 	m_mod_storage_save_timer -= dtime;
 	if (m_mod_storage_save_timer <= 0.0f) {
 		m_mod_storage_save_timer = g_settings->getFloat("server_map_save_interval");
-		int n = 0;
-		for (std::unordered_map<std::string, ModMetadata *>::const_iterator
-				it = m_mod_storages.begin(); it != m_mod_storages.end(); ++it) {
-			if (it->second->isModified()) {
-				it->second->save(getModStoragePath());
-				n++;
-			}
-		}
-		if (n > 0)
-			infostream << "Saved " << n << " modified mod storages." << std::endl;
+		m_mod_storage_database->endSave();
+		m_mod_storage_database->beginSave();
 	}
 
 	// Write server map
@@ -1960,16 +1963,8 @@ void Client::unregisterModStorage(const std::string &name)
 {
 	std::unordered_map<std::string, ModMetadata *>::const_iterator it =
 		m_mod_storages.find(name);
-	if (it != m_mod_storages.end()) {
-		// Save unconditionaly on unregistration
-		it->second->save(getModStoragePath());
+	if (it != m_mod_storages.end())
 		m_mod_storages.erase(name);
-	}
-}
-
-std::string Client::getModStoragePath() const
-{
-	return porting::path_user + DIR_DELIM + "client" + DIR_DELIM + "mod_storage";
 }
 
 /*

src/client/client.h

@@ -380,8 +380,8 @@ public:
 	{ return checkPrivilege(priv); }
 	virtual scene::IAnimatedMesh* getMesh(const std::string &filename, bool cache = false);
 	const std::string* getModFile(std::string filename);
+	ModMetadataDatabase *getModStorageDatabase() override { return m_mod_storage_database; }
 
-	std::string getModStoragePath() const override;
 	bool registerModStorage(ModMetadata *meta) override;
 	void unregisterModStorage(const std::string &name) override;
 
@@ -590,6 +590,7 @@ private:
 	// Client modding
 	ClientScripting *m_script = nullptr;
 	std::unordered_map<std::string, ModMetadata *> m_mod_storages;
+	ModMetadataDatabase *m_mod_storage_database = nullptr;
 	float m_mod_storage_save_timer = 10.0f;
 	std::vector<ModSpec> m_mods;
 	StringMap m_mod_vfs;

src/content/mods.cpp

@@ -22,6 +22,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include <json/json.h>
 #include <algorithm>
 #include "content/mods.h"
+#include "database/database.h"
 #include "filesys.h"
 #include "log.h"
 #include "content/subgames.h"
@@ -422,83 +423,29 @@ ClientModConfiguration::ClientModConfiguration(const std::string &path) :
 }
 #endif
 
-ModMetadata::ModMetadata(const std::string &mod_name) : m_mod_name(mod_name)
+ModMetadata::ModMetadata(const std::string &mod_name, ModMetadataDatabase *database):
+	m_mod_name(mod_name), m_database(database)
 {
+	m_database->getModEntries(m_mod_name, &m_stringvars);
 }
 
 void ModMetadata::clear()
 {
+	for (const auto &pair : m_stringvars) {
+		m_database->removeModEntry(m_mod_name, pair.first);
+	}
 	Metadata::clear();
-	m_modified = true;
-}
-
-bool ModMetadata::save(const std::string &root_path)
-{
-	Json::Value json;
-	for (StringMap::const_iterator it = m_stringvars.begin();
-			it != m_stringvars.end(); ++it) {
-		json[it->first] = it->second;
-	}
-
-	if (!fs::PathExists(root_path)) {
-		if (!fs::CreateAllDirs(root_path)) {
-			errorstream << "ModMetadata[" << m_mod_name
-				    << "]: Unable to save. '" << root_path
-				    << "' tree cannot be created." << std::endl;
-			return false;
-		}
-	} else if (!fs::IsDir(root_path)) {
-		errorstream << "ModMetadata[" << m_mod_name << "]: Unable to save. '"
-			    << root_path << "' is not a directory." << std::endl;
-		return false;
-	}
-
-	bool w_ok = fs::safeWriteToFile(
-			root_path + DIR_DELIM + m_mod_name, fastWriteJson(json));
-
-	if (w_ok) {
-		m_modified = false;
-	} else {
-		errorstream << "ModMetadata[" << m_mod_name << "]: failed write file."
-			    << std::endl;
-	}
-	return w_ok;
-}
-
-bool ModMetadata::load(const std::string &root_path)
-{
-	m_stringvars.clear();
-
-	std::ifstream is((root_path + DIR_DELIM + m_mod_name).c_str(),
-			std::ios_base::binary);
-	if (!is.good()) {
-		return false;
-	}
-
-	Json::Value root;
-	Json::CharReaderBuilder builder;
-	builder.settings_["collectComments"] = false;
-	std::string errs;
-
-	if (!Json::parseFromStream(builder, is, &root, &errs)) {
-		errorstream << "ModMetadata[" << m_mod_name
-			    << "]: failed read data "
-			       "(Json decoding failure). Message: "
-			    << errs << std::endl;
-		return false;
-	}
-
-	const Json::Value::Members attr_list = root.getMemberNames();
-	for (const auto &it : attr_list) {
-		Json::Value attr_value = root[it];
-		m_stringvars[it] = attr_value.asString();
-	}
-
-	return true;
 }
 
 bool ModMetadata::setString(const std::string &name, const std::string &var)
 {
-	m_modified = Metadata::setString(name, var);
-	return m_modified;
+	if (Metadata::setString(name, var)) {
+		if (var.empty()) {
+			m_database->removeModEntry(m_mod_name, name);
+		} else {
+			m_database->setModEntry(m_mod_name, name, var);
+		}
+		return true;
+	}
+	return false;
 }

src/content/mods.h

@@ -31,6 +31,8 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "config.h"
 #include "metadata.h"
 
+class ModMetadataDatabase;
+
 #define MODNAME_ALLOWED_CHARS "abcdefghijklmnopqrstuvwxyz0123456789_"
 
 struct ModSpec
@@ -149,20 +151,16 @@ class ModMetadata : public Metadata
 {
 public:
 	ModMetadata() = delete;
-	ModMetadata(const std::string &mod_name);
+	ModMetadata(const std::string &mod_name, ModMetadataDatabase *database);
 	~ModMetadata() = default;
 
 	virtual void clear();
 
-	bool save(const std::string &root_path);
-	bool load(const std::string &root_path);
-
-	bool isModified() const { return m_modified; }
 	const std::string &getModName() const { return m_mod_name; }
 
 	virtual bool setString(const std::string &name, const std::string &var);
 
 private:
 	std::string m_mod_name;
-	bool m_modified = false;
+	ModMetadataDatabase *m_database;
 };

src/content/subgames.cpp

@@ -358,6 +358,7 @@ void loadGameConfAndInitWorld(const std::string &path, const std::string &name,
 		conf.set("backend", "sqlite3");
 		conf.set("player_backend", "sqlite3");
 		conf.set("auth_backend", "sqlite3");
+		conf.set("mod_storage_backend", "sqlite3");
 		conf.setBool("creative_mode", g_settings->getBool("creative_mode"));
 		conf.setBool("enable_damage", g_settings->getBool("enable_damage"));
 

src/database/database-dummy.cpp

@@ -80,3 +80,41 @@ void Database_Dummy::listPlayers(std::vector<std::string> &res)
 		res.emplace_back(player);
 	}
 }
+
+bool Database_Dummy::getModEntries(const std::string &modname, StringMap *storage)
+{
+	const auto mod_pair = m_mod_meta_database.find(modname);
+	if (mod_pair != m_mod_meta_database.cend()) {
+		for (const auto &pair : mod_pair->second) {
+			(*storage)[pair.first] = pair.second;
+		}
+	}
+	return true;
+}
+
+bool Database_Dummy::setModEntry(const std::string &modname,
+	const std::string &key, const std::string &value)
+{
+	auto mod_pair = m_mod_meta_database.find(modname);
+	if (mod_pair == m_mod_meta_database.end()) {
+		m_mod_meta_database[modname] = StringMap({{key, value}});
+	} else {
+		mod_pair->second[key] = value;
+	}
+	return true;
+}
+
+bool Database_Dummy::removeModEntry(const std::string &modname, const std::string &key)
+{
+	auto mod_pair = m_mod_meta_database.find(modname);
+	if (mod_pair != m_mod_meta_database.end())
+		return mod_pair->second.erase(key) > 0;
+	return false;
+}
+
+void Database_Dummy::listMods(std::vector<std::string> *res)
+{
+	for (const auto &pair : m_mod_meta_database) {
+		res->push_back(pair.first);
+	}
+}

src/database/database-dummy.h

@@ -24,7 +24,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "database.h"
 #include "irrlichttypes.h"
 
-class Database_Dummy : public MapDatabase, public PlayerDatabase
+class Database_Dummy : public MapDatabase, public PlayerDatabase, public ModMetadataDatabase
 {
 public:
 	bool saveBlock(const v3s16 &pos, const std::string &data);
@@ -37,10 +37,17 @@ public:
 	bool removePlayer(const std::string &name);
 	void listPlayers(std::vector<std::string> &res);
 
+	bool getModEntries(const std::string &modname, StringMap *storage);
+	bool setModEntry(const std::string &modname,
+			const std::string &key, const std::string &value);
+	bool removeModEntry(const std::string &modname, const std::string &key);
+	void listMods(std::vector<std::string> *res);
+
 	void beginSave() {}
 	void endSave() {}
 
 private:
 	std::map<s64, std::string> m_database;
 	std::set<std::string> m_player_database;
+	std::unordered_map<std::string, StringMap> m_mod_meta_database;
 };

src/database/database-files.cpp

@@ -18,7 +18,6 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 */
 
 #include <cassert>
-#include <json/json.h>
 #include "convert_json.h"
 #include "database-files.h"
 #include "remoteplayer.h"
@@ -376,3 +375,138 @@ bool AuthDatabaseFiles::writeAuthFile()
 	}
 	return true;
 }
+
+ModMetadataDatabaseFiles::ModMetadataDatabaseFiles(const std::string &savedir):
+	m_storage_dir(savedir + DIR_DELIM + "mod_storage")
+{
+}
+
+bool ModMetadataDatabaseFiles::getModEntries(const std::string &modname, StringMap *storage)
+{
+	Json::Value *meta = getOrCreateJson(modname);
+	if (!meta)
+		return false;
+
+	const Json::Value::Members attr_list = meta->getMemberNames();
+	for (const auto &it : attr_list) {
+		Json::Value attr_value = (*meta)[it];
+		(*storage)[it] = attr_value.asString();
+	}
+
+	return true;
+}
+
+bool ModMetadataDatabaseFiles::setModEntry(const std::string &modname,
+	const std::string &key, const std::string &value)
+{
+	Json::Value *meta = getOrCreateJson(modname);
+	if (!meta)
+		return false;
+
+	(*meta)[key] = Json::Value(value);
+	m_modified.insert(modname);
+
+	return true;
+}
+
+bool ModMetadataDatabaseFiles::removeModEntry(const std::string &modname,
+		const std::string &key)
+{
+	Json::Value *meta = getOrCreateJson(modname);
+	if (!meta)
+		return false;
+
+	Json::Value removed;
+	if (meta->removeMember(key, &removed)) {
+		m_modified.insert(modname);
+		return true;
+	}
+	return false;
+}
+
+void ModMetadataDatabaseFiles::beginSave()
+{
+}
+
+void ModMetadataDatabaseFiles::endSave()
+{
+	if (!fs::CreateAllDirs(m_storage_dir)) {
+		errorstream << "ModMetadataDatabaseFiles: Unable to save. '" << m_storage_dir
+				<< "' tree cannot be created." << std::endl;
+		return;
+	}
+
+	for (auto it = m_modified.begin(); it != m_modified.end();) {
+		const std::string &modname = *it;
+
+		if (!fs::PathExists(m_storage_dir)) {
+			if (!fs::CreateAllDirs(m_storage_dir)) {
+				errorstream << "ModMetadataDatabaseFiles[" << modname
+						<< "]: Unable to save. '" << m_storage_dir
+						<< "' tree cannot be created." << std::endl;
+				++it;
+				continue;
+			}
+		} else if (!fs::IsDir(m_storage_dir)) {
+			errorstream << "ModMetadataDatabaseFiles[" << modname << "]: Unable to save. '"
+					<< m_storage_dir << "' is not a directory." << std::endl;
+			++it;
+			continue;
+		}
+
+		const Json::Value &json = m_mod_meta[modname];
+
+		if (!fs::safeWriteToFile(m_storage_dir + DIR_DELIM + modname, fastWriteJson(json))) {
+			errorstream << "ModMetadataDatabaseFiles[" << modname
+					<< "]: failed write file." << std::endl;
+			++it;
+			continue;
+		}
+
+		it = m_modified.erase(it);
+	}
+}
+
+void ModMetadataDatabaseFiles::listMods(std::vector<std::string> *res)
+{
+	// List in-memory metadata first.
+	for (const auto &pair : m_mod_meta) {
+		res->push_back(pair.first);
+	}
+
+	// List other metadata present in the filesystem.
+	for (const auto &entry : fs::GetDirListing(m_storage_dir)) {
+		if (!entry.dir && m_mod_meta.count(entry.name) == 0)
+			res->push_back(entry.name);
+	}
+}
+
+Json::Value *ModMetadataDatabaseFiles::getOrCreateJson(const std::string &modname)
+{
+	auto found = m_mod_meta.find(modname);
+	if (found == m_mod_meta.end()) {
+		fs::CreateAllDirs(m_storage_dir);
+
+		Json::Value meta(Json::objectValue);
+
+		std::string path = m_storage_dir + DIR_DELIM + modname;
+		if (fs::PathExists(path)) {
+			std::ifstream is(path.c_str(), std::ios_base::binary);
+
+			Json::CharReaderBuilder builder;
+			builder.settings_["collectComments"] = false;
+			std::string errs;
+
+			if (!Json::parseFromStream(builder, is, &meta, &errs)) {
+				errorstream << "ModMetadataDatabaseFiles[" << modname
+						<< "]: failed read data (Json decoding failure). Message: "
+						<< errs << std::endl;
+				return nullptr;
+			}
+		}
+
+		return &(m_mod_meta[modname] = meta);
+	} else {
+		return &found->second;
+	}
+}

src/database/database-files.h

@@ -25,6 +25,8 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 
 #include "database.h"
 #include <unordered_map>
+#include <unordered_set>
+#include <json/json.h>
 
 class PlayerDatabaseFiles : public PlayerDatabase
 {
@@ -69,3 +71,27 @@ private:
 	bool readAuthFile();
 	bool writeAuthFile();
 };
+
+class ModMetadataDatabaseFiles : public ModMetadataDatabase
+{
+public:
+	ModMetadataDatabaseFiles(const std::string &savedir);
+	virtual ~ModMetadataDatabaseFiles() = default;
+
+	virtual bool getModEntries(const std::string &modname, StringMap *storage);
+	virtual bool setModEntry(const std::string &modname,
+		const std::string &key, const std::string &value);
+	virtual bool removeModEntry(const std::string &modname, const std::string &key);
+	virtual void listMods(std::vector<std::string> *res);
+
+	virtual void beginSave();
+	virtual void endSave();
+
+private:
+	Json::Value *getOrCreateJson(const std::string &modname);
+	bool writeJson(const std::string &modname, const Json::Value &json);
+
+	std::string m_storage_dir;
+	std::unordered_map<std::string, Json::Value> m_mod_meta;
+	std::unordered_set<std::string> m_modified;
+};

src/database/database-sqlite3.cpp

@@ -779,3 +779,108 @@ void AuthDatabaseSQLite3::writePrivileges(const AuthEntry &authEntry)
 		sqlite3_reset(m_stmt_write_privs);
 	}
 }
+
+ModMetadataDatabaseSQLite3::ModMetadataDatabaseSQLite3(const std::string &savedir):
+	Database_SQLite3(savedir, "mod_storage"), ModMetadataDatabase()
+{
+}
+
+ModMetadataDatabaseSQLite3::~ModMetadataDatabaseSQLite3()
+{
+	FINALIZE_STATEMENT(m_stmt_remove)
+	FINALIZE_STATEMENT(m_stmt_set)
+	FINALIZE_STATEMENT(m_stmt_get)
+}
+
+void ModMetadataDatabaseSQLite3::createDatabase()
+{
+	assert(m_database); // Pre-condition
+
+	SQLOK(sqlite3_exec(m_database,
+		"CREATE TABLE IF NOT EXISTS `entries` (\n"
+			"	`modname` TEXT NOT NULL,\n"
+			"	`key` BLOB NOT NULL,\n"
+			"	`value` BLOB NOT NULL,\n"
+			"	PRIMARY KEY (`modname`, `key`)\n"
+			");\n",
+		NULL, NULL, NULL),
+		"Failed to create database table");
+}
+
+void ModMetadataDatabaseSQLite3::initStatements()
+{
+	PREPARE_STATEMENT(get, "SELECT `key`, `value` FROM `entries` WHERE `modname` = ?");
+	PREPARE_STATEMENT(set,
+		"REPLACE INTO `entries` (`modname`, `key`, `value`) VALUES (?, ?, ?)");
+	PREPARE_STATEMENT(remove, "DELETE FROM `entries` WHERE `modname` = ? AND `key` = ?");
+}
+
+bool ModMetadataDatabaseSQLite3::getModEntries(const std::string &modname, StringMap *storage)
+{
+	verifyDatabase();
+
+	str_to_sqlite(m_stmt_get, 1, modname);
+	while (sqlite3_step(m_stmt_get) == SQLITE_ROW) {
+		const char *key_data = (const char *) sqlite3_column_blob(m_stmt_get, 0);
+		size_t key_len = sqlite3_column_bytes(m_stmt_get, 0);
+		const char *value_data = (const char *) sqlite3_column_blob(m_stmt_get, 1);
+		size_t value_len = sqlite3_column_bytes(m_stmt_get, 1);
+		(*storage)[std::string(key_data, key_len)] = std::string(value_data, value_len);
+	}
+	sqlite3_vrfy(sqlite3_errcode(m_database), SQLITE_DONE);
+
+	sqlite3_reset(m_stmt_get);
+
+	return true;
+}
+
+bool ModMetadataDatabaseSQLite3::setModEntry(const std::string &modname,
+	const std::string &key, const std::string &value)
+{
+	verifyDatabase();
+
+	str_to_sqlite(m_stmt_set, 1, modname);
+	SQLOK(sqlite3_bind_blob(m_stmt_set, 2, key.data(), key.size(), NULL),
+		"Internal error: failed to bind query at " __FILE__ ":" TOSTRING(__LINE__));
+	SQLOK(sqlite3_bind_blob(m_stmt_set, 3, value.data(), value.size(), NULL),
+		"Internal error: failed to bind query at " __FILE__ ":" TOSTRING(__LINE__));
+	SQLRES(sqlite3_step(m_stmt_set), SQLITE_DONE, "Failed to set mod entry")
+
+	sqlite3_reset(m_stmt_set);
+
+	return true;
+}
+
+bool ModMetadataDatabaseSQLite3::removeModEntry(const std::string &modname,
+		const std::string &key)
+{
+	verifyDatabase();
+
+	str_to_sqlite(m_stmt_remove, 1, modname);
+	SQLOK(sqlite3_bind_blob(m_stmt_remove, 2, key.data(), key.size(), NULL),
+		"Internal error: failed to bind query at " __FILE__ ":" TOSTRING(__LINE__));
+	sqlite3_vrfy(sqlite3_step(m_stmt_remove), SQLITE_DONE);
+	int changes = sqlite3_changes(m_database);
+
+	sqlite3_reset(m_stmt_remove);
+
+	return changes > 0;
+}
+
+void ModMetadataDatabaseSQLite3::listMods(std::vector<std::string> *res)
+{
+	verifyDatabase();
+
+	char *errmsg;
+	int status = sqlite3_exec(m_database,
+		"SELECT `modname` FROM `entries` GROUP BY `modname`;",
+		[](void *res_vp, int n_col, char **cols, char **col_names) -> int {
+			((decltype(res)) res_vp)->emplace_back(cols[0]);
+			return 0;
+		}, (void *) res, &errmsg);
+	if (status != SQLITE_OK) {
+		DatabaseException e(std::string("Error trying to list mods with metadata: ") + errmsg);
+		sqlite3_free(errmsg);
+		throw e;
+	}
+}

src/database/database-sqlite3.h

@@ -232,3 +232,28 @@ private:
 	sqlite3_stmt *m_stmt_delete_privs = nullptr;
 	sqlite3_stmt *m_stmt_last_insert_rowid = nullptr;
 };
+
+class ModMetadataDatabaseSQLite3 : private Database_SQLite3, public ModMetadataDatabase
+{
+public:
+	ModMetadataDatabaseSQLite3(const std::string &savedir);
+	virtual ~ModMetadataDatabaseSQLite3();
+
+	virtual bool getModEntries(const std::string &modname, StringMap *storage);
+	virtual bool setModEntry(const std::string &modname,
+		const std::string &key, const std::string &value);
+	virtual bool removeModEntry(const std::string &modname, const std::string &key);
+	virtual void listMods(std::vector<std::string> *res);
+
+	virtual void beginSave() { Database_SQLite3::beginSave(); }
+	virtual void endSave() { Database_SQLite3::endSave(); }
+
+protected:
+	virtual void createDatabase();
+	virtual void initStatements();
+
+private:
+	sqlite3_stmt *m_stmt_get = nullptr;
+	sqlite3_stmt *m_stmt_set = nullptr;
+	sqlite3_stmt *m_stmt_remove = nullptr;
+};

src/database/database.h

@@ -25,6 +25,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "irr_v3d.h"
 #include "irrlichttypes.h"
 #include "util/basic_macros.h"
+#include "util/string.h"
 
 class Database
 {
@@ -84,3 +85,15 @@ public:
 	virtual void listNames(std::vector<std::string> &res) = 0;
 	virtual void reload() = 0;
 };
+
+class ModMetadataDatabase : public Database
+{
+public:
+	virtual ~ModMetadataDatabase() = default;
+
+	virtual bool getModEntries(const std::string &modname, StringMap *storage) = 0;
+	virtual bool setModEntry(const std::string &modname,
+		const std::string &key, const std::string &value) = 0;
+	virtual bool removeModEntry(const std::string &modname, const std::string &key) = 0;
+	virtual void listMods(std::vector<std::string> *res) = 0;
+};

src/gamedef.h

@@ -33,6 +33,7 @@ class EmergeManager;
 class Camera;
 class ModChannel;
 class ModMetadata;
+class ModMetadataDatabase;
 
 namespace irr { namespace scene {
 	class IAnimatedMesh;
@@ -70,9 +71,9 @@ public:
 	virtual const std::vector<ModSpec> &getMods() const = 0;
 	virtual const ModSpec* getModSpec(const std::string &modname) const = 0;
 	virtual std::string getWorldPath() const { return ""; }
-	virtual std::string getModStoragePath() const = 0;
 	virtual bool registerModStorage(ModMetadata *storage) = 0;
 	virtual void unregisterModStorage(const std::string &name) = 0;
+	virtual ModMetadataDatabase *getModStorageDatabase() = 0;
 
 	virtual bool joinModChannel(const std::string &channel) = 0;
 	virtual bool leaveModChannel(const std::string &channel) = 0;

src/main.cpp

@@ -299,6 +299,8 @@ static void set_allowed_options(OptionList *allowed_options)
 		_("Migrate from current players backend to another (Only works when using minetestserver or with --server)"))));
 	allowed_options->insert(std::make_pair("migrate-auth", ValueSpec(VALUETYPE_STRING,
 		_("Migrate from current auth backend to another (Only works when using minetestserver or with --server)"))));
+	allowed_options->insert(std::make_pair("migrate-mod-storage", ValueSpec(VALUETYPE_STRING,
+		_("Migrate from current mod storage backend to another (Only works when using minetestserver or with --server)"))));
 	allowed_options->insert(std::make_pair("terminal", ValueSpec(VALUETYPE_FLAG,
 			_("Feature an interactive terminal (Only works when using minetestserver or with --server)"))));
 	allowed_options->insert(std::make_pair("recompress", ValueSpec(VALUETYPE_FLAG,
@@ -886,6 +888,9 @@ static bool run_dedicated_server(const GameParams &game_params, const Settings &
 	if (cmd_args.exists("migrate-auth"))
 		return ServerEnvironment::migrateAuthDatabase(game_params, cmd_args);
 
+	if (cmd_args.exists("migrate-mod-storage"))
+		return Server::migrateModStorageDatabase(game_params, cmd_args);
+
 	if (cmd_args.getFlag("recompress"))
 		return recompress_map_database(game_params, cmd_args, bind_addr);
 

src/script/lua_api/l_storage.cpp

@@ -32,19 +32,23 @@ int ModApiStorage::l_get_mod_storage(lua_State *L)
 
 	std::string mod_name = readParam<std::string>(L, -1);
 
-	ModMetadata *store = new ModMetadata(mod_name);
+	ModMetadata *store = nullptr;
+
 	if (IGameDef *gamedef = getGameDef(L)) {
-		store->load(gamedef->getModStoragePath());
-		gamedef->registerModStorage(store);
+		store = new ModMetadata(mod_name, gamedef->getModStorageDatabase());
+		if (gamedef->registerModStorage(store)) {
+			StorageRef::create(L, store);
+			int object = lua_gettop(L);
+			lua_pushvalue(L, object);
+			return 1;
+		}
 	} else {
-		delete store;
 		assert(false); // this should not happen
 	}
 
-	StorageRef::create(L, store);
-	int object = lua_gettop(L);
+	delete store;
 
-	lua_pushvalue(L, object);
+	lua_pushnil(L);
 	return 1;
 }
 

src/server.cpp

@@ -66,6 +66,10 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "server/player_sao.h"
 #include "server/serverinventorymgr.h"
 #include "translation.h"
+#include "database/database-sqlite3.h"
+#include "database/database-files.h"
+#include "database/database-dummy.h"
+#include "gameparams.h"
 
 class ClientNotFoundException : public BaseException
 {
@@ -344,10 +348,15 @@ Server::~Server()
 		delete m_thread;
 	}
 
+	// Write any changes before deletion.
+	if (m_mod_storage_database)
+		m_mod_storage_database->endSave();
+
 	// Delete things in the reverse order of creation
 	delete m_emerge;
 	delete m_env;
 	delete m_rollback;
+	delete m_mod_storage_database;
 	delete m_banmanager;
 	delete m_itemdef;
 	delete m_nodedef;
@@ -393,6 +402,10 @@ void Server::init()
 	std::string ban_path = m_path_world + DIR_DELIM "ipban.txt";
 	m_banmanager = new BanManager(ban_path);
 
+	// Create mod storage database and begin a save for later
+	m_mod_storage_database = openModStorageDatabase(m_path_world);
+	m_mod_storage_database->beginSave();
+
 	m_modmgr = std::unique_ptr<ServerModManager>(new ServerModManager(m_path_world));
 	std::vector<ModSpec> unsatisfied_mods = m_modmgr->getUnsatisfiedMods();
 	// complain about mods with unsatisfied dependencies
@@ -733,20 +746,12 @@ void Server::AsyncRunStep(bool initial_step)
 		}
 		m_clients.unlock();
 
-		// Save mod storages if modified
+		// Write changes to the mod storage
 		m_mod_storage_save_timer -= dtime;
 		if (m_mod_storage_save_timer <= 0.0f) {
 			m_mod_storage_save_timer = g_settings->getFloat("server_map_save_interval");
-			int n = 0;
-			for (std::unordered_map<std::string, ModMetadata *>::const_iterator
-				it = m_mod_storages.begin(); it != m_mod_storages.end(); ++it) {
-				if (it->second->isModified()) {
-					it->second->save(getModStoragePath());
-					n++;
-				}
-			}
-			if (n > 0)
-				infostream << "Saved " << n << " modified mod storages." << std::endl;
+			m_mod_storage_database->endSave();
+			m_mod_storage_database->beginSave();
 		}
 	}
 
@@ -3689,11 +3694,6 @@ std::string Server::getBuiltinLuaPath()
 	return porting::path_share + DIR_DELIM + "builtin";
 }
 
-std::string Server::getModStoragePath() const
-{
-	return m_path_world + DIR_DELIM + "mod_storage";
-}
-
 v3f Server::findSpawnPos()
 {
 	ServerMap &map = m_env->getServerMap();
@@ -3857,11 +3857,8 @@ bool Server::registerModStorage(ModMetadata *storage)
 void Server::unregisterModStorage(const std::string &name)
 {
 	std::unordered_map<std::string, ModMetadata *>::const_iterator it = m_mod_storages.find(name);
-	if (it != m_mod_storages.end()) {
-		// Save unconditionaly on unregistration
-		it->second->save(getModStoragePath());
+	if (it != m_mod_storages.end())
 		m_mod_storages.erase(name);
-	}
 }
 
 void dedicated_server_loop(Server &server, bool &kill)
@@ -3999,3 +3996,106 @@ Translations *Server::getTranslationLanguage(const std::string &lang_code)
 
 	return translations;
 }
+
+ModMetadataDatabase *Server::openModStorageDatabase(const std::string &world_path)
+{
+	std::string world_mt_path = world_path + DIR_DELIM + "world.mt";
+	Settings world_mt;
+	if (!world_mt.readConfigFile(world_mt_path.c_str()))
+		throw BaseException("Cannot read world.mt!");
+
+	std::string backend = world_mt.exists("mod_storage_backend") ?
+		world_mt.get("mod_storage_backend") : "files";
+	if (backend == "files")
+		warningstream << "/!\\ You are using the old mod storage files backend. "
+			<< "This backend is deprecated and may be removed in a future release /!\\"
+			<< std::endl << "Switching to SQLite3 is advised, "
+			<< "please read http://wiki.minetest.net/Database_backends." << std::endl;
+
+	return openModStorageDatabase(backend, world_path, world_mt);
+}
+
+ModMetadataDatabase *Server::openModStorageDatabase(const std::string &backend,
+		const std::string &world_path, const Settings &world_mt)
+{
+	if (backend == "sqlite3")
+		return new ModMetadataDatabaseSQLite3(world_path);
+
+	if (backend == "files")
+		return new ModMetadataDatabaseFiles(world_path);
+
+	if (backend == "dummy")
+		return new Database_Dummy();
+
+	throw BaseException("Mod storage database backend " + backend + " not supported");
+}
+
+bool Server::migrateModStorageDatabase(const GameParams &game_params, const Settings &cmd_args)
+{
+	std::string migrate_to = cmd_args.get("migrate-mod-storage");
+	Settings world_mt;
+	std::string world_mt_path = game_params.world_path + DIR_DELIM + "world.mt";
+	if (!world_mt.readConfigFile(world_mt_path.c_str())) {
+		errorstream << "Cannot read world.mt!" << std::endl;
+		return false;
+	}
+
+	std::string backend = world_mt.exists("mod_storage_backend") ?
+		world_mt.get("mod_storage_backend") : "files";
+	if (backend == migrate_to) {
+		errorstream << "Cannot migrate: new backend is same"
+			<< " as the old one" << std::endl;
+		return false;
+	}
+
+	ModMetadataDatabase *srcdb = nullptr;
+	ModMetadataDatabase *dstdb = nullptr;
+
+	bool succeeded = false;
+
+	try {
+		srcdb = Server::openModStorageDatabase(backend, game_params.world_path, world_mt);
+		dstdb = Server::openModStorageDatabase(migrate_to, game_params.world_path, world_mt);
+
+		dstdb->beginSave();
+
+		std::vector<std::string> mod_list;
+		srcdb->listMods(&mod_list);
+		for (const std::string &modname : mod_list) {
+			StringMap meta;
+			srcdb->getModEntries(modname, &meta);
+			for (const auto &pair : meta) {
+				dstdb->setModEntry(modname, pair.first, pair.second);
+			}
+		}
+
+		dstdb->endSave();
+
+		succeeded = true;
+
+		actionstream << "Successfully migrated the metadata of "
+			<< mod_list.size() << " mods" << std::endl;
+		world_mt.set("mod_storage_backend", migrate_to);
+		if (!world_mt.updateConfigFile(world_mt_path.c_str()))
+			errorstream << "Failed to update world.mt!" << std::endl;
+		else
+			actionstream << "world.mt updated" << std::endl;
+
+	} catch (BaseException &e) {
+		errorstream << "An error occurred during migration: " << e.what() << std::endl;
+	}
+
+	delete srcdb;
+	delete dstdb;
+
+	if (succeeded && backend == "files") {
+		// Back up files
+		const std::string storage_path = game_params.world_path + DIR_DELIM + "mod_storage";
+		const std::string backup_path = game_params.world_path + DIR_DELIM + "mod_storage.bak";
+		if (!fs::Rename(storage_path, backup_path))
+			warningstream << "After migration, " << storage_path
+				<< " could not be renamed to " << backup_path << std::endl;
+	}
+
+	return succeeded;
+}

src/server.h

@@ -283,6 +283,7 @@ public:
 	virtual u16 allocateUnknownNodeId(const std::string &name);
 	IRollbackManager *getRollbackManager() { return m_rollback; }
 	virtual EmergeManager *getEmergeManager() { return m_emerge; }
+	virtual ModMetadataDatabase *getModStorageDatabase() { return m_mod_storage_database; }
 
 	IWritableItemDefManager* getWritableItemDefManager();
 	NodeDefManager* getWritableNodeDefManager();
@@ -293,7 +294,6 @@ public:
 	void getModNames(std::vector<std::string> &modlist);
 	std::string getBuiltinLuaPath();
 	virtual std::string getWorldPath() const { return m_path_world; }
-	virtual std::string getModStoragePath() const;
 
 	inline bool isSingleplayer()
 			{ return m_simple_singleplayer_mode; }
@@ -377,6 +377,14 @@ public:
 	// Get or load translations for a language
 	Translations *getTranslationLanguage(const std::string &lang_code);
 
+	static ModMetadataDatabase *openModStorageDatabase(const std::string &world_path);
+
+	static ModMetadataDatabase *openModStorageDatabase(const std::string &backend,
+			const std::string &world_path, const Settings &world_mt);
+
+	static bool migrateModStorageDatabase(const GameParams &game_params,
+			const Settings &cmd_args);
+
 	// Bind address
 	Address m_bind_addr;
 
@@ -678,6 +686,7 @@ private:
 	s32 nextSoundId();
 
 	std::unordered_map<std::string, ModMetadata *> m_mod_storages;
+	ModMetadataDatabase *m_mod_storage_database = nullptr;
 	float m_mod_storage_save_timer = 10.0f;
 
 	// CSM restrictions byteflag

src/unittest/CMakeLists.txt

@@ -14,6 +14,7 @@ set (UNITTEST_SRCS
 	${CMAKE_CURRENT_SOURCE_DIR}/test_map_settings_manager.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/test_mapnode.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/test_modchannels.cpp
+	${CMAKE_CURRENT_SOURCE_DIR}/test_modmetadatadatabase.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/test_nodedef.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/test_noderesolver.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/test_noise.cpp

src/unittest/test.cpp

@@ -25,6 +25,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "gamedef.h"
 #include "modchannels.h"
 #include "content/mods.h"
+#include "database/database-dummy.h"
 #include "util/numeric.h"
 #include "porting.h"
 
@@ -55,6 +56,7 @@ public:
 	scene::ISceneManager *getSceneManager() { return m_scenemgr; }
 	IRollbackManager *getRollbackManager() { return m_rollbackmgr; }
 	EmergeManager *getEmergeManager() { return m_emergemgr; }
+	ModMetadataDatabase *getModStorageDatabase() { return m_mod_storage_database; }
 
 	scene::IAnimatedMesh *getMesh(const std::string &filename) { return NULL; }
 	bool checkLocalPrivilege(const std::string &priv) { return false; }
@@ -68,7 +70,6 @@ public:
 		return testmodspec;
 	}
 	virtual const ModSpec* getModSpec(const std::string &modname) const { return NULL; }
-	virtual std::string getModStoragePath() const { return "."; }
 	virtual bool registerModStorage(ModMetadata *meta) { return true; }
 	virtual void unregisterModStorage(const std::string &name) {}
 	bool joinModChannel(const std::string &channel);
@@ -89,11 +90,13 @@ private:
 	scene::ISceneManager *m_scenemgr = nullptr;
 	IRollbackManager *m_rollbackmgr = nullptr;
 	EmergeManager *m_emergemgr = nullptr;
+	ModMetadataDatabase *m_mod_storage_database = nullptr;
 	std::unique_ptr<ModChannelMgr> m_modchannel_mgr;
 };
 
 
 TestGameDef::TestGameDef() :
+	m_mod_storage_database(new Database_Dummy()),
 	m_modchannel_mgr(new ModChannelMgr())
 {
 	m_itemdef = createItemDefManager();
@@ -107,6 +110,7 @@ TestGameDef::~TestGameDef()
 {
 	delete m_itemdef;
 	delete m_nodedef;
+	delete m_mod_storage_database;
 }
 
 

src/unittest/test_modmetadatadatabase.cpp

@@ -0,0 +1,253 @@
+/*
+Minetest
+Copyright (C) 2018 bendeutsch, Ben Deutsch <ben@bendeutsch.de>
+Copyright (C) 2021 TurkeyMcMac, Jude Melton-Houghton <jwmhjwmh@gmail.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU Lesser General Public License as published by
+the Free Software Foundation; either version 2.1 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+*/
+
+// This file is an edited copy of test_authdatabase.cpp
+
+#include "test.h"
+
+#include <algorithm>
+#include "database/database-files.h"
+#include "database/database-sqlite3.h"
+#include "filesys.h"
+
+namespace
+{
+// Anonymous namespace to create classes that are only
+// visible to this file
+//
+// These are helpers that return a *ModMetadataDatabase and
+// allow us to run the same tests on different databases and
+// database acquisition strategies.
+
+class ModMetadataDatabaseProvider
+{
+public:
+	virtual ~ModMetadataDatabaseProvider() = default;
+	virtual ModMetadataDatabase *getModMetadataDatabase() = 0;
+};
+
+class FixedProvider : public ModMetadataDatabaseProvider
+{
+public:
+	FixedProvider(ModMetadataDatabase *mod_meta_db) : mod_meta_db(mod_meta_db){};
+	virtual ~FixedProvider(){};
+	virtual ModMetadataDatabase *getModMetadataDatabase() { return mod_meta_db; };
+
+private:
+	ModMetadataDatabase *mod_meta_db;
+};
+
+class FilesProvider : public ModMetadataDatabaseProvider
+{
+public:
+	FilesProvider(const std::string &dir) : dir(dir){};
+	virtual ~FilesProvider()
+	{
+		if (mod_meta_db)
+			mod_meta_db->endSave();
+		delete mod_meta_db;
+	}
+	virtual ModMetadataDatabase *getModMetadataDatabase()
+	{
+		if (mod_meta_db)
+			mod_meta_db->endSave();
+		delete mod_meta_db;
+		mod_meta_db = new ModMetadataDatabaseFiles(dir);
+		mod_meta_db->beginSave();
+		return mod_meta_db;
+	};
+
+private:
+	std::string dir;
+	ModMetadataDatabase *mod_meta_db = nullptr;
+};
+
+class SQLite3Provider : public ModMetadataDatabaseProvider
+{
+public:
+	SQLite3Provider(const std::string &dir) : dir(dir){};
+	virtual ~SQLite3Provider()
+	{
+		if (mod_meta_db)
+			mod_meta_db->endSave();
+		delete mod_meta_db;
+	}
+	virtual ModMetadataDatabase *getModMetadataDatabase()
+	{
+		if (mod_meta_db)
+			mod_meta_db->endSave();
+		delete mod_meta_db;
+		mod_meta_db = new ModMetadataDatabaseSQLite3(dir);
+		mod_meta_db->beginSave();
+		return mod_meta_db;
+	};
+
+private:
+	std::string dir;
+	ModMetadataDatabase *mod_meta_db = nullptr;
+};
+}
+
+class TestModMetadataDatabase : public TestBase
+{
+public:
+	TestModMetadataDatabase() { TestManager::registerTestModule(this); }
+	const char *getName() { return "TestModMetadataDatabase"; }
+
+	void runTests(IGameDef *gamedef);
+	void runTestsForCurrentDB();
+
+	void testRecallFail();
+	void testCreate();
+	void testRecall();
+	void testChange();
+	void testRecallChanged();
+	void testListMods();
+	void testRemove();
+
+private:
+	ModMetadataDatabaseProvider *mod_meta_provider;
+};
+
+static TestModMetadataDatabase g_test_instance;
+
+void TestModMetadataDatabase::runTests(IGameDef *gamedef)
+{
+	// fixed directory, for persistence
+	thread_local const std::string test_dir = getTestTempDirectory();
+
+	// Each set of tests is run twice for each database type:
+	// one where we reuse the same ModMetadataDatabase object (to test local caching),
+	// and one where we create a new ModMetadataDatabase object for each call
+	// (to test actual persistence).
+
+	rawstream << "-------- Files database (same object)" << std::endl;
+
+	ModMetadataDatabase *mod_meta_db = new ModMetadataDatabaseFiles(test_dir);
+	mod_meta_provider = new FixedProvider(mod_meta_db);
+
+	runTestsForCurrentDB();
+
+	delete mod_meta_db;
+	delete mod_meta_provider;
+
+	// reset database
+	fs::RecursiveDelete(test_dir + DIR_DELIM + "mod_storage");
+
+	rawstream << "-------- Files database (new objects)" << std::endl;
+
+	mod_meta_provider = new FilesProvider(test_dir);
+
+	runTestsForCurrentDB();
+
+	delete mod_meta_provider;
+
+	rawstream << "-------- SQLite3 database (same object)" << std::endl;
+
+	mod_meta_db = new ModMetadataDatabaseSQLite3(test_dir);
+	mod_meta_provider = new FixedProvider(mod_meta_db);
+
+	runTestsForCurrentDB();
+
+	delete mod_meta_db;
+	delete mod_meta_provider;
+
+	// reset database
+	fs::DeleteSingleFileOrEmptyDirectory(test_dir + DIR_DELIM + "mod_storage.sqlite");
+
+	rawstream << "-------- SQLite3 database (new objects)" << std::endl;
+
+	mod_meta_provider = new SQLite3Provider(test_dir);
+
+	runTestsForCurrentDB();
+
+	delete mod_meta_provider;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+void TestModMetadataDatabase::runTestsForCurrentDB()
+{
+	TEST(testRecallFail);
+	TEST(testCreate);
+	TEST(testRecall);
+	TEST(testChange);
+	TEST(testRecallChanged);
+	TEST(testListMods);
+	TEST(testRemove);
+	TEST(testRecallFail);
+}
+
+void TestModMetadataDatabase::testRecallFail()
+{
+	ModMetadataDatabase *mod_meta_db = mod_meta_provider->getModMetadataDatabase();
+	StringMap recalled;
+	mod_meta_db->getModEntries("mod1", &recalled);
+	UASSERT(recalled.empty());
+}
+
+void TestModMetadataDatabase::testCreate()
+{
+	ModMetadataDatabase *mod_meta_db = mod_meta_provider->getModMetadataDatabase();
+	StringMap recalled;
+	UASSERT(mod_meta_db->setModEntry("mod1", "key1", "value1"));
+}
+
+void TestModMetadataDatabase::testRecall()
+{
+	ModMetadataDatabase *mod_meta_db = mod_meta_provider->getModMetadataDatabase();
+	StringMap recalled;
+	mod_meta_db->getModEntries("mod1", &recalled);
+	UASSERT(recalled.size() == 1);
+	UASSERT(recalled["key1"] == "value1");
+}
+
+void TestModMetadataDatabase::testChange()
+{
+	ModMetadataDatabase *mod_meta_db = mod_meta_provider->getModMetadataDatabase();
+	StringMap recalled;
+	UASSERT(mod_meta_db->setModEntry("mod1", "key1", "value2"));
+}
+
+void TestModMetadataDatabase::testRecallChanged()
+{
+	ModMetadataDatabase *mod_meta_db = mod_meta_provider->getModMetadataDatabase();
+	StringMap recalled;
+	mod_meta_db->getModEntries("mod1", &recalled);
+	UASSERT(recalled.size() == 1);
+	UASSERT(recalled["key1"] == "value2");
+}
+
+void TestModMetadataDatabase::testListMods()
+{
+	ModMetadataDatabase *mod_meta_db = mod_meta_provider->getModMetadataDatabase();
+	UASSERT(mod_meta_db->setModEntry("mod2", "key1", "value1"));
+	std::vector<std::string> mod_list;
+	mod_meta_db->listMods(&mod_list);
+	UASSERT(mod_list.size() == 2);
+	UASSERT(std::find(mod_list.cbegin(), mod_list.cend(), "mod1") != mod_list.cend());
+	UASSERT(std::find(mod_list.cbegin(), mod_list.cend(), "mod2") != mod_list.cend());
+}
+
+void TestModMetadataDatabase::testRemove()
+{
+	ModMetadataDatabase *mod_meta_db = mod_meta_provider->getModMetadataDatabase();
+	UASSERT(mod_meta_db->removeModEntry("mod1", "key1"));
+}