Server pushing media at runtime (#9961)

doc/lua_api.txt

@@ -5217,6 +5217,20 @@ Server
     * Returns a code (0: successful, 1: no such player, 2: player is connected)
 * `minetest.remove_player_auth(name)`: remove player authentication data
     * Returns boolean indicating success (false if player nonexistant)
+* `minetest.dynamic_add_media(filepath)`
+    * Adds the file at the given path to the media sent to clients by the server
+      on startup and also pushes this file to already connected clients.
+      The file must be a supported image, sound or model format. It must not be
+      modified, deleted, moved or renamed after calling this function.
+      The list of dynamically added media is not persisted.
+    * Returns boolean indicating success (duplicate files count as error)
+    * The media will be ready to use (in e.g. entity textures, sound_play)
+      immediately after calling this function.
+      Old clients that lack support for this feature will not see the media
+      unless they reconnect to the server.
+    * Since media transferred this way does not use client caching or HTTP
+      transfers, dynamic media should not be used with big files or performance
+      will suffer.
 
 Bans
 ----

src/client/client.cpp

@@ -670,11 +670,9 @@ void Client::step(float dtime)
 	}
 }
 
-bool Client::loadMedia(const std::string &data, const std::string &filename)
+bool Client::loadMedia(const std::string &data, const std::string &filename,
+	bool from_media_push)
 {
-	// Silly irrlicht's const-incorrectness
-	Buffer<char> data_rw(data.c_str(), data.size());
-
 	std::string name;
 
 	const char *image_ext[] = {
@@ -690,6 +688,9 @@ bool Client::loadMedia(const std::string &data, const std::string &filename)
 		io::IFileSystem *irrfs = RenderingEngine::get_filesystem();
 		video::IVideoDriver *vdrv = RenderingEngine::get_video_driver();
 
+		// Silly irrlicht's const-incorrectness
+		Buffer<char> data_rw(data.c_str(), data.size());
+
 		// Create an irrlicht memory file
 		io::IReadFile *rfile = irrfs->createMemoryReadFile(
 				*data_rw, data_rw.getSize(), "_tempreadfile");
@@ -727,7 +728,6 @@ bool Client::loadMedia(const std::string &data, const std::string &filename)
 		".x", ".b3d", ".md2", ".obj",
 		NULL
 	};
-
 	name = removeStringEnd(filename, model_ext);
 	if (!name.empty()) {
 		verbosestream<<"Client: Storing model into memory: "
@@ -744,6 +744,8 @@ bool Client::loadMedia(const std::string &data, const std::string &filename)
 	};
 	name = removeStringEnd(filename, translate_ext);
 	if (!name.empty()) {
+		if (from_media_push)
+			return false;
 		TRACESTREAM(<< "Client: Loading translation: "
 				<< "\"" << filename << "\"" << std::endl);
 		g_client_translations->loadTranslation(data);

src/client/client.h

@@ -222,6 +222,7 @@ public:
 	void handleCommand_FormspecPrepend(NetworkPacket *pkt);
 	void handleCommand_CSMRestrictionFlags(NetworkPacket *pkt);
 	void handleCommand_PlayerSpeed(NetworkPacket *pkt);
+	void handleCommand_MediaPush(NetworkPacket *pkt);
 
 	void ProcessData(NetworkPacket *pkt);
 
@@ -376,7 +377,8 @@ public:
 
 	// The following set of functions is used by ClientMediaDownloader
 	// Insert a media file appropriately into the appropriate manager
-	bool loadMedia(const std::string &data, const std::string &filename);
+	bool loadMedia(const std::string &data, const std::string &filename,
+		bool from_media_push = false);
 	// Send a request for conventional media transfer
 	void request_media(const std::vector<std::string> &file_requests);
 
@@ -488,6 +490,7 @@ private:
 	Camera *m_camera = nullptr;
 	Minimap *m_minimap = nullptr;
 	bool m_minimap_disabled_by_server = false;
+
 	// Server serialization version
 	u8 m_server_ser_ver;
 
@@ -529,7 +532,6 @@ private:
 	AuthMechanism m_chosen_auth_mech;
 	void *m_auth_data = nullptr;
 
-
 	bool m_access_denied = false;
 	bool m_access_denied_reconnect = false;
 	std::string m_access_denied_reason = "";
@@ -538,7 +540,10 @@ private:
 	bool m_nodedef_received = false;
 	bool m_activeobjects_received = false;
 	bool m_mods_loaded = false;
+
 	ClientMediaDownloader *m_media_downloader;
+	// Set of media filenames pushed by server at runtime
+	std::unordered_set<std::string> m_media_pushed_files;
 
 	// time_of_day speed approximation for old protocol
 	bool m_time_of_day_set = false;

src/client/clientmedia.cpp

@@ -35,6 +35,15 @@ static std::string getMediaCacheDir()
 	return porting::path_cache + DIR_DELIM + "media";
 }
 
+bool clientMediaUpdateCache(const std::string &raw_hash, const std::string &filedata)
+{
+	FileCache media_cache(getMediaCacheDir());
+	std::string sha1_hex = hex_encode(raw_hash);
+	if (!media_cache.exists(sha1_hex))
+		return media_cache.update(sha1_hex, filedata);
+	return true;
+}
+
 /*
 	ClientMediaDownloader
 */
@@ -559,7 +568,6 @@ bool ClientMediaDownloader::checkAndLoad(
 	return true;
 }
 
-
 /*
 	Minetest Hashset File Format
 

src/client/clientmedia.h

@@ -33,6 +33,11 @@ struct HTTPFetchResult;
 #define MTHASHSET_FILE_SIGNATURE 0x4d544853 // 'MTHS'
 #define MTHASHSET_FILE_NAME "index.mth"
 
+// Store file into media cache (unless it exists already)
+// Validating the hash is responsibility of the caller
+bool clientMediaUpdateCache(const std::string &raw_hash,
+	const std::string &filedata);
+
 class ClientMediaDownloader
 {
 public:

src/client/filecache.cpp

@@ -82,8 +82,16 @@ bool FileCache::update(const std::string &name, const std::string &data)
 	std::string path = m_dir + DIR_DELIM + name;
 	return updateByPath(path, data);
 }
+
 bool FileCache::load(const std::string &name, std::ostream &os)
 {
 	std::string path = m_dir + DIR_DELIM + name;
 	return loadByPath(path, os);
 }
+
+bool FileCache::exists(const std::string &name)
+{
+	std::string path = m_dir + DIR_DELIM + name;
+	std::ifstream fis(path.c_str(), std::ios_base::binary);
+	return fis.good();
+}

src/client/filecache.h

@@ -33,6 +33,7 @@ public:
 
 	bool update(const std::string &name, const std::string &data);
 	bool load(const std::string &name, std::ostream &os);
+	bool exists(const std::string &name);
 
 private:
 	std::string m_dir;

src/filesys.cpp

@@ -691,6 +691,12 @@ std::string AbsolutePath(const std::string &path)
 const char *GetFilenameFromPath(const char *path)
 {
 	const char *filename = strrchr(path, DIR_DELIM_CHAR);
+	// Consistent with IsDirDelimiter this function handles '/' too
+	if (DIR_DELIM_CHAR != '/') {
+		const char *tmp = strrchr(path, '/');
+		if (tmp && tmp > filename)
+			filename = tmp;
+	}
 	return filename ? filename + 1 : path;
 }
 

src/network/clientopcodes.cpp

@@ -68,7 +68,7 @@ const ToClientCommandHandler toClientCommandTable[TOCLIENT_NUM_MSG_TYPES] =
 	{ "TOCLIENT_TIME_OF_DAY",              TOCLIENT_STATE_CONNECTED, &Client::handleCommand_TimeOfDay }, // 0x29
 	{ "TOCLIENT_CSM_RESTRICTION_FLAGS",    TOCLIENT_STATE_CONNECTED, &Client::handleCommand_CSMRestrictionFlags }, // 0x2A
 	{ "TOCLIENT_PLAYER_SPEED",             TOCLIENT_STATE_CONNECTED, &Client::handleCommand_PlayerSpeed }, // 0x2B
-	null_command_handler,
+	{ "TOCLIENT_MEDIA_PUSH",               TOCLIENT_STATE_CONNECTED, &Client::handleCommand_MediaPush }, // 0x2C
 	null_command_handler,
 	null_command_handler,
 	{ "TOCLIENT_CHAT_MESSAGE",             TOCLIENT_STATE_CONNECTED, &Client::handleCommand_ChatMessage }, // 0x2F

src/network/clientpackethandler.cpp

@@ -39,6 +39,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "script/scripting_client.h"
 #include "util/serialize.h"
 #include "util/srp.h"
+#include "util/sha1.h"
 #include "tileanimation.h"
 #include "gettext.h"
 #include "skyparams.h"
@@ -1471,6 +1472,51 @@ void Client::handleCommand_PlayerSpeed(NetworkPacket *pkt)
 	player->addVelocity(added_vel);
 }
 
+void Client::handleCommand_MediaPush(NetworkPacket *pkt)
+{
+	std::string raw_hash, filename, filedata;
+	bool cached;
+
+	*pkt >> raw_hash >> filename >> cached;
+	filedata = pkt->readLongString();
+
+	if (raw_hash.size() != 20 || filedata.empty() || filename.empty() ||
+			!string_allowed(filename, TEXTURENAME_ALLOWED_CHARS)) {
+		throw PacketError("Illegal filename, data or hash");
+	}
+
+	verbosestream << "Server pushes media file \"" << filename << "\" with "
+		<< filedata.size() << " bytes of data (cached=" << cached
+		<< ")" << std::endl;
+
+	if (m_media_pushed_files.count(filename) != 0) {
+		// Silently ignore for synchronization purposes
+		return;
+	}
+
+	// Compute and check checksum of data
+	std::string computed_hash;
+	{
+		SHA1 ctx;
+		ctx.addBytes(filedata.c_str(), filedata.size());
+		unsigned char *buf = ctx.getDigest();
+		computed_hash.assign((char*) buf, 20);
+		free(buf);
+	}
+	if (raw_hash != computed_hash) {
+		verbosestream << "Hash of file data mismatches, ignoring." << std::endl;
+		return;
+	}
+
+	// Actually load media
+	loadMedia(filedata, filename, true);
+	m_media_pushed_files.insert(filename);
+
+	// Cache file for the next time when this client joins the same server
+	if (cached)
+		clientMediaUpdateCache(raw_hash, filedata);
+}
+
 /*
  * Mod channels
  */

src/network/networkprotocol.h

@@ -323,6 +323,15 @@ enum ToClientCommand
 		v3f added_vel
 	 */
 
+	TOCLIENT_MEDIA_PUSH = 0x2C,
+	/*
+		std::string raw_hash
+		std::string filename
+		bool should_be_cached
+		u32 len
+		char filedata[len]
+	*/
+
 	// (oops, there is some gap here)
 
 	TOCLIENT_CHAT_MESSAGE = 0x2F,

src/network/serveropcodes.cpp

@@ -167,7 +167,7 @@ const ClientCommandFactory clientCommandFactoryTable[TOCLIENT_NUM_MSG_TYPES] =
 	{ "TOCLIENT_TIME_OF_DAY",              0, true }, // 0x29
 	{ "TOCLIENT_CSM_RESTRICTION_FLAGS",    0, true }, // 0x2A
 	{ "TOCLIENT_PLAYER_SPEED",             0, true }, // 0x2B
-	null_command_factory, // 0x2C
+	{ "TOCLIENT_MEDIA_PUSH",               0, true }, // 0x2C (sent over channel 1 too)
 	null_command_factory, // 0x2D
 	null_command_factory, // 0x2E
 	{ "TOCLIENT_CHAT_MESSAGE",             0, true }, // 0x2F

src/script/lua_api/l_server.cpp

@@ -22,6 +22,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "common/c_converter.h"
 #include "common/c_content.h"
 #include "cpp_api/s_base.h"
+#include "cpp_api/s_security.h"
 #include "server.h"
 #include "environment.h"
 #include "remoteplayer.h"
@@ -412,9 +413,6 @@ int ModApiServer::l_get_modnames(lua_State *L)
 	std::vector<std::string> modlist;
 	getServer(L)->getModNames(modlist);
 
-	// Take unsorted items from mods_unsorted and sort them into
-	// mods_sorted; not great performance but the number of mods on a
-	// server will likely be small.
 	std::sort(modlist.begin(), modlist.end());
 
 	// Package them up for Lua
@@ -474,6 +472,23 @@ int ModApiServer::l_sound_fade(lua_State *L)
 	return 0;
 }
 
+// dynamic_add_media(filepath)
+int ModApiServer::l_dynamic_add_media(lua_State *L)
+{
+	NO_MAP_LOCK_REQUIRED;
+
+	// Reject adding media before the server has started up
+	if (!getEnv(L))
+		throw LuaError("Dynamic media cannot be added before server has started up");
+
+	std::string filepath = readParam<std::string>(L, 1);
+	CHECK_SECURE_PATH(L, filepath.c_str(), false);
+
+	bool ok = getServer(L)->dynamicAddMedia(filepath);
+	lua_pushboolean(L, ok);
+	return 1;
+}
+
 // is_singleplayer()
 int ModApiServer::l_is_singleplayer(lua_State *L)
 {
@@ -538,6 +553,7 @@ void ModApiServer::Initialize(lua_State *L, int top)
 	API_FCT(sound_play);
 	API_FCT(sound_stop);
 	API_FCT(sound_fade);
+	API_FCT(dynamic_add_media);
 
 	API_FCT(get_player_information);
 	API_FCT(get_player_privs);

src/script/lua_api/l_server.h

@@ -70,6 +70,9 @@ private:
 	// sound_fade(handle, step, gain)
 	static int l_sound_fade(lua_State *L);
 
+	// dynamic_add_media(filepath)
+	static int l_dynamic_add_media(lua_State *L);
+
 	// get_player_privs(name, text)
 	static int l_get_player_privs(lua_State *L);
 

src/server.cpp

@@ -2405,9 +2405,87 @@ bool Server::SendBlock(session_t peer_id, const v3s16 &blockpos)
 	return true;
 }
 
+bool Server::addMediaFile(const std::string &filename,
+	const std::string &filepath, std::string *filedata_to,
+	std::string *digest_to)
+{
+	// If name contains illegal characters, ignore the file
+	if (!string_allowed(filename, TEXTURENAME_ALLOWED_CHARS)) {
+		infostream << "Server: ignoring illegal file name: \""
+				<< filename << "\"" << std::endl;
+		return false;
+	}
+	// If name is not in a supported format, ignore it
+	const char *supported_ext[] = {
+		".png", ".jpg", ".bmp", ".tga",
+		".pcx", ".ppm", ".psd", ".wal", ".rgb",
+		".ogg",
+		".x", ".b3d", ".md2", ".obj",
+		// Custom translation file format
+		".tr",
+		NULL
+	};
+	if (removeStringEnd(filename, supported_ext).empty()) {
+		infostream << "Server: ignoring unsupported file extension: \""
+				<< filename << "\"" << std::endl;
+		return false;
+	}
+	// Ok, attempt to load the file and add to cache
+
+	// Read data
+	std::ifstream fis(filepath.c_str(), std::ios_base::binary);
+	if (!fis.good()) {
+		errorstream << "Server::addMediaFile(): Could not open \""
+				<< filename << "\" for reading" << std::endl;
+		return false;
+	}
+	std::string filedata;
+	bool bad = false;
+	for (;;) {
+		char buf[1024];
+		fis.read(buf, sizeof(buf));
+		std::streamsize len = fis.gcount();
+		filedata.append(buf, len);
+		if (fis.eof())
+			break;
+		if (!fis.good()) {
+			bad = true;
+			break;
+		}
+	}
+	if (bad) {
+		errorstream << "Server::addMediaFile(): Failed to read \""
+				<< filename << "\"" << std::endl;
+		return false;
+	} else if (filedata.empty()) {
+		errorstream << "Server::addMediaFile(): Empty file \""
+				<< filepath << "\"" << std::endl;
+		return false;
+	}
+
+	SHA1 sha1;
+	sha1.addBytes(filedata.c_str(), filedata.length());
+
+	unsigned char *digest = sha1.getDigest();
+	std::string sha1_base64 = base64_encode(digest, 20);
+	std::string sha1_hex = hex_encode((char*) digest, 20);
+	if (digest_to)
+		*digest_to = std::string((char*) digest, 20);
+	free(digest);
+
+	// Put in list
+	m_media[filename] = MediaInfo(filepath, sha1_base64);
+	verbosestream << "Server: " << sha1_hex << " is " << filename
+			<< std::endl;
+
+	if (filedata_to)
+		*filedata_to = std::move(filedata);
+	return true;
+}
+
 void Server::fillMediaCache()
 {
-	infostream<<"Server: Calculating media file checksums"<<std::endl;
+	infostream << "Server: Calculating media file checksums" << std::endl;
 
 	// Collect all media file paths
 	std::vector<std::string> paths;
@@ -2419,80 +2497,15 @@ void Server::fillMediaCache()
 	for (const std::string &mediapath : paths) {
 		std::vector<fs::DirListNode> dirlist = fs::GetDirListing(mediapath);
 		for (const fs::DirListNode &dln : dirlist) {
-			if (dln.dir) // Ignode dirs
+			if (dln.dir) // Ignore dirs
 				continue;
-			std::string filename = dln.name;
-			// If name contains illegal characters, ignore the file
-			if (!string_allowed(filename, TEXTURENAME_ALLOWED_CHARS)) {
-				infostream<<"Server: ignoring illegal file name: \""
-						<< filename << "\"" << std::endl;
-				continue;
-			}
-			// If name is not in a supported format, ignore it
-			const char *supported_ext[] = {
-				".png", ".jpg", ".bmp", ".tga",
-				".pcx", ".ppm", ".psd", ".wal", ".rgb",
-				".ogg",
-				".x", ".b3d", ".md2", ".obj",
-				// Custom translation file format
-				".tr",
-				NULL
-			};
-			if (removeStringEnd(filename, supported_ext).empty()){
-				infostream << "Server: ignoring unsupported file extension: \""
-						<< filename << "\"" << std::endl;
-				continue;
-			}
-			// Ok, attempt to load the file and add to cache
-			std::string filepath;
-			filepath.append(mediapath).append(DIR_DELIM).append(filename);
-
-			// Read data
-			std::ifstream fis(filepath.c_str(), std::ios_base::binary);
-			if (!fis.good()) {
-				errorstream << "Server::fillMediaCache(): Could not open \""
-						<< filename << "\" for reading" << std::endl;
-				continue;
-			}
-			std::ostringstream tmp_os(std::ios_base::binary);
-			bool bad = false;
-			for(;;) {
-				char buf[1024];
-				fis.read(buf, 1024);
-				std::streamsize len = fis.gcount();
-				tmp_os.write(buf, len);
-				if (fis.eof())
-					break;
-				if (!fis.good()) {
-					bad = true;
-					break;
-				}
-			}
-			if(bad) {
-				errorstream<<"Server::fillMediaCache(): Failed to read \""
-						<< filename << "\"" << std::endl;
-				continue;
-			}
-			if(tmp_os.str().length() == 0) {
-				errorstream << "Server::fillMediaCache(): Empty file \""
-						<< filepath << "\"" << std::endl;
-				continue;
-			}
-
-			SHA1 sha1;
-			sha1.addBytes(tmp_os.str().c_str(), tmp_os.str().length());
-
-			unsigned char *digest = sha1.getDigest();
-			std::string sha1_base64 = base64_encode(digest, 20);
-			std::string sha1_hex = hex_encode((char*)digest, 20);
-			free(digest);
-
-			// Put in list
-			m_media[filename] = MediaInfo(filepath, sha1_base64);
-			verbosestream << "Server: " << sha1_hex << " is " << filename
-					<< std::endl;
+			std::string filepath = mediapath;
+			filepath.append(DIR_DELIM).append(dln.name);
+			addMediaFile(dln.name, filepath);
 		}
 	}
+
+	infostream << "Server: " << m_media.size() << " media files collected" << std::endl;
 }
 
 void Server::sendMediaAnnouncement(session_t peer_id, const std::string &lang_code)
@@ -3428,6 +3441,44 @@ void Server::deleteParticleSpawner(const std::string &playername, u32 id)
 	SendDeleteParticleSpawner(peer_id, id);
 }
 
+bool Server::dynamicAddMedia(const std::string &filepath)
+{
+	std::string filename = fs::GetFilenameFromPath(filepath.c_str());
+	if (m_media.find(filename) != m_media.end()) {
+		errorstream << "Server::dynamicAddMedia(): file \"" << filename
+			<< "\" already exists in media cache" << std::endl;
+		return false;
+	}
+
+	// Load the file and add it to our media cache
+	std::string filedata, raw_hash;
+	bool ok = addMediaFile(filename, filepath, &filedata, &raw_hash);
+	if (!ok)
+		return false;
+
+	// Push file to existing clients
+	NetworkPacket pkt(TOCLIENT_MEDIA_PUSH, 0);
+	pkt << raw_hash << filename << (bool) true;
+	pkt.putLongString(filedata);
+
+	auto client_ids = m_clients.getClientIDs(CS_DefinitionsSent);
+	for (session_t client_id : client_ids) {
+		/*
+			The network layer only guarantees ordered delivery inside a channel.
+			Since the very next packet could be one that uses the media, we have
+			to push the media over ALL channels to ensure it is processed before
+			it is used.
+			In practice this means we have to send it twice:
+			- channel 1 (HUD)
+			- channel 0 (everything else: e.g. play_sound, object messages)
+		*/
+		m_clients.send(client_id, 1, &pkt, true);
+		m_clients.send(client_id, 0, &pkt, true);
+	}
+
+	return true;
+}
+
 // actions: time-reversed list
 // Return value: success/failure
 bool Server::rollbackRevertActions(const std::list<RollbackAction> &actions,

src/server.h

@@ -236,6 +236,8 @@ public:
 
 	void deleteParticleSpawner(const std::string &playername, u32 id);
 
+	bool dynamicAddMedia(const std::string &filepath);
+
 	ServerInventoryManager *getInventoryMgr() const { return m_inventory_mgr.get(); }
 	void sendDetachedInventory(Inventory *inventory, const std::string &name, session_t peer_id);
 
@@ -435,6 +437,8 @@ private:
 	// Sends blocks to clients (locks env and con on its own)
 	void SendBlocks(float dtime);
 
+	bool addMediaFile(const std::string &filename, const std::string &filepath,
+			std::string *filedata = nullptr, std::string *digest = nullptr);
 	void fillMediaCache();
 	void sendMediaAnnouncement(session_t peer_id, const std::string &lang_code);
 	void sendRequestedMedia(session_t peer_id,