Dynamic_Add_Media v2 (#11550)

builtin/game/misc.lua

@@ -269,27 +269,8 @@ function core.cancel_shutdown_requests()
 end
 
 
--- Callback handling for dynamic_add_media
+-- Used for callback handling with dynamic_add_media
-
+core.dynamic_media_callbacks = {}
-local dynamic_add_media_raw = core.dynamic_add_media_raw
-core.dynamic_add_media_raw = nil
-function core.dynamic_add_media(filepath, callback)
-	local ret = dynamic_add_media_raw(filepath)
-	if ret == false then
-		return ret
-	end
-	if callback == nil then
-		core.log("deprecated", "Calling minetest.dynamic_add_media without "..
-			"a callback is deprecated and will stop working in future versions.")
-	else
-		-- At the moment async loading is not actually implemented, so we
-		-- immediately call the callback ourselves
-		for _, name in ipairs(ret) do
-			callback(name)
-		end
-	end
-	return true
-end
 
 
 -- PNG encoder safety wrapper

doc/lua_api.txt

@@ -5649,22 +5649,33 @@ Server
     * Returns a code (0: successful, 1: no such player, 2: player is connected)
 * `minetest.remove_player_auth(name)`: remove player authentication data
     * Returns boolean indicating success (false if player nonexistant)
-* `minetest.dynamic_add_media(filepath, callback)`
+* `minetest.dynamic_add_media(options, callback)`
-    * `filepath`: path to a media file on the filesystem
+    * `options`: table containing the following parameters
-    * `callback`: function with arguments `name`, where name is a player name
+        * `filepath`: path to a media file on the filesystem
-      (previously there was no callback argument; omitting it is deprecated)
+        * `to_player`: name of the player the media should be sent to instead of
-    * Adds the file to the media sent to clients by the server on startup
+                       all players (optional)
-      and also pushes this file to already connected clients.
+        * `ephemeral`: boolean that marks the media as ephemeral,
-      The file must be a supported image, sound or model format. It must not be
+                       it will not be cached on the client (optional, default false)
-      modified, deleted, moved or renamed after calling this function.
+    * `callback`: function with arguments `name`, which is a player name
-      The list of dynamically added media is not persisted.
+    * Pushes the specified media file to client(s). (details below)
+      The file must be a supported image, sound or model format.
+      Dynamically added media is not persisted between server restarts.
     * Returns false on error, true if the request was accepted
     * The given callback will be called for every player as soon as the
       media is available on the client.
-      Old clients that lack support for this feature will not see the media
+    * Details/Notes:
-      unless they reconnect to the server. (callback won't be called)
+      * If `ephemeral`=false and `to_player` is unset the file is added to the media
-    * Since media transferred this way currently does not use client caching
+        sent to clients on startup, this means the media will appear even on
-       or HTTP transfers, dynamic media should not be used with big files.
+        old clients if they rejoin the server.
+      * If `ephemeral`=false the file must not be modified, deleted, moved or
+        renamed after calling this function.
+      * Regardless of any use of `ephemeral`, adding media files with the same
+        name twice is not possible/guaranteed to work. An exception to this is the
+        use of `to_player` to send the same, already existent file to multiple
+        chosen players.
+    * Clients will attempt to fetch files added this way via remote media,
+      this can make transfer of bigger files painless (if set up). Nevertheless
+      it is advised not to use dynamic media for big media files.
 
 Bans
 ----

src/client/client.cpp

@@ -555,6 +555,29 @@ void Client::step(float dtime)
 			m_media_downloader = NULL;
 		}
 	}
+	{
+		// Acknowledge dynamic media downloads to server
+		std::vector<u32> done;
+		for (auto it = m_pending_media_downloads.begin();
+				it != m_pending_media_downloads.end();) {
+			assert(it->second->isStarted());
+			it->second->step(this);
+			if (it->second->isDone()) {
+				done.emplace_back(it->first);
+
+				it = m_pending_media_downloads.erase(it);
+			} else {
+				it++;
+			}
+
+			if (done.size() == 255) { // maximum in one packet
+				sendHaveMedia(done);
+				done.clear();
+			}
+		}
+		if (!done.empty())
+			sendHaveMedia(done);
+	}
 
 	/*
 		If the server didn't update the inventory in a while, revert
@@ -770,7 +793,8 @@ void Client::request_media(const std::vector<std::string> &file_requests)
 	Send(&pkt);
 
 	infostream << "Client: Sending media request list to server ("
-			<< file_requests.size() << " files. packet size)" << std::endl;
+			<< file_requests.size() << " files, packet size "
+			<< pkt.getSize() << ")" << std::endl;
 }
 
 void Client::initLocalMapSaving(const Address &address,
@@ -1295,6 +1319,19 @@ void Client::sendPlayerPos()
 	Send(&pkt);
 }
 
+void Client::sendHaveMedia(const std::vector<u32> &tokens)
+{
+	NetworkPacket pkt(TOSERVER_HAVE_MEDIA, 1 + tokens.size() * 4);
+
+	sanity_check(tokens.size() < 256);
+
+	pkt << static_cast<u8>(tokens.size());
+	for (u32 token : tokens)
+		pkt << token;
+
+	Send(&pkt);
+}
+
 void Client::removeNode(v3s16 p)
 {
 	std::map<v3s16, MapBlock*> modified_blocks;

src/client/client.h

@@ -53,6 +53,7 @@ class ISoundManager;
 class NodeDefManager;
 //class IWritableCraftDefManager;
 class ClientMediaDownloader;
+class SingleMediaDownloader;
 struct MapDrawControl;
 class ModChannelMgr;
 class MtEventManager;
@@ -245,6 +246,7 @@ public:
 	void sendDamage(u16 damage);
 	void sendRespawn();
 	void sendReady();
+	void sendHaveMedia(const std::vector<u32> &tokens);
 
 	ClientEnvironment& getEnv() { return m_env; }
 	ITextureSource *tsrc() { return getTextureSource(); }
@@ -536,9 +538,13 @@ private:
 	bool m_activeobjects_received = false;
 	bool m_mods_loaded = false;
 
+	std::vector<std::string> m_remote_media_servers;
+	// Media downloader, only exists during init
 	ClientMediaDownloader *m_media_downloader;
 	// Set of media filenames pushed by server at runtime
 	std::unordered_set<std::string> m_media_pushed_files;
+	// Pending downloads of dynamic media (key: token)
+	std::vector<std::pair<u32, std::unique_ptr<SingleMediaDownloader>>> m_pending_media_downloads;
 
 	// time_of_day speed approximation for old protocol
 	bool m_time_of_day_set = false;

src/client/clientmedia.cpp

@@ -49,7 +49,6 @@ bool clientMediaUpdateCache(const std::string &raw_hash, const std::string &file
 */
 
 ClientMediaDownloader::ClientMediaDownloader():
-	m_media_cache(getMediaCacheDir()),
 	m_httpfetch_caller(HTTPFETCH_DISCARD)
 {
 }
@@ -66,6 +65,12 @@ ClientMediaDownloader::~ClientMediaDownloader()
 		delete remote;
 }
 
+bool ClientMediaDownloader::loadMedia(Client *client, const std::string &data,
+		const std::string &name)
+{
+	return client->loadMedia(data, name);
+}
+
 void ClientMediaDownloader::addFile(const std::string &name, const std::string &sha1)
 {
 	assert(!m_initial_step_done); // pre-condition
@@ -105,7 +110,7 @@ void ClientMediaDownloader::addRemoteServer(const std::string &baseurl)
 {
 	assert(!m_initial_step_done);	// pre-condition
 
-	#ifdef USE_CURL
+#ifdef USE_CURL
 
 	if (g_settings->getBool("enable_remote_media_server")) {
 		infostream << "Client: Adding remote server \""
@@ -117,13 +122,13 @@ void ClientMediaDownloader::addRemoteServer(const std::string &baseurl)
 		m_remotes.push_back(remote);
 	}
 
-	#else
+#else
 
 	infostream << "Client: Ignoring remote server \""
 		<< baseurl << "\" because cURL support is not compiled in"
 		<< std::endl;
 
-	#endif
+#endif
 }
 
 void ClientMediaDownloader::step(Client *client)
@@ -172,36 +177,21 @@ void ClientMediaDownloader::initialStep(Client *client)
 	// Check media cache
 	m_uncached_count = m_files.size();
 	for (auto &file_it : m_files) {
-		std::string name = file_it.first;
+		const std::string &name = file_it.first;
 		FileStatus *filestatus = file_it.second;
 		const std::string &sha1 = filestatus->sha1;
 
-		std::ostringstream tmp_os(std::ios_base::binary);
+		if (tryLoadFromCache(name, sha1, client)) {
-		bool found_in_cache = m_media_cache.load(hex_encode(sha1), tmp_os);
+			filestatus->received = true;
-
+			m_uncached_count--;
-		// If found in cache, try to load it from there
-		if (found_in_cache) {
-			bool success = checkAndLoad(name, sha1,
-					tmp_os.str(), true, client);
-			if (success) {
-				filestatus->received = true;
-				m_uncached_count--;
-			}
 		}
 	}
 
 	assert(m_uncached_received_count == 0);
 
 	// Create the media cache dir if we are likely to write to it
-	if (m_uncached_count != 0) {
+	if (m_uncached_count != 0)
-		bool did = fs::CreateAllDirs(getMediaCacheDir());
+		createCacheDirs();
-		if (!did) {
-			errorstream << "Client: "
-				<< "Could not create media cache directory: "
-				<< getMediaCacheDir()
-				<< std::endl;
-		}
-	}
 
 	// If we found all files in the cache, report this fact to the server.
 	// If the server reported no remote servers, immediately start
@@ -301,8 +291,7 @@ void ClientMediaDownloader::remoteHashSetReceived(
 			// available on this server, add this server
 			// to the available_remotes array
 
-			for(std::map<std::string, FileStatus*>::iterator
+			for(auto it = m_files.upper_bound(m_name_bound);
-					it = m_files.upper_bound(m_name_bound);
 					it != m_files.end(); ++it) {
 				FileStatus *f = it->second;
 				if (!f->received && sha1_set.count(f->sha1))
@@ -328,8 +317,7 @@ void ClientMediaDownloader::remoteMediaReceived(
 
 	std::string name;
 	{
-		std::unordered_map<unsigned long, std::string>::iterator it =
+		auto it = m_remote_file_transfers.find(fetch_result.request_id);
-			m_remote_file_transfers.find(fetch_result.request_id);
 		assert(it != m_remote_file_transfers.end());
 		name = it->second;
 		m_remote_file_transfers.erase(it);
@@ -398,8 +386,7 @@ void ClientMediaDownloader::startRemoteMediaTransfers()
 {
 	bool changing_name_bound = true;
 
-	for (std::map<std::string, FileStatus*>::iterator
+	for (auto files_iter = m_files.upper_bound(m_name_bound);
-			files_iter = m_files.upper_bound(m_name_bound);
 			files_iter != m_files.end(); ++files_iter) {
 
 		// Abort if active fetch limit is exceeded
@@ -477,19 +464,18 @@ void ClientMediaDownloader::startConventionalTransfers(Client *client)
 	}
 }
 
-void ClientMediaDownloader::conventionalTransferDone(
+bool ClientMediaDownloader::conventionalTransferDone(
 		const std::string &name,
 		const std::string &data,
 		Client *client)
 {
 	// Check that file was announced
-	std::map<std::string, FileStatus*>::iterator
+	auto file_iter = m_files.find(name);
-		file_iter = m_files.find(name);
 	if (file_iter == m_files.end()) {
 		errorstream << "Client: server sent media file that was"
 			<< "not announced, ignoring it: \"" << name << "\""
 			<< std::endl;
-		return;
+		return false;
 	}
 	FileStatus *filestatus = file_iter->second;
 	assert(filestatus != NULL);
@@ -499,7 +485,7 @@ void ClientMediaDownloader::conventionalTransferDone(
 		errorstream << "Client: server sent media file that we already"
 			<< "received, ignoring it: \"" << name << "\""
 			<< std::endl;
-		return;
+		return true;
 	}
 
 	// Mark file as received, regardless of whether loading it works and
@@ -512,9 +498,45 @@ void ClientMediaDownloader::conventionalTransferDone(
 	// Check that received file matches announced checksum
 	// If so, load it
 	checkAndLoad(name, filestatus->sha1, data, false, client);
+
+	return true;
 }
 
-bool ClientMediaDownloader::checkAndLoad(
+/*
+	IClientMediaDownloader
+*/
+
+IClientMediaDownloader::IClientMediaDownloader():
+	m_media_cache(getMediaCacheDir()), m_write_to_cache(true)
+{
+}
+
+void IClientMediaDownloader::createCacheDirs()
+{
+	if (!m_write_to_cache)
+		return;
+
+	std::string path = getMediaCacheDir();
+	if (!fs::CreateAllDirs(path)) {
+		errorstream << "Client: Could not create media cache directory: "
+			<< path << std::endl;
+	}
+}
+
+bool IClientMediaDownloader::tryLoadFromCache(const std::string &name,
+	const std::string &sha1, Client *client)
+{
+	std::ostringstream tmp_os(std::ios_base::binary);
+	bool found_in_cache = m_media_cache.load(hex_encode(sha1), tmp_os);
+
+	// If found in cache, try to load it from there
+	if (found_in_cache)
+		return checkAndLoad(name, sha1, tmp_os.str(), true, client);
+
+	return false;
+}
+
+bool IClientMediaDownloader::checkAndLoad(
 		const std::string &name, const std::string &sha1,
 		const std::string &data, bool is_from_cache, Client *client)
 {
@@ -544,7 +566,7 @@ bool ClientMediaDownloader::checkAndLoad(
 	}
 
 	// Checksum is ok, try loading the file
-	bool success = client->loadMedia(data, name);
+	bool success = loadMedia(client, data, name);
 	if (!success) {
 		infostream << "Client: "
 			<< "Failed to load " << cached_or_received << " media: "
@@ -559,7 +581,7 @@ bool ClientMediaDownloader::checkAndLoad(
 		<< std::endl;
 
 	// Update cache (unless we just loaded the file from the cache)
-	if (!is_from_cache)
+	if (!is_from_cache && m_write_to_cache)
 		m_media_cache.update(sha1_hex, data);
 
 	return true;
@@ -587,12 +609,10 @@ std::string ClientMediaDownloader::serializeRequiredHashSet()
 
 	// Write list of hashes of files that have not been
 	// received (found in cache) yet
-	for (std::map<std::string, FileStatus*>::iterator
+	for (const auto &it : m_files) {
-			it = m_files.begin();
+		if (!it.second->received) {
-			it != m_files.end(); ++it) {
+			FATAL_ERROR_IF(it.second->sha1.size() != 20, "Invalid SHA1 size");
-		if (!it->second->received) {
+			os << it.second->sha1;
-			FATAL_ERROR_IF(it->second->sha1.size() != 20, "Invalid SHA1 size");
-			os << it->second->sha1;
 		}
 	}
 
@@ -628,3 +648,145 @@ void ClientMediaDownloader::deSerializeHashSet(const std::string &data,
 		result.insert(data.substr(pos, 20));
 	}
 }
+
+/*
+	SingleMediaDownloader
+*/
+
+SingleMediaDownloader::SingleMediaDownloader(bool write_to_cache):
+	m_httpfetch_caller(HTTPFETCH_DISCARD)
+{
+	m_write_to_cache = write_to_cache;
+}
+
+SingleMediaDownloader::~SingleMediaDownloader()
+{
+	if (m_httpfetch_caller != HTTPFETCH_DISCARD)
+		httpfetch_caller_free(m_httpfetch_caller);
+}
+
+bool SingleMediaDownloader::loadMedia(Client *client, const std::string &data,
+		const std::string &name)
+{
+	return client->loadMedia(data, name, true);
+}
+
+void SingleMediaDownloader::addFile(const std::string &name, const std::string &sha1)
+{
+	assert(m_stage == STAGE_INIT); // pre-condition
+
+	assert(!name.empty());
+	assert(sha1.size() == 20);
+
+	FATAL_ERROR_IF(!m_file_name.empty(), "Cannot add a second file");
+	m_file_name = name;
+	m_file_sha1 = sha1;
+}
+
+void SingleMediaDownloader::addRemoteServer(const std::string &baseurl)
+{
+	assert(m_stage == STAGE_INIT); // pre-condition
+
+	if (g_settings->getBool("enable_remote_media_server"))
+		m_remotes.emplace_back(baseurl);
+}
+
+void SingleMediaDownloader::step(Client *client)
+{
+	if (m_stage == STAGE_INIT) {
+		m_stage = STAGE_CACHE_CHECKED;
+		initialStep(client);
+	}
+
+	// Remote media: check for completion of fetches
+	if (m_httpfetch_caller != HTTPFETCH_DISCARD) {
+		HTTPFetchResult fetch_result;
+		while (httpfetch_async_get(m_httpfetch_caller, fetch_result)) {
+			remoteMediaReceived(fetch_result, client);
+		}
+	}
+}
+
+bool SingleMediaDownloader::conventionalTransferDone(const std::string &name,
+		const std::string &data, Client *client)
+{
+	if (name != m_file_name)
+		return false;
+
+	// Mark file as received unconditionally and try to load it
+	m_stage = STAGE_DONE;
+	checkAndLoad(name, m_file_sha1, data, false, client);
+	return true;
+}
+
+void SingleMediaDownloader::initialStep(Client *client)
+{
+	if (tryLoadFromCache(m_file_name, m_file_sha1, client))
+		m_stage = STAGE_DONE;
+	if (isDone())
+		return;
+
+	createCacheDirs();
+
+	// If the server reported no remote servers, immediately fall back to
+	// conventional transfer.
+	if (!USE_CURL || m_remotes.empty()) {
+		startConventionalTransfer(client);
+	} else {
+		// Otherwise start by requesting the file from the first remote media server
+		m_httpfetch_caller = httpfetch_caller_alloc();
+		m_current_remote = 0;
+		startRemoteMediaTransfer();
+	}
+}
+
+void SingleMediaDownloader::remoteMediaReceived(
+		const HTTPFetchResult &fetch_result, Client *client)
+{
+	sanity_check(!isDone());
+	sanity_check(m_current_remote >= 0);
+
+	// If fetch succeeded, try to load it
+	if (fetch_result.succeeded) {
+		bool success = checkAndLoad(m_file_name, m_file_sha1,
+				fetch_result.data, false, client);
+		if (success) {
+			m_stage = STAGE_DONE;
+			return;
+		}
+	}
+
+	// Otherwise try the next remote server or fall back to conventional transfer
+	m_current_remote++;
+	if (m_current_remote >= (int)m_remotes.size()) {
+		infostream << "Client: Failed to remote-fetch \"" << m_file_name
+				<< "\". Requesting it the usual way." << std::endl;
+		m_current_remote = -1;
+		startConventionalTransfer(client);
+	} else {
+		startRemoteMediaTransfer();
+	}
+}
+
+void SingleMediaDownloader::startRemoteMediaTransfer()
+{
+	std::string url = m_remotes.at(m_current_remote) + hex_encode(m_file_sha1);
+	verbosestream << "Client: Requesting remote media file "
+		<< "\"" << m_file_name << "\" " << "\"" << url << "\"" << std::endl;
+
+	HTTPFetchRequest fetch_request;
+	fetch_request.url = url;
+	fetch_request.caller = m_httpfetch_caller;
+	fetch_request.request_id = m_httpfetch_next_id;
+	fetch_request.timeout = g_settings->getS32("curl_file_download_timeout");
+	httpfetch_async(fetch_request);
+
+	m_httpfetch_next_id++;
+}
+
+void SingleMediaDownloader::startConventionalTransfer(Client *client)
+{
+	std::vector<std::string> requests;
+	requests.emplace_back(m_file_name);
+	client->request_media(requests);
+}

src/client/clientmedia.h

@@ -21,6 +21,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 
 #include "irrlichttypes.h"
 #include "filecache.h"
+#include "util/basic_macros.h"
 #include <ostream>
 #include <map>
 #include <set>
@@ -38,7 +39,62 @@ struct HTTPFetchResult;
 bool clientMediaUpdateCache(const std::string &raw_hash,
 	const std::string &filedata);
 
-class ClientMediaDownloader
+// more of a base class than an interface but this name was most convenient...
+class IClientMediaDownloader
+{
+public:
+	DISABLE_CLASS_COPY(IClientMediaDownloader)
+
+	virtual bool isStarted() const = 0;
+
+	// If this returns true, the downloader is done and can be deleted
+	virtual bool isDone() const = 0;
+
+	// Add a file to the list of required file (but don't fetch it yet)
+	virtual void addFile(const std::string &name, const std::string &sha1) = 0;
+
+	// Add a remote server to the list; ignored if not built with cURL
+	virtual void addRemoteServer(const std::string &baseurl) = 0;
+
+	// Steps the media downloader:
+	// - May load media into client by calling client->loadMedia()
+	// - May check media cache for files
+	// - May add files to media cache
+	// - May start remote transfers by calling httpfetch_async
+	// - May check for completion of current remote transfers
+	// - May start conventional transfers by calling client->request_media()
+	// - May inform server that all media has been loaded
+	//   by calling client->received_media()
+	// After step has been called once, don't call addFile/addRemoteServer.
+	virtual void step(Client *client) = 0;
+
+	// Must be called for each file received through TOCLIENT_MEDIA
+	// returns true if this file belongs to this downloader
+	virtual bool conventionalTransferDone(const std::string &name,
+			const std::string &data, Client *client) = 0;
+
+protected:
+	IClientMediaDownloader();
+	virtual ~IClientMediaDownloader() = default;
+
+	// Forwards the call to the appropriate Client method
+	virtual bool loadMedia(Client *client, const std::string &data,
+		const std::string &name) = 0;
+
+	void createCacheDirs();
+
+	bool tryLoadFromCache(const std::string &name, const std::string &sha1,
+			Client *client);
+
+	bool checkAndLoad(const std::string &name, const std::string &sha1,
+			const std::string &data, bool is_from_cache, Client *client);
+
+	// Filesystem-based media cache
+	FileCache m_media_cache;
+	bool m_write_to_cache;
+};
+
+class ClientMediaDownloader : public IClientMediaDownloader
 {
 public:
 	ClientMediaDownloader();
@@ -52,39 +108,29 @@ public:
 		return 0.0f;
 	}
 
-	bool isStarted() const {
+	bool isStarted() const override {
 		return m_initial_step_done;
 	}
 
-	// If this returns true, the downloader is done and can be deleted
+	bool isDone() const override {
-	bool isDone() const {
 		return m_initial_step_done &&
 			m_uncached_received_count == m_uncached_count;
 	}
 
-	// Add a file to the list of required file (but don't fetch it yet)
+	void addFile(const std::string &name, const std::string &sha1) override;
-	void addFile(const std::string &name, const std::string &sha1);
 
-	// Add a remote server to the list; ignored if not built with cURL
+	void addRemoteServer(const std::string &baseurl) override;
-	void addRemoteServer(const std::string &baseurl);
 
-	// Steps the media downloader:
+	void step(Client *client) override;
-	// - May load media into client by calling client->loadMedia()
-	// - May check media cache for files
-	// - May add files to media cache
-	// - May start remote transfers by calling httpfetch_async
-	// - May check for completion of current remote transfers
-	// - May start conventional transfers by calling client->request_media()
-	// - May inform server that all media has been loaded
-	//   by calling client->received_media()
-	// After step has been called once, don't call addFile/addRemoteServer.
-	void step(Client *client);
 
-	// Must be called for each file received through TOCLIENT_MEDIA
+	bool conventionalTransferDone(
-	void conventionalTransferDone(
 			const std::string &name,
 			const std::string &data,
-			Client *client);
+			Client *client) override;
+
+protected:
+	bool loadMedia(Client *client, const std::string &data,
+			const std::string &name) override;
 
 private:
 	struct FileStatus {
@@ -107,13 +153,9 @@ private:
 	void startRemoteMediaTransfers();
 	void startConventionalTransfers(Client *client);
 
-	bool checkAndLoad(const std::string &name, const std::string &sha1,
-			const std::string &data, bool is_from_cache,
-			Client *client);
-
-	std::string serializeRequiredHashSet();
 	static void deSerializeHashSet(const std::string &data,
 			std::set<std::string> &result);
+	std::string serializeRequiredHashSet();
 
 	// Maps filename to file status
 	std::map<std::string, FileStatus*> m_files;
@@ -121,9 +163,6 @@ private:
 	// Array of remote media servers
 	std::vector<RemoteServerStatus*> m_remotes;
 
-	// Filesystem-based media cache
-	FileCache m_media_cache;
-
 	// Has an attempt been made to load media files from the file cache?
 	// Have hash sets been requested from remote servers?
 	bool m_initial_step_done = false;
@@ -149,3 +188,63 @@ private:
 	std::string m_name_bound = "";
 
 };
+
+// A media downloader that only downloads a single file.
+// It does/doesn't do several things the normal downloader does:
+// - won't fetch hash sets from remote servers
+// - will mark loaded media as coming from file push
+// - writing to file cache is optional
+class SingleMediaDownloader : public IClientMediaDownloader
+{
+public:
+	SingleMediaDownloader(bool write_to_cache);
+	~SingleMediaDownloader();
+
+	bool isStarted() const override {
+		return m_stage > STAGE_INIT;
+	}
+
+	bool isDone() const override {
+		return m_stage >= STAGE_DONE;
+	}
+
+	void addFile(const std::string &name, const std::string &sha1) override;
+
+	void addRemoteServer(const std::string &baseurl) override;
+
+	void step(Client *client) override;
+
+	bool conventionalTransferDone(const std::string &name,
+			const std::string &data, Client *client) override;
+
+protected:
+	bool loadMedia(Client *client, const std::string &data,
+			const std::string &name) override;
+
+private:
+	void initialStep(Client *client);
+	void remoteMediaReceived(const HTTPFetchResult &fetch_result, Client *client);
+	void startRemoteMediaTransfer();
+	void startConventionalTransfer(Client *client);
+
+	enum Stage {
+		STAGE_INIT,
+		STAGE_CACHE_CHECKED, // we have tried to load the file from cache
+		STAGE_DONE
+	};
+
+	// Information about the one file we want to fetch
+	std::string m_file_name;
+	std::string m_file_sha1;
+	s32 m_current_remote;
+
+	// Array of remote media servers
+	std::vector<std::string> m_remotes;
+
+	enum Stage m_stage = STAGE_INIT;
+
+	// Status of remote transfers
+	unsigned long m_httpfetch_caller;
+	unsigned long m_httpfetch_next_id = 0;
+
+};

src/filesys.cpp

@@ -21,8 +21,10 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "util/string.h"
 #include <iostream>
 #include <cstdio>
+#include <cstdlib>
 #include <cstring>
 #include <cerrno>
+#include <unistd.h>
 #include <fstream>
 #include "log.h"
 #include "config.h"
@@ -811,5 +813,15 @@ bool Rename(const std::string &from, const std::string &to)
 	return rename(from.c_str(), to.c_str()) == 0;
 }
 
+std::string CreateTempFile()
+{
+	std::string path = TempPath() + DIR_DELIM "MT_XXXXXX";
+	int fd = mkstemp(&path[0]); // modifies path
+	if (fd == -1)
+		return "";
+	close(fd);
+	return path;
+}
+
 } // namespace fs
 

src/filesys.h

@@ -71,6 +71,10 @@ bool DeleteSingleFileOrEmptyDirectory(const std::string &path);
 // Returns path to temp directory, can return "" on error
 std::string TempPath();
 
+// Returns path to securely-created temporary file (will already exist when this function returns)
+// can return "" on error
+std::string CreateTempFile();
+
 /* Returns a list of subdirectories, including the path itself, but excluding
        hidden directories (whose names start with . or _)
 */

src/network/clientopcodes.cpp

@@ -204,7 +204,7 @@ const ServerCommandFactory serverCommandFactoryTable[TOSERVER_NUM_MSG_TYPES] =
 	null_command_factory, // 0x3e
 	null_command_factory, // 0x3f
 	{ "TOSERVER_REQUEST_MEDIA",      1, true }, // 0x40
-	null_command_factory, // 0x41
+	{ "TOSERVER_HAVE_MEDIA",         2, true }, // 0x41
 	null_command_factory, // 0x42
 	{ "TOSERVER_CLIENT_READY",       1, true }, // 0x43
 	null_command_factory, // 0x44

src/network/clientpackethandler.cpp

@@ -670,21 +670,19 @@ void Client::handleCommand_AnnounceMedia(NetworkPacket* pkt)
 		m_media_downloader->addFile(name, sha1_raw);
 	}
 
-	try {
+	{
 		std::string str;
-
 		*pkt >> str;
 
 		Strfnd sf(str);
-		while(!sf.at_end()) {
+		while (!sf.at_end()) {
 			std::string baseurl = trim(sf.next(","));
-			if (!baseurl.empty())
+			if (!baseurl.empty()) {
+				m_remote_media_servers.emplace_back(baseurl);
 				m_media_downloader->addRemoteServer(baseurl);
+			}
 		}
 	}
-	catch(SerializationError& e) {
-		// not supported by server or turned off
-	}
 
 	m_media_downloader->step(this);
 }
@@ -716,31 +714,38 @@ void Client::handleCommand_Media(NetworkPacket* pkt)
 	if (num_files == 0)
 		return;
 
-	if (!m_media_downloader || !m_media_downloader->isStarted()) {
+	bool init_phase = m_media_downloader && m_media_downloader->isStarted();
-		const char *problem = m_media_downloader ?
+
-			"media has not been requested" :
+	if (init_phase) {
-			"all media has been received already";
+		// Mesh update thread must be stopped while
-		errorstream << "Client: Received media but "
+		// updating content definitions
-			<< problem << "! "
+		sanity_check(!m_mesh_update_thread.isRunning());
-			<< " bunch " << bunch_i << "/" << num_bunches
-			<< " files=" << num_files
-			<< " size=" << pkt->getSize() << std::endl;
-		return;
 	}
 
-	// Mesh update thread must be stopped while
+	for (u32 i = 0; i < num_files; i++) {
-	// updating content definitions
+		std::string name, data;
-	sanity_check(!m_mesh_update_thread.isRunning());
-
-	for (u32 i=0; i < num_files; i++) {
-		std::string name;
 
 		*pkt >> name;
+		data = pkt->readLongString();
 
-		std::string data = pkt->readLongString();
+		bool ok = false;
-
+		if (init_phase) {
-		m_media_downloader->conventionalTransferDone(
+			ok = m_media_downloader->conventionalTransferDone(name, data, this);
-				name, data, this);
+		} else {
+			// Check pending dynamic transfers, one of them must be it
+			for (const auto &it : m_pending_media_downloads) {
+				if (it.second->conventionalTransferDone(name, data, this)) {
+					ok = true;
+					break;
+				}
+			}
+		}
+		if (!ok) {
+			errorstream << "Client: Received media \"" << name
+				<< "\" but no downloads pending. " << num_bunches << " bunches, "
+				<< num_files << " in this one. (init_phase=" << init_phase
+				<< ")" << std::endl;
+		}
 	}
 }
 
@@ -1497,46 +1502,72 @@ void Client::handleCommand_PlayerSpeed(NetworkPacket *pkt)
 void Client::handleCommand_MediaPush(NetworkPacket *pkt)
 {
 	std::string raw_hash, filename, filedata;
+	u32 token;
 	bool cached;
 
 	*pkt >> raw_hash >> filename >> cached;
-	filedata = pkt->readLongString();
+	if (m_proto_ver >= 40)
+		*pkt >> token;
+	else
+		filedata = pkt->readLongString();
 
-	if (raw_hash.size() != 20 || filedata.empty() || filename.empty() ||
+	if (raw_hash.size() != 20 || filename.empty() ||
+			(m_proto_ver < 40 && filedata.empty()) ||
 			!string_allowed(filename, TEXTURENAME_ALLOWED_CHARS)) {
 		throw PacketError("Illegal filename, data or hash");
 	}
 
-	verbosestream << "Server pushes media file \"" << filename << "\" with "
+	verbosestream << "Server pushes media file \"" << filename << "\" ";
-		<< filedata.size() << " bytes of data (cached=" << cached
+	if (filedata.empty())
-		<< ")" << std::endl;
+		verbosestream << "to be fetched ";
+	else
+		verbosestream << "with " << filedata.size() << " bytes ";
+	verbosestream << "(cached=" << cached << ")" << std::endl;
 
 	if (m_media_pushed_files.count(filename) != 0) {
-		// Silently ignore for synchronization purposes
+		// Ignore (but acknowledge). Previously this was for sync purposes,
+		// but even in new versions media cannot be replaced at runtime.
+		if (m_proto_ver >= 40)
+			sendHaveMedia({ token });
 		return;
 	}
 
-	// Compute and check checksum of data
+	if (!filedata.empty()) {
-	std::string computed_hash;
+		// LEGACY CODEPATH
-	{
+		// Compute and check checksum of data
-		SHA1 ctx;
+		std::string computed_hash;
-		ctx.addBytes(filedata.c_str(), filedata.size());
+		{
-		unsigned char *buf = ctx.getDigest();
+			SHA1 ctx;
-		computed_hash.assign((char*) buf, 20);
+			ctx.addBytes(filedata.c_str(), filedata.size());
-		free(buf);
+			unsigned char *buf = ctx.getDigest();
-	}
+			computed_hash.assign((char*) buf, 20);
-	if (raw_hash != computed_hash) {
+			free(buf);
-		verbosestream << "Hash of file data mismatches, ignoring." << std::endl;
+		}
+		if (raw_hash != computed_hash) {
+			verbosestream << "Hash of file data mismatches, ignoring." << std::endl;
+			return;
+		}
+
+		// Actually load media
+		loadMedia(filedata, filename, true);
+		m_media_pushed_files.insert(filename);
+
+		// Cache file for the next time when this client joins the same server
+		if (cached)
+			clientMediaUpdateCache(raw_hash, filedata);
 		return;
 	}
 
-	// Actually load media
-	loadMedia(filedata, filename, true);
 	m_media_pushed_files.insert(filename);
 
-	// Cache file for the next time when this client joins the same server
+	// create a downloader for this file
-	if (cached)
+	auto downloader = new SingleMediaDownloader(cached);
-		clientMediaUpdateCache(raw_hash, filedata);
+	m_pending_media_downloads.emplace_back(token, downloader);
+	downloader->addFile(filename, raw_hash);
+	for (const auto &baseurl : m_remote_media_servers)
+		downloader->addRemoteServer(baseurl);
+
+	downloader->step(this);
 }
 
 /*

src/network/networkprotocol.h

@@ -207,6 +207,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 		Minimap modes
 	PROTOCOL VERSION 40:
 		Added 'basic_debug' privilege
+		TOCLIENT_MEDIA_PUSH changed, TOSERVER_HAVE_MEDIA added
 */
 
 #define LATEST_PROTOCOL_VERSION 40
@@ -317,9 +318,8 @@ enum ToClientCommand
 	/*
 		std::string raw_hash
 		std::string filename
+		u32 callback_token
 		bool should_be_cached
-		u32 len
-		char filedata[len]
 	*/
 
 	// (oops, there is some gap here)
@@ -938,7 +938,13 @@ enum ToServerCommand
 		}
 	*/
 
-	TOSERVER_RECEIVED_MEDIA = 0x41, // Obsolete
+	TOSERVER_HAVE_MEDIA = 0x41,
+	/*
+		u8 number of callback tokens
+		for each:
+			u32 token
+	*/
+
 	TOSERVER_BREATH = 0x42, // Obsolete
 
 	TOSERVER_CLIENT_READY = 0x43,

src/network/serveropcodes.cpp

@@ -89,7 +89,7 @@ const ToServerCommandHandler toServerCommandTable[TOSERVER_NUM_MSG_TYPES] =
 	null_command_handler, // 0x3e
 	null_command_handler, // 0x3f
 	{ "TOSERVER_REQUEST_MEDIA",            TOSERVER_STATE_STARTUP, &Server::handleCommand_RequestMedia }, // 0x40
-	null_command_handler, // 0x41
+	{ "TOSERVER_HAVE_MEDIA",               TOSERVER_STATE_INGAME, &Server::handleCommand_HaveMedia }, // 0x41
 	null_command_handler, // 0x42
 	{ "TOSERVER_CLIENT_READY",             TOSERVER_STATE_STARTUP, &Server::handleCommand_ClientReady }, // 0x43
 	null_command_handler, // 0x44
@@ -167,7 +167,7 @@ const ClientCommandFactory clientCommandFactoryTable[TOCLIENT_NUM_MSG_TYPES] =
 	{ "TOCLIENT_TIME_OF_DAY",              0, true }, // 0x29
 	{ "TOCLIENT_CSM_RESTRICTION_FLAGS",    0, true }, // 0x2A
 	{ "TOCLIENT_PLAYER_SPEED",             0, true }, // 0x2B
-	{ "TOCLIENT_MEDIA_PUSH",               0, true }, // 0x2C (sent over channel 1 too)
+	{ "TOCLIENT_MEDIA_PUSH",               0, true }, // 0x2C (sent over channel 1 too if legacy)
 	null_command_factory, // 0x2D
 	null_command_factory, // 0x2E
 	{ "TOCLIENT_CHAT_MESSAGE",             0, true }, // 0x2F

src/network/serverpackethandler.cpp

@@ -362,16 +362,15 @@ void Server::handleCommand_RequestMedia(NetworkPacket* pkt)
 	session_t peer_id = pkt->getPeerId();
 	infostream << "Sending " << numfiles << " files to " <<
 		getPlayerName(peer_id) << std::endl;
-	verbosestream << "TOSERVER_REQUEST_MEDIA: " << std::endl;
+	verbosestream << "TOSERVER_REQUEST_MEDIA: requested file(s)" << std::endl;
 
 	for (u16 i = 0; i < numfiles; i++) {
 		std::string name;
 
 		*pkt >> name;
 
-		tosend.push_back(name);
+		tosend.emplace_back(name);
-		verbosestream << "TOSERVER_REQUEST_MEDIA: requested file "
+		verbosestream << "  " << name << std::endl;
-				<< name << std::endl;
 	}
 
 	sendRequestedMedia(peer_id, tosend);
@@ -1801,3 +1800,30 @@ void Server::handleCommand_ModChannelMsg(NetworkPacket *pkt)
 
 	broadcastModChannelMessage(channel_name, channel_msg, peer_id);
 }
+
+void Server::handleCommand_HaveMedia(NetworkPacket *pkt)
+{
+	std::vector<u32> tokens;
+	u8 numtokens;
+
+	*pkt >> numtokens;
+	for (u16 i = 0; i < numtokens; i++) {
+		u32 n;
+		*pkt >> n;
+		tokens.emplace_back(n);
+	}
+
+	const session_t peer_id = pkt->getPeerId();
+	auto player = m_env->getPlayer(peer_id);
+
+	for (const u32 token : tokens) {
+		auto it = m_pending_dyn_media.find(token);
+		if (it == m_pending_dyn_media.end())
+			continue;
+		if (it->second.waiting_players.count(peer_id)) {
+			it->second.waiting_players.erase(peer_id);
+			if (player)
+				getScriptIface()->on_dynamic_media_added(token, player->getName());
+		}
+	}
+}

src/script/cpp_api/s_server.cpp

@@ -20,6 +20,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "cpp_api/s_server.h"
 #include "cpp_api/s_internal.h"
 #include "common/c_converter.h"
+#include "util/numeric.h" // myrand
 
 bool ScriptApiServer::getAuth(const std::string &playername,
 		std::string *dst_password,
@@ -196,3 +197,68 @@ std::string ScriptApiServer::formatChatMessage(const std::string &name,
 
 	return ret;
 }
+
+u32 ScriptApiServer::allocateDynamicMediaCallback(int f_idx)
+{
+	lua_State *L = getStack();
+
+	if (f_idx < 0)
+		f_idx = lua_gettop(L) + f_idx + 1;
+
+	lua_getglobal(L, "core");
+	lua_getfield(L, -1, "dynamic_media_callbacks");
+	luaL_checktype(L, -1, LUA_TTABLE);
+
+	// Find a randomly generated token that doesn't exist yet
+	int tries = 100;
+	u32 token;
+	while (1) {
+		token = myrand();
+		lua_rawgeti(L, -2, token);
+		bool is_free = lua_isnil(L, -1);
+		lua_pop(L, 1);
+		if (is_free)
+			break;
+		if (--tries < 0)
+			FATAL_ERROR("Ran out of callbacks IDs?!");
+	}
+
+	// core.dynamic_media_callbacks[token] = callback_func
+	lua_pushvalue(L, f_idx);
+	lua_rawseti(L, -2, token);
+
+	lua_pop(L, 2);
+
+	verbosestream << "allocateDynamicMediaCallback() = " << token << std::endl;
+	return token;
+}
+
+void ScriptApiServer::freeDynamicMediaCallback(u32 token)
+{
+	lua_State *L = getStack();
+
+	verbosestream << "freeDynamicMediaCallback(" << token << ")" << std::endl;
+
+	// core.dynamic_media_callbacks[token] = nil
+	lua_getglobal(L, "core");
+	lua_getfield(L, -1, "dynamic_media_callbacks");
+	luaL_checktype(L, -1, LUA_TTABLE);
+	lua_pushnil(L);
+	lua_rawseti(L, -2, token);
+	lua_pop(L, 2);
+}
+
+void ScriptApiServer::on_dynamic_media_added(u32 token, const char *playername)
+{
+	SCRIPTAPI_PRECHECKHEADER
+
+	int error_handler = PUSH_ERROR_HANDLER(L);
+	lua_getglobal(L, "core");
+	lua_getfield(L, -1, "dynamic_media_callbacks");
+	luaL_checktype(L, -1, LUA_TTABLE);
+	lua_rawgeti(L, -1, token);
+	luaL_checktype(L, -1, LUA_TFUNCTION);
+
+	lua_pushstring(L, playername);
+	PCALL_RES(lua_pcall(L, 1, 0, error_handler));
+}

src/script/cpp_api/s_server.h

@@ -49,6 +49,12 @@ public:
 		const std::string &password);
 	bool setPassword(const std::string &playername,
 		const std::string &password);
+
+	/* dynamic media handling */
+	u32 allocateDynamicMediaCallback(int f_idx);
+	void freeDynamicMediaCallback(u32 token);
+	void on_dynamic_media_added(u32 token, const char *playername);
+
 private:
 	void getAuthHandler();
 	void readPrivileges(int index, std::set<std::string> &result);

src/script/lua_api/l_server.cpp

@@ -453,29 +453,37 @@ int ModApiServer::l_sound_fade(lua_State *L)
 }
 
 // dynamic_add_media(filepath)
-int ModApiServer::l_dynamic_add_media_raw(lua_State *L)
+int ModApiServer::l_dynamic_add_media(lua_State *L)
 {
 	NO_MAP_LOCK_REQUIRED;
 
 	if (!getEnv(L))
 		throw LuaError("Dynamic media cannot be added before server has started up");
+	Server *server = getServer(L);
+
+	std::string filepath;
+	std::string to_player;
+	bool ephemeral = false;
+
+	if (lua_istable(L, 1)) {
+		getstringfield(L, 1, "filepath", filepath);
+		getstringfield(L, 1, "to_player", to_player);
+		getboolfield(L, 1, "ephemeral", ephemeral);
+	} else {
+		filepath = readParam<std::string>(L, 1);
+	}
+	if (filepath.empty())
+		luaL_typerror(L, 1, "non-empty string");
+	luaL_checktype(L, 2, LUA_TFUNCTION);
 
-	std::string filepath = readParam<std::string>(L, 1);
 	CHECK_SECURE_PATH(L, filepath.c_str(), false);
 
-	std::vector<RemotePlayer*> sent_to;
+	u32 token = server->getScriptIface()->allocateDynamicMediaCallback(2);
-	bool ok = getServer(L)->dynamicAddMedia(filepath, sent_to);
+
-	if (ok) {
+	bool ok = server->dynamicAddMedia(filepath, token, to_player, ephemeral);
-		// (see wrapper code in builtin)
+	if (!ok)
-		lua_createtable(L, sent_to.size(), 0);
+		server->getScriptIface()->freeDynamicMediaCallback(token);
-		int i = 0;
+	lua_pushboolean(L, ok);
-		for (RemotePlayer *player : sent_to) {
-			lua_pushstring(L, player->getName());
-			lua_rawseti(L, -2, ++i);
-		}
-	} else {
-		lua_pushboolean(L, false);
-	}
 
 	return 1;
 }
@@ -519,7 +527,7 @@ void ModApiServer::Initialize(lua_State *L, int top)
 	API_FCT(sound_play);
 	API_FCT(sound_stop);
 	API_FCT(sound_fade);
-	API_FCT(dynamic_add_media_raw);
+	API_FCT(dynamic_add_media);
 
 	API_FCT(get_player_information);
 	API_FCT(get_player_privs);

src/script/lua_api/l_server.h

@@ -71,7 +71,7 @@ private:
 	static int l_sound_fade(lua_State *L);
 
 	// dynamic_add_media(filepath)
-	static int l_dynamic_add_media_raw(lua_State *L);
+	static int l_dynamic_add_media(lua_State *L);
 
 	// get_player_privs(name, text)
 	static int l_get_player_privs(lua_State *L);

src/server.cpp

@@ -665,6 +665,17 @@ void Server::AsyncRunStep(bool initial_step)
 	} else {
 		m_lag_gauge->increment(dtime/100);
 	}
+
+	{
+		float &counter = m_step_pending_dyn_media_timer;
+		counter += dtime;
+		if (counter >= 5.0f) {
+			stepPendingDynMediaCallbacks(counter);
+			counter = 0;
+		}
+	}
+
+
 #if USE_CURL
 	// send masterserver announce
 	{
@@ -2527,6 +2538,8 @@ void Server::sendMediaAnnouncement(session_t peer_id, const std::string &lang_co
 	std::string lang_suffix;
 	lang_suffix.append(".").append(lang_code).append(".tr");
 	for (const auto &i : m_media) {
+		if (i.second.no_announce)
+			continue;
 		if (str_ends_with(i.first, ".tr") && !str_ends_with(i.first, lang_suffix))
 			continue;
 		media_sent++;
@@ -2535,6 +2548,8 @@ void Server::sendMediaAnnouncement(session_t peer_id, const std::string &lang_co
 	pkt << media_sent;
 
 	for (const auto &i : m_media) {
+		if (i.second.no_announce)
+			continue;
 		if (str_ends_with(i.first, ".tr") && !str_ends_with(i.first, lang_suffix))
 			continue;
 		pkt << i.first << i.second.sha1_digest;
@@ -2553,11 +2568,9 @@ struct SendableMedia
 	std::string path;
 	std::string data;
 
-	SendableMedia(const std::string &name_="", const std::string &path_="",
+	SendableMedia(const std::string &name, const std::string &path,
-	              const std::string &data_=""):
+			std::string &&data):
-		name(name_),
+		name(name), path(path), data(std::move(data))
-		path(path_),
-		data(data_)
 	{}
 };
 
@@ -2584,40 +2597,19 @@ void Server::sendRequestedMedia(session_t peer_id,
 			continue;
 		}
 
-		//TODO get path + name
+		const auto &m = m_media[name];
-		std::string tpath = m_media[name].path;
 
 		// Read data
-		std::ifstream fis(tpath.c_str(), std::ios_base::binary);
+		std::string data;
-		if(!fis.good()){
+		if (!fs::ReadFile(m.path, data)) {
-			errorstream<<"Server::sendRequestedMedia(): Could not open \""
+			errorstream << "Server::sendRequestedMedia(): Failed to read \""
-					<<tpath<<"\" for reading"<<std::endl;
+					<< name << "\"" << std::endl;
 			continue;
 		}
-		std::ostringstream tmp_os(std::ios_base::binary);
+		file_size_bunch_total += data.size();
-		bool bad = false;
+
-		for(;;) {
-			char buf[1024];
-			fis.read(buf, 1024);
-			std::streamsize len = fis.gcount();
-			tmp_os.write(buf, len);
-			file_size_bunch_total += len;
-			if(fis.eof())
-				break;
-			if(!fis.good()) {
-				bad = true;
-				break;
-			}
-		}
-		if (bad) {
-			errorstream<<"Server::sendRequestedMedia(): Failed to read \""
-					<<name<<"\""<<std::endl;
-			continue;
-		}
-		/*infostream<<"Server::sendRequestedMedia(): Loaded \""
-				<<tname<<"\""<<std::endl;*/
 		// Put in list
-		file_bunches[file_bunches.size()-1].emplace_back(name, tpath, tmp_os.str());
+		file_bunches.back().emplace_back(name, m.path, std::move(data));
 
 		// Start next bunch if got enough data
 		if(file_size_bunch_total >= bytes_per_bunch) {
@@ -2660,6 +2652,33 @@ void Server::sendRequestedMedia(session_t peer_id,
 	}
 }
 
+void Server::stepPendingDynMediaCallbacks(float dtime)
+{
+	MutexAutoLock lock(m_env_mutex);
+
+	for (auto it = m_pending_dyn_media.begin(); it != m_pending_dyn_media.end();) {
+		it->second.expiry_timer -= dtime;
+		bool del = it->second.waiting_players.empty() || it->second.expiry_timer < 0;
+
+		if (!del) {
+			it++;
+			continue;
+		}
+
+		const auto &name = it->second.filename;
+		if (!name.empty()) {
+			assert(m_media.count(name));
+			// if no_announce isn't set we're definitely deleting the wrong file!
+			sanity_check(m_media[name].no_announce);
+
+			fs::DeleteSingleFileOrEmptyDirectory(m_media[name].path);
+			m_media.erase(name);
+		}
+		getScriptIface()->freeDynamicMediaCallback(it->first);
+		it = m_pending_dyn_media.erase(it);
+	}
+}
+
 void Server::SendMinimapModes(session_t peer_id,
 		std::vector<MinimapMode> &modes, size_t wanted_mode)
 {
@@ -3457,14 +3476,18 @@ void Server::deleteParticleSpawner(const std::string &playername, u32 id)
 	SendDeleteParticleSpawner(peer_id, id);
 }
 
-bool Server::dynamicAddMedia(const std::string &filepath,
+bool Server::dynamicAddMedia(std::string filepath,
-	std::vector<RemotePlayer*> &sent_to)
+	const u32 token, const std::string &to_player, bool ephemeral)
 {
 	std::string filename = fs::GetFilenameFromPath(filepath.c_str());
-	if (m_media.find(filename) != m_media.end()) {
+	auto it = m_media.find(filename);
-		errorstream << "Server::dynamicAddMedia(): file \"" << filename
+	if (it != m_media.end()) {
-			<< "\" already exists in media cache" << std::endl;
+		// Allow the same path to be "added" again in certain conditions
-		return false;
+		if (ephemeral || it->second.path != filepath) {
+			errorstream << "Server::dynamicAddMedia(): file \"" << filename
+				<< "\" already exists in media cache" << std::endl;
+			return false;
+		}
 	}
 
 	// Load the file and add it to our media cache
@@ -3473,35 +3496,91 @@ bool Server::dynamicAddMedia(const std::string &filepath,
 	if (!ok)
 		return false;
 
+	if (ephemeral) {
+		// Create a copy of the file and swap out the path, this removes the
+		// requirement that mods keep the file accessible at the original path.
+		filepath = fs::CreateTempFile();
+		bool ok = ([&] () -> bool {
+			if (filepath.empty())
+				return false;
+			std::ofstream os(filepath.c_str(), std::ios::binary);
+			if (!os.good())
+				return false;
+			os << filedata;
+			os.close();
+			return !os.fail();
+		})();
+		if (!ok) {
+			errorstream << "Server: failed to create a copy of media file "
+				<< "\"" << filename << "\"" << std::endl;
+			m_media.erase(filename);
+			return false;
+		}
+		verbosestream << "Server: \"" << filename << "\" temporarily copied to "
+			<< filepath << std::endl;
+
+		m_media[filename].path = filepath;
+		m_media[filename].no_announce = true;
+		// stepPendingDynMediaCallbacks will clean this up later.
+	} else if (!to_player.empty()) {
+		m_media[filename].no_announce = true;
+	}
+
 	// Push file to existing clients
 	NetworkPacket pkt(TOCLIENT_MEDIA_PUSH, 0);
-	pkt << raw_hash << filename << (bool) true;
+	pkt << raw_hash << filename << (bool)ephemeral;
-	pkt.putLongString(filedata);
 
+	NetworkPacket legacy_pkt = pkt;
+
+	// Newer clients get asked to fetch the file (asynchronous)
+	pkt << token;
+	// Older clients have an awful hack that just throws the data at them
+	legacy_pkt.putLongString(filedata);
+
+	std::unordered_set<session_t> delivered, waiting;
 	m_clients.lock();
 	for (auto &pair : m_clients.getClientList()) {
 		if (pair.second->getState() < CS_DefinitionsSent)
 			continue;
-		if (pair.second->net_proto_version < 39)
+		const auto proto_ver = pair.second->net_proto_version;
+		if (proto_ver < 39)
 			continue;
 
-		if (auto player = m_env->getPlayer(pair.second->peer_id))
+		const session_t peer_id = pair.second->peer_id;
-			sent_to.emplace_back(player);
+		if (!to_player.empty() && getPlayerName(peer_id) != to_player)
-		/*
+			continue;
-			FIXME: this is a very awful hack
+
-			The network layer only guarantees ordered delivery inside a channel.
+		if (proto_ver < 40) {
-			Since the very next packet could be one that uses the media, we have
+			delivered.emplace(peer_id);
-			to push the media over ALL channels to ensure it is processed before
+			/*
-			it is used.
+				The network layer only guarantees ordered delivery inside a channel.
-			In practice this means we have to send it twice:
+				Since the very next packet could be one that uses the media, we have
-			- channel 1 (HUD)
+				to push the media over ALL channels to ensure it is processed before
-			- channel 0 (everything else: e.g. play_sound, object messages)
+				it is used. In practice this means channels 1 and 0.
-		*/
+			*/
-		m_clients.send(pair.second->peer_id, 1, &pkt, true);
+			m_clients.send(peer_id, 1, &legacy_pkt, true);
-		m_clients.send(pair.second->peer_id, 0, &pkt, true);
+			m_clients.send(peer_id, 0, &legacy_pkt, true);
+		} else {
+			waiting.emplace(peer_id);
+			Send(peer_id, &pkt);
+		}
 	}
 	m_clients.unlock();
 
+	// Run callback for players that already had the file delivered (legacy-only)
+	for (session_t peer_id : delivered) {
+		if (auto player = m_env->getPlayer(peer_id))
+			getScriptIface()->on_dynamic_media_added(token, player->getName());
+	}
+
+	// Save all others in our pending state
+	auto &state = m_pending_dyn_media[token];
+	state.waiting_players = std::move(waiting);
+	// regardless of success throw away the callback after a while
+	state.expiry_timer = 60.0f;
+	if (ephemeral)
+		state.filename = filename;
+
 	return true;
 }
 

src/server.h

@@ -43,6 +43,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include <list>
 #include <map>
 #include <vector>
+#include <unordered_set>
 
 class ChatEvent;
 struct ChatEventChat;
@@ -81,12 +82,14 @@ enum ClientDeletionReason {
 struct MediaInfo
 {
 	std::string path;
-	std::string sha1_digest;
+	std::string sha1_digest; // base64-encoded
+	bool no_announce; // true: not announced in TOCLIENT_ANNOUNCE_MEDIA (at player join)
 
 	MediaInfo(const std::string &path_="",
 	          const std::string &sha1_digest_=""):
 		path(path_),
-		sha1_digest(sha1_digest_)
+		sha1_digest(sha1_digest_),
+		no_announce(false)
 	{
 	}
 };
@@ -197,6 +200,7 @@ public:
 	void handleCommand_FirstSrp(NetworkPacket* pkt);
 	void handleCommand_SrpBytesA(NetworkPacket* pkt);
 	void handleCommand_SrpBytesM(NetworkPacket* pkt);
+	void handleCommand_HaveMedia(NetworkPacket *pkt);
 
 	void ProcessData(NetworkPacket *pkt);
 
@@ -257,7 +261,8 @@ public:
 
 	void deleteParticleSpawner(const std::string &playername, u32 id);
 
-	bool dynamicAddMedia(const std::string &filepath, std::vector<RemotePlayer*> &sent_to);
+	bool dynamicAddMedia(std::string filepath, u32 token,
+		const std::string &to_player, bool ephemeral);
 
 	ServerInventoryManager *getInventoryMgr() const { return m_inventory_mgr.get(); }
 	void sendDetachedInventory(Inventory *inventory, const std::string &name, session_t peer_id);
@@ -395,6 +400,12 @@ private:
 			float m_timer = 0.0f;
 	};
 
+	struct PendingDynamicMediaCallback {
+		std::string filename; // only set if media entry and file is to be deleted
+		float expiry_timer;
+		std::unordered_set<session_t> waiting_players;
+	};
+
 	void init();
 
 	void SendMovement(session_t peer_id);
@@ -466,6 +477,7 @@ private:
 	void sendMediaAnnouncement(session_t peer_id, const std::string &lang_code);
 	void sendRequestedMedia(session_t peer_id,
 			const std::vector<std::string> &tosend);
+	void stepPendingDynMediaCallbacks(float dtime);
 
 	// Adds a ParticleSpawner on peer with peer_id (PEER_ID_INEXISTENT == all)
 	void SendAddParticleSpawner(session_t peer_id, u16 protocol_version,
@@ -650,6 +662,10 @@ private:
 	// media files known to server
 	std::unordered_map<std::string, MediaInfo> m_media;
 
+	// pending dynamic media callbacks, clients inform the server when they have a file fetched
+	std::unordered_map<u32, PendingDynamicMediaCallback> m_pending_dyn_media;
+	float m_step_pending_dyn_media_timer = 0.0f;
+
 	/*
 		Sounds
 	*/