From bbfae0cc673d3abdc21224c53e09b209ee4688a2 Mon Sep 17 00:00:00 2001 From: sfan5 Date: Thu, 9 Sep 2021 16:51:35 +0200 Subject: [PATCH] Dynamic_Add_Media v2 (#11550) --- builtin/game/misc.lua | 23 +-- doc/lua_api.txt | 37 ++-- src/client/client.cpp | 39 ++++- src/client/client.h | 6 + src/client/clientmedia.cpp | 252 +++++++++++++++++++++++----- src/client/clientmedia.h | 159 ++++++++++++++---- src/filesys.cpp | 12 ++ src/filesys.h | 4 + src/network/clientopcodes.cpp | 2 +- src/network/clientpackethandler.cpp | 129 ++++++++------ src/network/networkprotocol.h | 12 +- src/network/serveropcodes.cpp | 4 +- src/network/serverpackethandler.cpp | 34 +++- src/script/cpp_api/s_server.cpp | 66 ++++++++ src/script/cpp_api/s_server.h | 6 + src/script/lua_api/l_server.cpp | 40 +++-- src/script/lua_api/l_server.h | 2 +- src/server.cpp | 193 ++++++++++++++------- src/server.h | 22 ++- 19 files changed, 796 insertions(+), 246 deletions(-) diff --git a/builtin/game/misc.lua b/builtin/game/misc.lua index aac6c2d18..63d64817c 100644 --- a/builtin/game/misc.lua +++ b/builtin/game/misc.lua @@ -269,27 +269,8 @@ function core.cancel_shutdown_requests() end --- Callback handling for dynamic_add_media - -local dynamic_add_media_raw = core.dynamic_add_media_raw -core.dynamic_add_media_raw = nil -function core.dynamic_add_media(filepath, callback) - local ret = dynamic_add_media_raw(filepath) - if ret == false then - return ret - end - if callback == nil then - core.log("deprecated", "Calling minetest.dynamic_add_media without ".. - "a callback is deprecated and will stop working in future versions.") - else - -- At the moment async loading is not actually implemented, so we - -- immediately call the callback ourselves - for _, name in ipairs(ret) do - callback(name) - end - end - return true -end +-- Used for callback handling with dynamic_add_media +core.dynamic_media_callbacks = {} -- PNG encoder safety wrapper diff --git a/doc/lua_api.txt b/doc/lua_api.txt index e99c1d1e6..3a1a3f02f 100644 --- a/doc/lua_api.txt +++ b/doc/lua_api.txt @@ -5649,22 +5649,33 @@ Server * Returns a code (0: successful, 1: no such player, 2: player is connected) * `minetest.remove_player_auth(name)`: remove player authentication data * Returns boolean indicating success (false if player nonexistant) -* `minetest.dynamic_add_media(filepath, callback)` - * `filepath`: path to a media file on the filesystem - * `callback`: function with arguments `name`, where name is a player name - (previously there was no callback argument; omitting it is deprecated) - * Adds the file to the media sent to clients by the server on startup - and also pushes this file to already connected clients. - The file must be a supported image, sound or model format. It must not be - modified, deleted, moved or renamed after calling this function. - The list of dynamically added media is not persisted. +* `minetest.dynamic_add_media(options, callback)` + * `options`: table containing the following parameters + * `filepath`: path to a media file on the filesystem + * `to_player`: name of the player the media should be sent to instead of + all players (optional) + * `ephemeral`: boolean that marks the media as ephemeral, + it will not be cached on the client (optional, default false) + * `callback`: function with arguments `name`, which is a player name + * Pushes the specified media file to client(s). (details below) + The file must be a supported image, sound or model format. + Dynamically added media is not persisted between server restarts. * Returns false on error, true if the request was accepted * The given callback will be called for every player as soon as the media is available on the client. - Old clients that lack support for this feature will not see the media - unless they reconnect to the server. (callback won't be called) - * Since media transferred this way currently does not use client caching - or HTTP transfers, dynamic media should not be used with big files. + * Details/Notes: + * If `ephemeral`=false and `to_player` is unset the file is added to the media + sent to clients on startup, this means the media will appear even on + old clients if they rejoin the server. + * If `ephemeral`=false the file must not be modified, deleted, moved or + renamed after calling this function. + * Regardless of any use of `ephemeral`, adding media files with the same + name twice is not possible/guaranteed to work. An exception to this is the + use of `to_player` to send the same, already existent file to multiple + chosen players. + * Clients will attempt to fetch files added this way via remote media, + this can make transfer of bigger files painless (if set up). Nevertheless + it is advised not to use dynamic media for big media files. Bans ---- diff --git a/src/client/client.cpp b/src/client/client.cpp index 3c5559fca..13ff22e8e 100644 --- a/src/client/client.cpp +++ b/src/client/client.cpp @@ -555,6 +555,29 @@ void Client::step(float dtime) m_media_downloader = NULL; } } + { + // Acknowledge dynamic media downloads to server + std::vector done; + for (auto it = m_pending_media_downloads.begin(); + it != m_pending_media_downloads.end();) { + assert(it->second->isStarted()); + it->second->step(this); + if (it->second->isDone()) { + done.emplace_back(it->first); + + it = m_pending_media_downloads.erase(it); + } else { + it++; + } + + if (done.size() == 255) { // maximum in one packet + sendHaveMedia(done); + done.clear(); + } + } + if (!done.empty()) + sendHaveMedia(done); + } /* If the server didn't update the inventory in a while, revert @@ -770,7 +793,8 @@ void Client::request_media(const std::vector &file_requests) Send(&pkt); infostream << "Client: Sending media request list to server (" - << file_requests.size() << " files. packet size)" << std::endl; + << file_requests.size() << " files, packet size " + << pkt.getSize() << ")" << std::endl; } void Client::initLocalMapSaving(const Address &address, @@ -1295,6 +1319,19 @@ void Client::sendPlayerPos() Send(&pkt); } +void Client::sendHaveMedia(const std::vector &tokens) +{ + NetworkPacket pkt(TOSERVER_HAVE_MEDIA, 1 + tokens.size() * 4); + + sanity_check(tokens.size() < 256); + + pkt << static_cast(tokens.size()); + for (u32 token : tokens) + pkt << token; + + Send(&pkt); +} + void Client::removeNode(v3s16 p) { std::map modified_blocks; diff --git a/src/client/client.h b/src/client/client.h index 85ca24049..c1a38ba48 100644 --- a/src/client/client.h +++ b/src/client/client.h @@ -53,6 +53,7 @@ class ISoundManager; class NodeDefManager; //class IWritableCraftDefManager; class ClientMediaDownloader; +class SingleMediaDownloader; struct MapDrawControl; class ModChannelMgr; class MtEventManager; @@ -245,6 +246,7 @@ public: void sendDamage(u16 damage); void sendRespawn(); void sendReady(); + void sendHaveMedia(const std::vector &tokens); ClientEnvironment& getEnv() { return m_env; } ITextureSource *tsrc() { return getTextureSource(); } @@ -536,9 +538,13 @@ private: bool m_activeobjects_received = false; bool m_mods_loaded = false; + std::vector m_remote_media_servers; + // Media downloader, only exists during init ClientMediaDownloader *m_media_downloader; // Set of media filenames pushed by server at runtime std::unordered_set m_media_pushed_files; + // Pending downloads of dynamic media (key: token) + std::vector>> m_pending_media_downloads; // time_of_day speed approximation for old protocol bool m_time_of_day_set = false; diff --git a/src/client/clientmedia.cpp b/src/client/clientmedia.cpp index 0f9ba5356..6c5d4a8bf 100644 --- a/src/client/clientmedia.cpp +++ b/src/client/clientmedia.cpp @@ -49,7 +49,6 @@ bool clientMediaUpdateCache(const std::string &raw_hash, const std::string &file */ ClientMediaDownloader::ClientMediaDownloader(): - m_media_cache(getMediaCacheDir()), m_httpfetch_caller(HTTPFETCH_DISCARD) { } @@ -66,6 +65,12 @@ ClientMediaDownloader::~ClientMediaDownloader() delete remote; } +bool ClientMediaDownloader::loadMedia(Client *client, const std::string &data, + const std::string &name) +{ + return client->loadMedia(data, name); +} + void ClientMediaDownloader::addFile(const std::string &name, const std::string &sha1) { assert(!m_initial_step_done); // pre-condition @@ -105,7 +110,7 @@ void ClientMediaDownloader::addRemoteServer(const std::string &baseurl) { assert(!m_initial_step_done); // pre-condition - #ifdef USE_CURL +#ifdef USE_CURL if (g_settings->getBool("enable_remote_media_server")) { infostream << "Client: Adding remote server \"" @@ -117,13 +122,13 @@ void ClientMediaDownloader::addRemoteServer(const std::string &baseurl) m_remotes.push_back(remote); } - #else +#else infostream << "Client: Ignoring remote server \"" << baseurl << "\" because cURL support is not compiled in" << std::endl; - #endif +#endif } void ClientMediaDownloader::step(Client *client) @@ -172,36 +177,21 @@ void ClientMediaDownloader::initialStep(Client *client) // Check media cache m_uncached_count = m_files.size(); for (auto &file_it : m_files) { - std::string name = file_it.first; + const std::string &name = file_it.first; FileStatus *filestatus = file_it.second; const std::string &sha1 = filestatus->sha1; - std::ostringstream tmp_os(std::ios_base::binary); - bool found_in_cache = m_media_cache.load(hex_encode(sha1), tmp_os); - - // If found in cache, try to load it from there - if (found_in_cache) { - bool success = checkAndLoad(name, sha1, - tmp_os.str(), true, client); - if (success) { - filestatus->received = true; - m_uncached_count--; - } + if (tryLoadFromCache(name, sha1, client)) { + filestatus->received = true; + m_uncached_count--; } } assert(m_uncached_received_count == 0); // Create the media cache dir if we are likely to write to it - if (m_uncached_count != 0) { - bool did = fs::CreateAllDirs(getMediaCacheDir()); - if (!did) { - errorstream << "Client: " - << "Could not create media cache directory: " - << getMediaCacheDir() - << std::endl; - } - } + if (m_uncached_count != 0) + createCacheDirs(); // If we found all files in the cache, report this fact to the server. // If the server reported no remote servers, immediately start @@ -301,8 +291,7 @@ void ClientMediaDownloader::remoteHashSetReceived( // available on this server, add this server // to the available_remotes array - for(std::map::iterator - it = m_files.upper_bound(m_name_bound); + for(auto it = m_files.upper_bound(m_name_bound); it != m_files.end(); ++it) { FileStatus *f = it->second; if (!f->received && sha1_set.count(f->sha1)) @@ -328,8 +317,7 @@ void ClientMediaDownloader::remoteMediaReceived( std::string name; { - std::unordered_map::iterator it = - m_remote_file_transfers.find(fetch_result.request_id); + auto it = m_remote_file_transfers.find(fetch_result.request_id); assert(it != m_remote_file_transfers.end()); name = it->second; m_remote_file_transfers.erase(it); @@ -398,8 +386,7 @@ void ClientMediaDownloader::startRemoteMediaTransfers() { bool changing_name_bound = true; - for (std::map::iterator - files_iter = m_files.upper_bound(m_name_bound); + for (auto files_iter = m_files.upper_bound(m_name_bound); files_iter != m_files.end(); ++files_iter) { // Abort if active fetch limit is exceeded @@ -477,19 +464,18 @@ void ClientMediaDownloader::startConventionalTransfers(Client *client) } } -void ClientMediaDownloader::conventionalTransferDone( +bool ClientMediaDownloader::conventionalTransferDone( const std::string &name, const std::string &data, Client *client) { // Check that file was announced - std::map::iterator - file_iter = m_files.find(name); + auto file_iter = m_files.find(name); if (file_iter == m_files.end()) { errorstream << "Client: server sent media file that was" << "not announced, ignoring it: \"" << name << "\"" << std::endl; - return; + return false; } FileStatus *filestatus = file_iter->second; assert(filestatus != NULL); @@ -499,7 +485,7 @@ void ClientMediaDownloader::conventionalTransferDone( errorstream << "Client: server sent media file that we already" << "received, ignoring it: \"" << name << "\"" << std::endl; - return; + return true; } // Mark file as received, regardless of whether loading it works and @@ -512,9 +498,45 @@ void ClientMediaDownloader::conventionalTransferDone( // Check that received file matches announced checksum // If so, load it checkAndLoad(name, filestatus->sha1, data, false, client); + + return true; } -bool ClientMediaDownloader::checkAndLoad( +/* + IClientMediaDownloader +*/ + +IClientMediaDownloader::IClientMediaDownloader(): + m_media_cache(getMediaCacheDir()), m_write_to_cache(true) +{ +} + +void IClientMediaDownloader::createCacheDirs() +{ + if (!m_write_to_cache) + return; + + std::string path = getMediaCacheDir(); + if (!fs::CreateAllDirs(path)) { + errorstream << "Client: Could not create media cache directory: " + << path << std::endl; + } +} + +bool IClientMediaDownloader::tryLoadFromCache(const std::string &name, + const std::string &sha1, Client *client) +{ + std::ostringstream tmp_os(std::ios_base::binary); + bool found_in_cache = m_media_cache.load(hex_encode(sha1), tmp_os); + + // If found in cache, try to load it from there + if (found_in_cache) + return checkAndLoad(name, sha1, tmp_os.str(), true, client); + + return false; +} + +bool IClientMediaDownloader::checkAndLoad( const std::string &name, const std::string &sha1, const std::string &data, bool is_from_cache, Client *client) { @@ -544,7 +566,7 @@ bool ClientMediaDownloader::checkAndLoad( } // Checksum is ok, try loading the file - bool success = client->loadMedia(data, name); + bool success = loadMedia(client, data, name); if (!success) { infostream << "Client: " << "Failed to load " << cached_or_received << " media: " @@ -559,7 +581,7 @@ bool ClientMediaDownloader::checkAndLoad( << std::endl; // Update cache (unless we just loaded the file from the cache) - if (!is_from_cache) + if (!is_from_cache && m_write_to_cache) m_media_cache.update(sha1_hex, data); return true; @@ -587,12 +609,10 @@ std::string ClientMediaDownloader::serializeRequiredHashSet() // Write list of hashes of files that have not been // received (found in cache) yet - for (std::map::iterator - it = m_files.begin(); - it != m_files.end(); ++it) { - if (!it->second->received) { - FATAL_ERROR_IF(it->second->sha1.size() != 20, "Invalid SHA1 size"); - os << it->second->sha1; + for (const auto &it : m_files) { + if (!it.second->received) { + FATAL_ERROR_IF(it.second->sha1.size() != 20, "Invalid SHA1 size"); + os << it.second->sha1; } } @@ -628,3 +648,145 @@ void ClientMediaDownloader::deSerializeHashSet(const std::string &data, result.insert(data.substr(pos, 20)); } } + +/* + SingleMediaDownloader +*/ + +SingleMediaDownloader::SingleMediaDownloader(bool write_to_cache): + m_httpfetch_caller(HTTPFETCH_DISCARD) +{ + m_write_to_cache = write_to_cache; +} + +SingleMediaDownloader::~SingleMediaDownloader() +{ + if (m_httpfetch_caller != HTTPFETCH_DISCARD) + httpfetch_caller_free(m_httpfetch_caller); +} + +bool SingleMediaDownloader::loadMedia(Client *client, const std::string &data, + const std::string &name) +{ + return client->loadMedia(data, name, true); +} + +void SingleMediaDownloader::addFile(const std::string &name, const std::string &sha1) +{ + assert(m_stage == STAGE_INIT); // pre-condition + + assert(!name.empty()); + assert(sha1.size() == 20); + + FATAL_ERROR_IF(!m_file_name.empty(), "Cannot add a second file"); + m_file_name = name; + m_file_sha1 = sha1; +} + +void SingleMediaDownloader::addRemoteServer(const std::string &baseurl) +{ + assert(m_stage == STAGE_INIT); // pre-condition + + if (g_settings->getBool("enable_remote_media_server")) + m_remotes.emplace_back(baseurl); +} + +void SingleMediaDownloader::step(Client *client) +{ + if (m_stage == STAGE_INIT) { + m_stage = STAGE_CACHE_CHECKED; + initialStep(client); + } + + // Remote media: check for completion of fetches + if (m_httpfetch_caller != HTTPFETCH_DISCARD) { + HTTPFetchResult fetch_result; + while (httpfetch_async_get(m_httpfetch_caller, fetch_result)) { + remoteMediaReceived(fetch_result, client); + } + } +} + +bool SingleMediaDownloader::conventionalTransferDone(const std::string &name, + const std::string &data, Client *client) +{ + if (name != m_file_name) + return false; + + // Mark file as received unconditionally and try to load it + m_stage = STAGE_DONE; + checkAndLoad(name, m_file_sha1, data, false, client); + return true; +} + +void SingleMediaDownloader::initialStep(Client *client) +{ + if (tryLoadFromCache(m_file_name, m_file_sha1, client)) + m_stage = STAGE_DONE; + if (isDone()) + return; + + createCacheDirs(); + + // If the server reported no remote servers, immediately fall back to + // conventional transfer. + if (!USE_CURL || m_remotes.empty()) { + startConventionalTransfer(client); + } else { + // Otherwise start by requesting the file from the first remote media server + m_httpfetch_caller = httpfetch_caller_alloc(); + m_current_remote = 0; + startRemoteMediaTransfer(); + } +} + +void SingleMediaDownloader::remoteMediaReceived( + const HTTPFetchResult &fetch_result, Client *client) +{ + sanity_check(!isDone()); + sanity_check(m_current_remote >= 0); + + // If fetch succeeded, try to load it + if (fetch_result.succeeded) { + bool success = checkAndLoad(m_file_name, m_file_sha1, + fetch_result.data, false, client); + if (success) { + m_stage = STAGE_DONE; + return; + } + } + + // Otherwise try the next remote server or fall back to conventional transfer + m_current_remote++; + if (m_current_remote >= (int)m_remotes.size()) { + infostream << "Client: Failed to remote-fetch \"" << m_file_name + << "\". Requesting it the usual way." << std::endl; + m_current_remote = -1; + startConventionalTransfer(client); + } else { + startRemoteMediaTransfer(); + } +} + +void SingleMediaDownloader::startRemoteMediaTransfer() +{ + std::string url = m_remotes.at(m_current_remote) + hex_encode(m_file_sha1); + verbosestream << "Client: Requesting remote media file " + << "\"" << m_file_name << "\" " << "\"" << url << "\"" << std::endl; + + HTTPFetchRequest fetch_request; + fetch_request.url = url; + fetch_request.caller = m_httpfetch_caller; + fetch_request.request_id = m_httpfetch_next_id; + fetch_request.timeout = g_settings->getS32("curl_file_download_timeout"); + httpfetch_async(fetch_request); + + m_httpfetch_next_id++; +} + +void SingleMediaDownloader::startConventionalTransfer(Client *client) +{ + std::vector requests; + requests.emplace_back(m_file_name); + client->request_media(requests); +} diff --git a/src/client/clientmedia.h b/src/client/clientmedia.h index e97a0f24b..aa7b0f398 100644 --- a/src/client/clientmedia.h +++ b/src/client/clientmedia.h @@ -21,6 +21,7 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "irrlichttypes.h" #include "filecache.h" +#include "util/basic_macros.h" #include #include #include @@ -38,7 +39,62 @@ struct HTTPFetchResult; bool clientMediaUpdateCache(const std::string &raw_hash, const std::string &filedata); -class ClientMediaDownloader +// more of a base class than an interface but this name was most convenient... +class IClientMediaDownloader +{ +public: + DISABLE_CLASS_COPY(IClientMediaDownloader) + + virtual bool isStarted() const = 0; + + // If this returns true, the downloader is done and can be deleted + virtual bool isDone() const = 0; + + // Add a file to the list of required file (but don't fetch it yet) + virtual void addFile(const std::string &name, const std::string &sha1) = 0; + + // Add a remote server to the list; ignored if not built with cURL + virtual void addRemoteServer(const std::string &baseurl) = 0; + + // Steps the media downloader: + // - May load media into client by calling client->loadMedia() + // - May check media cache for files + // - May add files to media cache + // - May start remote transfers by calling httpfetch_async + // - May check for completion of current remote transfers + // - May start conventional transfers by calling client->request_media() + // - May inform server that all media has been loaded + // by calling client->received_media() + // After step has been called once, don't call addFile/addRemoteServer. + virtual void step(Client *client) = 0; + + // Must be called for each file received through TOCLIENT_MEDIA + // returns true if this file belongs to this downloader + virtual bool conventionalTransferDone(const std::string &name, + const std::string &data, Client *client) = 0; + +protected: + IClientMediaDownloader(); + virtual ~IClientMediaDownloader() = default; + + // Forwards the call to the appropriate Client method + virtual bool loadMedia(Client *client, const std::string &data, + const std::string &name) = 0; + + void createCacheDirs(); + + bool tryLoadFromCache(const std::string &name, const std::string &sha1, + Client *client); + + bool checkAndLoad(const std::string &name, const std::string &sha1, + const std::string &data, bool is_from_cache, Client *client); + + // Filesystem-based media cache + FileCache m_media_cache; + bool m_write_to_cache; +}; + +class ClientMediaDownloader : public IClientMediaDownloader { public: ClientMediaDownloader(); @@ -52,39 +108,29 @@ public: return 0.0f; } - bool isStarted() const { + bool isStarted() const override { return m_initial_step_done; } - // If this returns true, the downloader is done and can be deleted - bool isDone() const { + bool isDone() const override { return m_initial_step_done && m_uncached_received_count == m_uncached_count; } - // Add a file to the list of required file (but don't fetch it yet) - void addFile(const std::string &name, const std::string &sha1); + void addFile(const std::string &name, const std::string &sha1) override; - // Add a remote server to the list; ignored if not built with cURL - void addRemoteServer(const std::string &baseurl); + void addRemoteServer(const std::string &baseurl) override; - // Steps the media downloader: - // - May load media into client by calling client->loadMedia() - // - May check media cache for files - // - May add files to media cache - // - May start remote transfers by calling httpfetch_async - // - May check for completion of current remote transfers - // - May start conventional transfers by calling client->request_media() - // - May inform server that all media has been loaded - // by calling client->received_media() - // After step has been called once, don't call addFile/addRemoteServer. - void step(Client *client); + void step(Client *client) override; - // Must be called for each file received through TOCLIENT_MEDIA - void conventionalTransferDone( + bool conventionalTransferDone( const std::string &name, const std::string &data, - Client *client); + Client *client) override; + +protected: + bool loadMedia(Client *client, const std::string &data, + const std::string &name) override; private: struct FileStatus { @@ -107,13 +153,9 @@ private: void startRemoteMediaTransfers(); void startConventionalTransfers(Client *client); - bool checkAndLoad(const std::string &name, const std::string &sha1, - const std::string &data, bool is_from_cache, - Client *client); - - std::string serializeRequiredHashSet(); static void deSerializeHashSet(const std::string &data, std::set &result); + std::string serializeRequiredHashSet(); // Maps filename to file status std::map m_files; @@ -121,9 +163,6 @@ private: // Array of remote media servers std::vector m_remotes; - // Filesystem-based media cache - FileCache m_media_cache; - // Has an attempt been made to load media files from the file cache? // Have hash sets been requested from remote servers? bool m_initial_step_done = false; @@ -149,3 +188,63 @@ private: std::string m_name_bound = ""; }; + +// A media downloader that only downloads a single file. +// It does/doesn't do several things the normal downloader does: +// - won't fetch hash sets from remote servers +// - will mark loaded media as coming from file push +// - writing to file cache is optional +class SingleMediaDownloader : public IClientMediaDownloader +{ +public: + SingleMediaDownloader(bool write_to_cache); + ~SingleMediaDownloader(); + + bool isStarted() const override { + return m_stage > STAGE_INIT; + } + + bool isDone() const override { + return m_stage >= STAGE_DONE; + } + + void addFile(const std::string &name, const std::string &sha1) override; + + void addRemoteServer(const std::string &baseurl) override; + + void step(Client *client) override; + + bool conventionalTransferDone(const std::string &name, + const std::string &data, Client *client) override; + +protected: + bool loadMedia(Client *client, const std::string &data, + const std::string &name) override; + +private: + void initialStep(Client *client); + void remoteMediaReceived(const HTTPFetchResult &fetch_result, Client *client); + void startRemoteMediaTransfer(); + void startConventionalTransfer(Client *client); + + enum Stage { + STAGE_INIT, + STAGE_CACHE_CHECKED, // we have tried to load the file from cache + STAGE_DONE + }; + + // Information about the one file we want to fetch + std::string m_file_name; + std::string m_file_sha1; + s32 m_current_remote; + + // Array of remote media servers + std::vector m_remotes; + + enum Stage m_stage = STAGE_INIT; + + // Status of remote transfers + unsigned long m_httpfetch_caller; + unsigned long m_httpfetch_next_id = 0; + +}; diff --git a/src/filesys.cpp b/src/filesys.cpp index 99b030624..0941739b8 100644 --- a/src/filesys.cpp +++ b/src/filesys.cpp @@ -21,8 +21,10 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "util/string.h" #include #include +#include #include #include +#include #include #include "log.h" #include "config.h" @@ -811,5 +813,15 @@ bool Rename(const std::string &from, const std::string &to) return rename(from.c_str(), to.c_str()) == 0; } +std::string CreateTempFile() +{ + std::string path = TempPath() + DIR_DELIM "MT_XXXXXX"; + int fd = mkstemp(&path[0]); // modifies path + if (fd == -1) + return ""; + close(fd); + return path; +} + } // namespace fs diff --git a/src/filesys.h b/src/filesys.h index a9584b036..f72cb0ba2 100644 --- a/src/filesys.h +++ b/src/filesys.h @@ -71,6 +71,10 @@ bool DeleteSingleFileOrEmptyDirectory(const std::string &path); // Returns path to temp directory, can return "" on error std::string TempPath(); +// Returns path to securely-created temporary file (will already exist when this function returns) +// can return "" on error +std::string CreateTempFile(); + /* Returns a list of subdirectories, including the path itself, but excluding hidden directories (whose names start with . or _) */ diff --git a/src/network/clientopcodes.cpp b/src/network/clientopcodes.cpp index 55cfdd4dc..a98a5e7d1 100644 --- a/src/network/clientopcodes.cpp +++ b/src/network/clientopcodes.cpp @@ -204,7 +204,7 @@ const ServerCommandFactory serverCommandFactoryTable[TOSERVER_NUM_MSG_TYPES] = null_command_factory, // 0x3e null_command_factory, // 0x3f { "TOSERVER_REQUEST_MEDIA", 1, true }, // 0x40 - null_command_factory, // 0x41 + { "TOSERVER_HAVE_MEDIA", 2, true }, // 0x41 null_command_factory, // 0x42 { "TOSERVER_CLIENT_READY", 1, true }, // 0x43 null_command_factory, // 0x44 diff --git a/src/network/clientpackethandler.cpp b/src/network/clientpackethandler.cpp index a631a3178..9c9c59d13 100644 --- a/src/network/clientpackethandler.cpp +++ b/src/network/clientpackethandler.cpp @@ -670,21 +670,19 @@ void Client::handleCommand_AnnounceMedia(NetworkPacket* pkt) m_media_downloader->addFile(name, sha1_raw); } - try { + { std::string str; - *pkt >> str; Strfnd sf(str); - while(!sf.at_end()) { + while (!sf.at_end()) { std::string baseurl = trim(sf.next(",")); - if (!baseurl.empty()) + if (!baseurl.empty()) { + m_remote_media_servers.emplace_back(baseurl); m_media_downloader->addRemoteServer(baseurl); + } } } - catch(SerializationError& e) { - // not supported by server or turned off - } m_media_downloader->step(this); } @@ -716,31 +714,38 @@ void Client::handleCommand_Media(NetworkPacket* pkt) if (num_files == 0) return; - if (!m_media_downloader || !m_media_downloader->isStarted()) { - const char *problem = m_media_downloader ? - "media has not been requested" : - "all media has been received already"; - errorstream << "Client: Received media but " - << problem << "! " - << " bunch " << bunch_i << "/" << num_bunches - << " files=" << num_files - << " size=" << pkt->getSize() << std::endl; - return; + bool init_phase = m_media_downloader && m_media_downloader->isStarted(); + + if (init_phase) { + // Mesh update thread must be stopped while + // updating content definitions + sanity_check(!m_mesh_update_thread.isRunning()); } - // Mesh update thread must be stopped while - // updating content definitions - sanity_check(!m_mesh_update_thread.isRunning()); - - for (u32 i=0; i < num_files; i++) { - std::string name; + for (u32 i = 0; i < num_files; i++) { + std::string name, data; *pkt >> name; + data = pkt->readLongString(); - std::string data = pkt->readLongString(); - - m_media_downloader->conventionalTransferDone( - name, data, this); + bool ok = false; + if (init_phase) { + ok = m_media_downloader->conventionalTransferDone(name, data, this); + } else { + // Check pending dynamic transfers, one of them must be it + for (const auto &it : m_pending_media_downloads) { + if (it.second->conventionalTransferDone(name, data, this)) { + ok = true; + break; + } + } + } + if (!ok) { + errorstream << "Client: Received media \"" << name + << "\" but no downloads pending. " << num_bunches << " bunches, " + << num_files << " in this one. (init_phase=" << init_phase + << ")" << std::endl; + } } } @@ -1497,46 +1502,72 @@ void Client::handleCommand_PlayerSpeed(NetworkPacket *pkt) void Client::handleCommand_MediaPush(NetworkPacket *pkt) { std::string raw_hash, filename, filedata; + u32 token; bool cached; *pkt >> raw_hash >> filename >> cached; - filedata = pkt->readLongString(); + if (m_proto_ver >= 40) + *pkt >> token; + else + filedata = pkt->readLongString(); - if (raw_hash.size() != 20 || filedata.empty() || filename.empty() || + if (raw_hash.size() != 20 || filename.empty() || + (m_proto_ver < 40 && filedata.empty()) || !string_allowed(filename, TEXTURENAME_ALLOWED_CHARS)) { throw PacketError("Illegal filename, data or hash"); } - verbosestream << "Server pushes media file \"" << filename << "\" with " - << filedata.size() << " bytes of data (cached=" << cached - << ")" << std::endl; + verbosestream << "Server pushes media file \"" << filename << "\" "; + if (filedata.empty()) + verbosestream << "to be fetched "; + else + verbosestream << "with " << filedata.size() << " bytes "; + verbosestream << "(cached=" << cached << ")" << std::endl; if (m_media_pushed_files.count(filename) != 0) { - // Silently ignore for synchronization purposes + // Ignore (but acknowledge). Previously this was for sync purposes, + // but even in new versions media cannot be replaced at runtime. + if (m_proto_ver >= 40) + sendHaveMedia({ token }); return; } - // Compute and check checksum of data - std::string computed_hash; - { - SHA1 ctx; - ctx.addBytes(filedata.c_str(), filedata.size()); - unsigned char *buf = ctx.getDigest(); - computed_hash.assign((char*) buf, 20); - free(buf); - } - if (raw_hash != computed_hash) { - verbosestream << "Hash of file data mismatches, ignoring." << std::endl; + if (!filedata.empty()) { + // LEGACY CODEPATH + // Compute and check checksum of data + std::string computed_hash; + { + SHA1 ctx; + ctx.addBytes(filedata.c_str(), filedata.size()); + unsigned char *buf = ctx.getDigest(); + computed_hash.assign((char*) buf, 20); + free(buf); + } + if (raw_hash != computed_hash) { + verbosestream << "Hash of file data mismatches, ignoring." << std::endl; + return; + } + + // Actually load media + loadMedia(filedata, filename, true); + m_media_pushed_files.insert(filename); + + // Cache file for the next time when this client joins the same server + if (cached) + clientMediaUpdateCache(raw_hash, filedata); return; } - // Actually load media - loadMedia(filedata, filename, true); m_media_pushed_files.insert(filename); - // Cache file for the next time when this client joins the same server - if (cached) - clientMediaUpdateCache(raw_hash, filedata); + // create a downloader for this file + auto downloader = new SingleMediaDownloader(cached); + m_pending_media_downloads.emplace_back(token, downloader); + downloader->addFile(filename, raw_hash); + for (const auto &baseurl : m_remote_media_servers) + downloader->addRemoteServer(baseurl); + + downloader->step(this); } /* diff --git a/src/network/networkprotocol.h b/src/network/networkprotocol.h index b647aab1a..8214cc5b1 100644 --- a/src/network/networkprotocol.h +++ b/src/network/networkprotocol.h @@ -207,6 +207,7 @@ with this program; if not, write to the Free Software Foundation, Inc., Minimap modes PROTOCOL VERSION 40: Added 'basic_debug' privilege + TOCLIENT_MEDIA_PUSH changed, TOSERVER_HAVE_MEDIA added */ #define LATEST_PROTOCOL_VERSION 40 @@ -317,9 +318,8 @@ enum ToClientCommand /* std::string raw_hash std::string filename + u32 callback_token bool should_be_cached - u32 len - char filedata[len] */ // (oops, there is some gap here) @@ -938,7 +938,13 @@ enum ToServerCommand } */ - TOSERVER_RECEIVED_MEDIA = 0x41, // Obsolete + TOSERVER_HAVE_MEDIA = 0x41, + /* + u8 number of callback tokens + for each: + u32 token + */ + TOSERVER_BREATH = 0x42, // Obsolete TOSERVER_CLIENT_READY = 0x43, diff --git a/src/network/serveropcodes.cpp b/src/network/serveropcodes.cpp index aea5d7174..44b65e8da 100644 --- a/src/network/serveropcodes.cpp +++ b/src/network/serveropcodes.cpp @@ -89,7 +89,7 @@ const ToServerCommandHandler toServerCommandTable[TOSERVER_NUM_MSG_TYPES] = null_command_handler, // 0x3e null_command_handler, // 0x3f { "TOSERVER_REQUEST_MEDIA", TOSERVER_STATE_STARTUP, &Server::handleCommand_RequestMedia }, // 0x40 - null_command_handler, // 0x41 + { "TOSERVER_HAVE_MEDIA", TOSERVER_STATE_INGAME, &Server::handleCommand_HaveMedia }, // 0x41 null_command_handler, // 0x42 { "TOSERVER_CLIENT_READY", TOSERVER_STATE_STARTUP, &Server::handleCommand_ClientReady }, // 0x43 null_command_handler, // 0x44 @@ -167,7 +167,7 @@ const ClientCommandFactory clientCommandFactoryTable[TOCLIENT_NUM_MSG_TYPES] = { "TOCLIENT_TIME_OF_DAY", 0, true }, // 0x29 { "TOCLIENT_CSM_RESTRICTION_FLAGS", 0, true }, // 0x2A { "TOCLIENT_PLAYER_SPEED", 0, true }, // 0x2B - { "TOCLIENT_MEDIA_PUSH", 0, true }, // 0x2C (sent over channel 1 too) + { "TOCLIENT_MEDIA_PUSH", 0, true }, // 0x2C (sent over channel 1 too if legacy) null_command_factory, // 0x2D null_command_factory, // 0x2E { "TOCLIENT_CHAT_MESSAGE", 0, true }, // 0x2F diff --git a/src/network/serverpackethandler.cpp b/src/network/serverpackethandler.cpp index 77fde2a66..4c609644f 100644 --- a/src/network/serverpackethandler.cpp +++ b/src/network/serverpackethandler.cpp @@ -362,16 +362,15 @@ void Server::handleCommand_RequestMedia(NetworkPacket* pkt) session_t peer_id = pkt->getPeerId(); infostream << "Sending " << numfiles << " files to " << getPlayerName(peer_id) << std::endl; - verbosestream << "TOSERVER_REQUEST_MEDIA: " << std::endl; + verbosestream << "TOSERVER_REQUEST_MEDIA: requested file(s)" << std::endl; for (u16 i = 0; i < numfiles; i++) { std::string name; *pkt >> name; - tosend.push_back(name); - verbosestream << "TOSERVER_REQUEST_MEDIA: requested file " - << name << std::endl; + tosend.emplace_back(name); + verbosestream << " " << name << std::endl; } sendRequestedMedia(peer_id, tosend); @@ -1801,3 +1800,30 @@ void Server::handleCommand_ModChannelMsg(NetworkPacket *pkt) broadcastModChannelMessage(channel_name, channel_msg, peer_id); } + +void Server::handleCommand_HaveMedia(NetworkPacket *pkt) +{ + std::vector tokens; + u8 numtokens; + + *pkt >> numtokens; + for (u16 i = 0; i < numtokens; i++) { + u32 n; + *pkt >> n; + tokens.emplace_back(n); + } + + const session_t peer_id = pkt->getPeerId(); + auto player = m_env->getPlayer(peer_id); + + for (const u32 token : tokens) { + auto it = m_pending_dyn_media.find(token); + if (it == m_pending_dyn_media.end()) + continue; + if (it->second.waiting_players.count(peer_id)) { + it->second.waiting_players.erase(peer_id); + if (player) + getScriptIface()->on_dynamic_media_added(token, player->getName()); + } + } +} diff --git a/src/script/cpp_api/s_server.cpp b/src/script/cpp_api/s_server.cpp index 96cb28b28..6ddb2630d 100644 --- a/src/script/cpp_api/s_server.cpp +++ b/src/script/cpp_api/s_server.cpp @@ -20,6 +20,7 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "cpp_api/s_server.h" #include "cpp_api/s_internal.h" #include "common/c_converter.h" +#include "util/numeric.h" // myrand bool ScriptApiServer::getAuth(const std::string &playername, std::string *dst_password, @@ -196,3 +197,68 @@ std::string ScriptApiServer::formatChatMessage(const std::string &name, return ret; } + +u32 ScriptApiServer::allocateDynamicMediaCallback(int f_idx) +{ + lua_State *L = getStack(); + + if (f_idx < 0) + f_idx = lua_gettop(L) + f_idx + 1; + + lua_getglobal(L, "core"); + lua_getfield(L, -1, "dynamic_media_callbacks"); + luaL_checktype(L, -1, LUA_TTABLE); + + // Find a randomly generated token that doesn't exist yet + int tries = 100; + u32 token; + while (1) { + token = myrand(); + lua_rawgeti(L, -2, token); + bool is_free = lua_isnil(L, -1); + lua_pop(L, 1); + if (is_free) + break; + if (--tries < 0) + FATAL_ERROR("Ran out of callbacks IDs?!"); + } + + // core.dynamic_media_callbacks[token] = callback_func + lua_pushvalue(L, f_idx); + lua_rawseti(L, -2, token); + + lua_pop(L, 2); + + verbosestream << "allocateDynamicMediaCallback() = " << token << std::endl; + return token; +} + +void ScriptApiServer::freeDynamicMediaCallback(u32 token) +{ + lua_State *L = getStack(); + + verbosestream << "freeDynamicMediaCallback(" << token << ")" << std::endl; + + // core.dynamic_media_callbacks[token] = nil + lua_getglobal(L, "core"); + lua_getfield(L, -1, "dynamic_media_callbacks"); + luaL_checktype(L, -1, LUA_TTABLE); + lua_pushnil(L); + lua_rawseti(L, -2, token); + lua_pop(L, 2); +} + +void ScriptApiServer::on_dynamic_media_added(u32 token, const char *playername) +{ + SCRIPTAPI_PRECHECKHEADER + + int error_handler = PUSH_ERROR_HANDLER(L); + lua_getglobal(L, "core"); + lua_getfield(L, -1, "dynamic_media_callbacks"); + luaL_checktype(L, -1, LUA_TTABLE); + lua_rawgeti(L, -1, token); + luaL_checktype(L, -1, LUA_TFUNCTION); + + lua_pushstring(L, playername); + PCALL_RES(lua_pcall(L, 1, 0, error_handler)); +} diff --git a/src/script/cpp_api/s_server.h b/src/script/cpp_api/s_server.h index d8639cba7..c5c3d5596 100644 --- a/src/script/cpp_api/s_server.h +++ b/src/script/cpp_api/s_server.h @@ -49,6 +49,12 @@ public: const std::string &password); bool setPassword(const std::string &playername, const std::string &password); + + /* dynamic media handling */ + u32 allocateDynamicMediaCallback(int f_idx); + void freeDynamicMediaCallback(u32 token); + void on_dynamic_media_added(u32 token, const char *playername); + private: void getAuthHandler(); void readPrivileges(int index, std::set &result); diff --git a/src/script/lua_api/l_server.cpp b/src/script/lua_api/l_server.cpp index 9866e0bc8..473faaa14 100644 --- a/src/script/lua_api/l_server.cpp +++ b/src/script/lua_api/l_server.cpp @@ -453,29 +453,37 @@ int ModApiServer::l_sound_fade(lua_State *L) } // dynamic_add_media(filepath) -int ModApiServer::l_dynamic_add_media_raw(lua_State *L) +int ModApiServer::l_dynamic_add_media(lua_State *L) { NO_MAP_LOCK_REQUIRED; if (!getEnv(L)) throw LuaError("Dynamic media cannot be added before server has started up"); + Server *server = getServer(L); + + std::string filepath; + std::string to_player; + bool ephemeral = false; + + if (lua_istable(L, 1)) { + getstringfield(L, 1, "filepath", filepath); + getstringfield(L, 1, "to_player", to_player); + getboolfield(L, 1, "ephemeral", ephemeral); + } else { + filepath = readParam(L, 1); + } + if (filepath.empty()) + luaL_typerror(L, 1, "non-empty string"); + luaL_checktype(L, 2, LUA_TFUNCTION); - std::string filepath = readParam(L, 1); CHECK_SECURE_PATH(L, filepath.c_str(), false); - std::vector sent_to; - bool ok = getServer(L)->dynamicAddMedia(filepath, sent_to); - if (ok) { - // (see wrapper code in builtin) - lua_createtable(L, sent_to.size(), 0); - int i = 0; - for (RemotePlayer *player : sent_to) { - lua_pushstring(L, player->getName()); - lua_rawseti(L, -2, ++i); - } - } else { - lua_pushboolean(L, false); - } + u32 token = server->getScriptIface()->allocateDynamicMediaCallback(2); + + bool ok = server->dynamicAddMedia(filepath, token, to_player, ephemeral); + if (!ok) + server->getScriptIface()->freeDynamicMediaCallback(token); + lua_pushboolean(L, ok); return 1; } @@ -519,7 +527,7 @@ void ModApiServer::Initialize(lua_State *L, int top) API_FCT(sound_play); API_FCT(sound_stop); API_FCT(sound_fade); - API_FCT(dynamic_add_media_raw); + API_FCT(dynamic_add_media); API_FCT(get_player_information); API_FCT(get_player_privs); diff --git a/src/script/lua_api/l_server.h b/src/script/lua_api/l_server.h index fb7a851f4..c688e494b 100644 --- a/src/script/lua_api/l_server.h +++ b/src/script/lua_api/l_server.h @@ -71,7 +71,7 @@ private: static int l_sound_fade(lua_State *L); // dynamic_add_media(filepath) - static int l_dynamic_add_media_raw(lua_State *L); + static int l_dynamic_add_media(lua_State *L); // get_player_privs(name, text) static int l_get_player_privs(lua_State *L); diff --git a/src/server.cpp b/src/server.cpp index b96db1209..1b5cbe395 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -665,6 +665,17 @@ void Server::AsyncRunStep(bool initial_step) } else { m_lag_gauge->increment(dtime/100); } + + { + float &counter = m_step_pending_dyn_media_timer; + counter += dtime; + if (counter >= 5.0f) { + stepPendingDynMediaCallbacks(counter); + counter = 0; + } + } + + #if USE_CURL // send masterserver announce { @@ -2527,6 +2538,8 @@ void Server::sendMediaAnnouncement(session_t peer_id, const std::string &lang_co std::string lang_suffix; lang_suffix.append(".").append(lang_code).append(".tr"); for (const auto &i : m_media) { + if (i.second.no_announce) + continue; if (str_ends_with(i.first, ".tr") && !str_ends_with(i.first, lang_suffix)) continue; media_sent++; @@ -2535,6 +2548,8 @@ void Server::sendMediaAnnouncement(session_t peer_id, const std::string &lang_co pkt << media_sent; for (const auto &i : m_media) { + if (i.second.no_announce) + continue; if (str_ends_with(i.first, ".tr") && !str_ends_with(i.first, lang_suffix)) continue; pkt << i.first << i.second.sha1_digest; @@ -2553,11 +2568,9 @@ struct SendableMedia std::string path; std::string data; - SendableMedia(const std::string &name_="", const std::string &path_="", - const std::string &data_=""): - name(name_), - path(path_), - data(data_) + SendableMedia(const std::string &name, const std::string &path, + std::string &&data): + name(name), path(path), data(std::move(data)) {} }; @@ -2584,40 +2597,19 @@ void Server::sendRequestedMedia(session_t peer_id, continue; } - //TODO get path + name - std::string tpath = m_media[name].path; + const auto &m = m_media[name]; // Read data - std::ifstream fis(tpath.c_str(), std::ios_base::binary); - if(!fis.good()){ - errorstream<<"Server::sendRequestedMedia(): Could not open \"" - <= bytes_per_bunch) { @@ -2660,6 +2652,33 @@ void Server::sendRequestedMedia(session_t peer_id, } } +void Server::stepPendingDynMediaCallbacks(float dtime) +{ + MutexAutoLock lock(m_env_mutex); + + for (auto it = m_pending_dyn_media.begin(); it != m_pending_dyn_media.end();) { + it->second.expiry_timer -= dtime; + bool del = it->second.waiting_players.empty() || it->second.expiry_timer < 0; + + if (!del) { + it++; + continue; + } + + const auto &name = it->second.filename; + if (!name.empty()) { + assert(m_media.count(name)); + // if no_announce isn't set we're definitely deleting the wrong file! + sanity_check(m_media[name].no_announce); + + fs::DeleteSingleFileOrEmptyDirectory(m_media[name].path); + m_media.erase(name); + } + getScriptIface()->freeDynamicMediaCallback(it->first); + it = m_pending_dyn_media.erase(it); + } +} + void Server::SendMinimapModes(session_t peer_id, std::vector &modes, size_t wanted_mode) { @@ -3457,14 +3476,18 @@ void Server::deleteParticleSpawner(const std::string &playername, u32 id) SendDeleteParticleSpawner(peer_id, id); } -bool Server::dynamicAddMedia(const std::string &filepath, - std::vector &sent_to) +bool Server::dynamicAddMedia(std::string filepath, + const u32 token, const std::string &to_player, bool ephemeral) { std::string filename = fs::GetFilenameFromPath(filepath.c_str()); - if (m_media.find(filename) != m_media.end()) { - errorstream << "Server::dynamicAddMedia(): file \"" << filename - << "\" already exists in media cache" << std::endl; - return false; + auto it = m_media.find(filename); + if (it != m_media.end()) { + // Allow the same path to be "added" again in certain conditions + if (ephemeral || it->second.path != filepath) { + errorstream << "Server::dynamicAddMedia(): file \"" << filename + << "\" already exists in media cache" << std::endl; + return false; + } } // Load the file and add it to our media cache @@ -3473,35 +3496,91 @@ bool Server::dynamicAddMedia(const std::string &filepath, if (!ok) return false; + if (ephemeral) { + // Create a copy of the file and swap out the path, this removes the + // requirement that mods keep the file accessible at the original path. + filepath = fs::CreateTempFile(); + bool ok = ([&] () -> bool { + if (filepath.empty()) + return false; + std::ofstream os(filepath.c_str(), std::ios::binary); + if (!os.good()) + return false; + os << filedata; + os.close(); + return !os.fail(); + })(); + if (!ok) { + errorstream << "Server: failed to create a copy of media file " + << "\"" << filename << "\"" << std::endl; + m_media.erase(filename); + return false; + } + verbosestream << "Server: \"" << filename << "\" temporarily copied to " + << filepath << std::endl; + + m_media[filename].path = filepath; + m_media[filename].no_announce = true; + // stepPendingDynMediaCallbacks will clean this up later. + } else if (!to_player.empty()) { + m_media[filename].no_announce = true; + } + // Push file to existing clients NetworkPacket pkt(TOCLIENT_MEDIA_PUSH, 0); - pkt << raw_hash << filename << (bool) true; - pkt.putLongString(filedata); + pkt << raw_hash << filename << (bool)ephemeral; + NetworkPacket legacy_pkt = pkt; + + // Newer clients get asked to fetch the file (asynchronous) + pkt << token; + // Older clients have an awful hack that just throws the data at them + legacy_pkt.putLongString(filedata); + + std::unordered_set delivered, waiting; m_clients.lock(); for (auto &pair : m_clients.getClientList()) { if (pair.second->getState() < CS_DefinitionsSent) continue; - if (pair.second->net_proto_version < 39) + const auto proto_ver = pair.second->net_proto_version; + if (proto_ver < 39) continue; - if (auto player = m_env->getPlayer(pair.second->peer_id)) - sent_to.emplace_back(player); - /* - FIXME: this is a very awful hack - The network layer only guarantees ordered delivery inside a channel. - Since the very next packet could be one that uses the media, we have - to push the media over ALL channels to ensure it is processed before - it is used. - In practice this means we have to send it twice: - - channel 1 (HUD) - - channel 0 (everything else: e.g. play_sound, object messages) - */ - m_clients.send(pair.second->peer_id, 1, &pkt, true); - m_clients.send(pair.second->peer_id, 0, &pkt, true); + const session_t peer_id = pair.second->peer_id; + if (!to_player.empty() && getPlayerName(peer_id) != to_player) + continue; + + if (proto_ver < 40) { + delivered.emplace(peer_id); + /* + The network layer only guarantees ordered delivery inside a channel. + Since the very next packet could be one that uses the media, we have + to push the media over ALL channels to ensure it is processed before + it is used. In practice this means channels 1 and 0. + */ + m_clients.send(peer_id, 1, &legacy_pkt, true); + m_clients.send(peer_id, 0, &legacy_pkt, true); + } else { + waiting.emplace(peer_id); + Send(peer_id, &pkt); + } } m_clients.unlock(); + // Run callback for players that already had the file delivered (legacy-only) + for (session_t peer_id : delivered) { + if (auto player = m_env->getPlayer(peer_id)) + getScriptIface()->on_dynamic_media_added(token, player->getName()); + } + + // Save all others in our pending state + auto &state = m_pending_dyn_media[token]; + state.waiting_players = std::move(waiting); + // regardless of success throw away the callback after a while + state.expiry_timer = 60.0f; + if (ephemeral) + state.filename = filename; + return true; } diff --git a/src/server.h b/src/server.h index 9857215d0..7b16845af 100644 --- a/src/server.h +++ b/src/server.h @@ -43,6 +43,7 @@ with this program; if not, write to the Free Software Foundation, Inc., #include #include #include +#include class ChatEvent; struct ChatEventChat; @@ -81,12 +82,14 @@ enum ClientDeletionReason { struct MediaInfo { std::string path; - std::string sha1_digest; + std::string sha1_digest; // base64-encoded + bool no_announce; // true: not announced in TOCLIENT_ANNOUNCE_MEDIA (at player join) MediaInfo(const std::string &path_="", const std::string &sha1_digest_=""): path(path_), - sha1_digest(sha1_digest_) + sha1_digest(sha1_digest_), + no_announce(false) { } }; @@ -197,6 +200,7 @@ public: void handleCommand_FirstSrp(NetworkPacket* pkt); void handleCommand_SrpBytesA(NetworkPacket* pkt); void handleCommand_SrpBytesM(NetworkPacket* pkt); + void handleCommand_HaveMedia(NetworkPacket *pkt); void ProcessData(NetworkPacket *pkt); @@ -257,7 +261,8 @@ public: void deleteParticleSpawner(const std::string &playername, u32 id); - bool dynamicAddMedia(const std::string &filepath, std::vector &sent_to); + bool dynamicAddMedia(std::string filepath, u32 token, + const std::string &to_player, bool ephemeral); ServerInventoryManager *getInventoryMgr() const { return m_inventory_mgr.get(); } void sendDetachedInventory(Inventory *inventory, const std::string &name, session_t peer_id); @@ -395,6 +400,12 @@ private: float m_timer = 0.0f; }; + struct PendingDynamicMediaCallback { + std::string filename; // only set if media entry and file is to be deleted + float expiry_timer; + std::unordered_set waiting_players; + }; + void init(); void SendMovement(session_t peer_id); @@ -466,6 +477,7 @@ private: void sendMediaAnnouncement(session_t peer_id, const std::string &lang_code); void sendRequestedMedia(session_t peer_id, const std::vector &tosend); + void stepPendingDynMediaCallbacks(float dtime); // Adds a ParticleSpawner on peer with peer_id (PEER_ID_INEXISTENT == all) void SendAddParticleSpawner(session_t peer_id, u16 protocol_version, @@ -650,6 +662,10 @@ private: // media files known to server std::unordered_map m_media; + // pending dynamic media callbacks, clients inform the server when they have a file fetched + std::unordered_map m_pending_dyn_media; + float m_step_pending_dyn_media_timer = 0.0f; + /* Sounds */