Server pushing media at runtime (#9961)

This commit is contained in:
sfan5 2020-06-13 19:03:26 +02:00 committed by GitHub
parent 982a030f33
commit 2424dfe007
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 263 additions and 85 deletions

@ -5217,6 +5217,20 @@ Server
* Returns a code (0: successful, 1: no such player, 2: player is connected)
* `minetest.remove_player_auth(name)`: remove player authentication data
* Returns boolean indicating success (false if player nonexistant)
* `minetest.dynamic_add_media(filepath)`
* Adds the file at the given path to the media sent to clients by the server
on startup and also pushes this file to already connected clients.
The file must be a supported image, sound or model format. It must not be
modified, deleted, moved or renamed after calling this function.
The list of dynamically added media is not persisted.
* Returns boolean indicating success (duplicate files count as error)
* The media will be ready to use (in e.g. entity textures, sound_play)
immediately after calling this function.
Old clients that lack support for this feature will not see the media
unless they reconnect to the server.
* Since media transferred this way does not use client caching or HTTP
transfers, dynamic media should not be used with big files or performance
will suffer.
Bans
----

@ -670,11 +670,9 @@ void Client::step(float dtime)
}
}
bool Client::loadMedia(const std::string &data, const std::string &filename)
bool Client::loadMedia(const std::string &data, const std::string &filename,
bool from_media_push)
{
// Silly irrlicht's const-incorrectness
Buffer<char> data_rw(data.c_str(), data.size());
std::string name;
const char *image_ext[] = {
@ -690,6 +688,9 @@ bool Client::loadMedia(const std::string &data, const std::string &filename)
io::IFileSystem *irrfs = RenderingEngine::get_filesystem();
video::IVideoDriver *vdrv = RenderingEngine::get_video_driver();
// Silly irrlicht's const-incorrectness
Buffer<char> data_rw(data.c_str(), data.size());
// Create an irrlicht memory file
io::IReadFile *rfile = irrfs->createMemoryReadFile(
*data_rw, data_rw.getSize(), "_tempreadfile");
@ -727,7 +728,6 @@ bool Client::loadMedia(const std::string &data, const std::string &filename)
".x", ".b3d", ".md2", ".obj",
NULL
};
name = removeStringEnd(filename, model_ext);
if (!name.empty()) {
verbosestream<<"Client: Storing model into memory: "
@ -744,6 +744,8 @@ bool Client::loadMedia(const std::string &data, const std::string &filename)
};
name = removeStringEnd(filename, translate_ext);
if (!name.empty()) {
if (from_media_push)
return false;
TRACESTREAM(<< "Client: Loading translation: "
<< "\"" << filename << "\"" << std::endl);
g_client_translations->loadTranslation(data);

@ -222,6 +222,7 @@ public:
void handleCommand_FormspecPrepend(NetworkPacket *pkt);
void handleCommand_CSMRestrictionFlags(NetworkPacket *pkt);
void handleCommand_PlayerSpeed(NetworkPacket *pkt);
void handleCommand_MediaPush(NetworkPacket *pkt);
void ProcessData(NetworkPacket *pkt);
@ -376,7 +377,8 @@ public:
// The following set of functions is used by ClientMediaDownloader
// Insert a media file appropriately into the appropriate manager
bool loadMedia(const std::string &data, const std::string &filename);
bool loadMedia(const std::string &data, const std::string &filename,
bool from_media_push = false);
// Send a request for conventional media transfer
void request_media(const std::vector<std::string> &file_requests);
@ -488,6 +490,7 @@ private:
Camera *m_camera = nullptr;
Minimap *m_minimap = nullptr;
bool m_minimap_disabled_by_server = false;
// Server serialization version
u8 m_server_ser_ver;
@ -529,7 +532,6 @@ private:
AuthMechanism m_chosen_auth_mech;
void *m_auth_data = nullptr;
bool m_access_denied = false;
bool m_access_denied_reconnect = false;
std::string m_access_denied_reason = "";
@ -538,7 +540,10 @@ private:
bool m_nodedef_received = false;
bool m_activeobjects_received = false;
bool m_mods_loaded = false;
ClientMediaDownloader *m_media_downloader;
// Set of media filenames pushed by server at runtime
std::unordered_set<std::string> m_media_pushed_files;
// time_of_day speed approximation for old protocol
bool m_time_of_day_set = false;

@ -35,6 +35,15 @@ static std::string getMediaCacheDir()
return porting::path_cache + DIR_DELIM + "media";
}
bool clientMediaUpdateCache(const std::string &raw_hash, const std::string &filedata)
{
FileCache media_cache(getMediaCacheDir());
std::string sha1_hex = hex_encode(raw_hash);
if (!media_cache.exists(sha1_hex))
return media_cache.update(sha1_hex, filedata);
return true;
}
/*
ClientMediaDownloader
*/
@ -559,7 +568,6 @@ bool ClientMediaDownloader::checkAndLoad(
return true;
}
/*
Minetest Hashset File Format

@ -33,6 +33,11 @@ struct HTTPFetchResult;
#define MTHASHSET_FILE_SIGNATURE 0x4d544853 // 'MTHS'
#define MTHASHSET_FILE_NAME "index.mth"
// Store file into media cache (unless it exists already)
// Validating the hash is responsibility of the caller
bool clientMediaUpdateCache(const std::string &raw_hash,
const std::string &filedata);
class ClientMediaDownloader
{
public:

@ -82,8 +82,16 @@ bool FileCache::update(const std::string &name, const std::string &data)
std::string path = m_dir + DIR_DELIM + name;
return updateByPath(path, data);
}
bool FileCache::load(const std::string &name, std::ostream &os)
{
std::string path = m_dir + DIR_DELIM + name;
return loadByPath(path, os);
}
bool FileCache::exists(const std::string &name)
{
std::string path = m_dir + DIR_DELIM + name;
std::ifstream fis(path.c_str(), std::ios_base::binary);
return fis.good();
}

@ -33,6 +33,7 @@ public:
bool update(const std::string &name, const std::string &data);
bool load(const std::string &name, std::ostream &os);
bool exists(const std::string &name);
private:
std::string m_dir;

@ -691,6 +691,12 @@ std::string AbsolutePath(const std::string &path)
const char *GetFilenameFromPath(const char *path)
{
const char *filename = strrchr(path, DIR_DELIM_CHAR);
// Consistent with IsDirDelimiter this function handles '/' too
if (DIR_DELIM_CHAR != '/') {
const char *tmp = strrchr(path, '/');
if (tmp && tmp > filename)
filename = tmp;
}
return filename ? filename + 1 : path;
}

@ -68,7 +68,7 @@ const ToClientCommandHandler toClientCommandTable[TOCLIENT_NUM_MSG_TYPES] =
{ "TOCLIENT_TIME_OF_DAY", TOCLIENT_STATE_CONNECTED, &Client::handleCommand_TimeOfDay }, // 0x29
{ "TOCLIENT_CSM_RESTRICTION_FLAGS", TOCLIENT_STATE_CONNECTED, &Client::handleCommand_CSMRestrictionFlags }, // 0x2A
{ "TOCLIENT_PLAYER_SPEED", TOCLIENT_STATE_CONNECTED, &Client::handleCommand_PlayerSpeed }, // 0x2B
null_command_handler,
{ "TOCLIENT_MEDIA_PUSH", TOCLIENT_STATE_CONNECTED, &Client::handleCommand_MediaPush }, // 0x2C
null_command_handler,
null_command_handler,
{ "TOCLIENT_CHAT_MESSAGE", TOCLIENT_STATE_CONNECTED, &Client::handleCommand_ChatMessage }, // 0x2F

@ -39,6 +39,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
#include "script/scripting_client.h"
#include "util/serialize.h"
#include "util/srp.h"
#include "util/sha1.h"
#include "tileanimation.h"
#include "gettext.h"
#include "skyparams.h"
@ -1471,6 +1472,51 @@ void Client::handleCommand_PlayerSpeed(NetworkPacket *pkt)
player->addVelocity(added_vel);
}
void Client::handleCommand_MediaPush(NetworkPacket *pkt)
{
std::string raw_hash, filename, filedata;
bool cached;
*pkt >> raw_hash >> filename >> cached;
filedata = pkt->readLongString();
if (raw_hash.size() != 20 || filedata.empty() || filename.empty() ||
!string_allowed(filename, TEXTURENAME_ALLOWED_CHARS)) {
throw PacketError("Illegal filename, data or hash");
}
verbosestream << "Server pushes media file \"" << filename << "\" with "
<< filedata.size() << " bytes of data (cached=" << cached
<< ")" << std::endl;
if (m_media_pushed_files.count(filename) != 0) {
// Silently ignore for synchronization purposes
return;
}
// Compute and check checksum of data
std::string computed_hash;
{
SHA1 ctx;
ctx.addBytes(filedata.c_str(), filedata.size());
unsigned char *buf = ctx.getDigest();
computed_hash.assign((char*) buf, 20);
free(buf);
}
if (raw_hash != computed_hash) {
verbosestream << "Hash of file data mismatches, ignoring." << std::endl;
return;
}
// Actually load media
loadMedia(filedata, filename, true);
m_media_pushed_files.insert(filename);
// Cache file for the next time when this client joins the same server
if (cached)
clientMediaUpdateCache(raw_hash, filedata);
}
/*
* Mod channels
*/

@ -323,6 +323,15 @@ enum ToClientCommand
v3f added_vel
*/
TOCLIENT_MEDIA_PUSH = 0x2C,
/*
std::string raw_hash
std::string filename
bool should_be_cached
u32 len
char filedata[len]
*/
// (oops, there is some gap here)
TOCLIENT_CHAT_MESSAGE = 0x2F,

@ -167,7 +167,7 @@ const ClientCommandFactory clientCommandFactoryTable[TOCLIENT_NUM_MSG_TYPES] =
{ "TOCLIENT_TIME_OF_DAY", 0, true }, // 0x29
{ "TOCLIENT_CSM_RESTRICTION_FLAGS", 0, true }, // 0x2A
{ "TOCLIENT_PLAYER_SPEED", 0, true }, // 0x2B
null_command_factory, // 0x2C
{ "TOCLIENT_MEDIA_PUSH", 0, true }, // 0x2C (sent over channel 1 too)
null_command_factory, // 0x2D
null_command_factory, // 0x2E
{ "TOCLIENT_CHAT_MESSAGE", 0, true }, // 0x2F

@ -22,6 +22,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
#include "common/c_converter.h"
#include "common/c_content.h"
#include "cpp_api/s_base.h"
#include "cpp_api/s_security.h"
#include "server.h"
#include "environment.h"
#include "remoteplayer.h"
@ -412,9 +413,6 @@ int ModApiServer::l_get_modnames(lua_State *L)
std::vector<std::string> modlist;
getServer(L)->getModNames(modlist);
// Take unsorted items from mods_unsorted and sort them into
// mods_sorted; not great performance but the number of mods on a
// server will likely be small.
std::sort(modlist.begin(), modlist.end());
// Package them up for Lua
@ -474,6 +472,23 @@ int ModApiServer::l_sound_fade(lua_State *L)
return 0;
}
// dynamic_add_media(filepath)
int ModApiServer::l_dynamic_add_media(lua_State *L)
{
NO_MAP_LOCK_REQUIRED;
// Reject adding media before the server has started up
if (!getEnv(L))
throw LuaError("Dynamic media cannot be added before server has started up");
std::string filepath = readParam<std::string>(L, 1);
CHECK_SECURE_PATH(L, filepath.c_str(), false);
bool ok = getServer(L)->dynamicAddMedia(filepath);
lua_pushboolean(L, ok);
return 1;
}
// is_singleplayer()
int ModApiServer::l_is_singleplayer(lua_State *L)
{
@ -538,6 +553,7 @@ void ModApiServer::Initialize(lua_State *L, int top)
API_FCT(sound_play);
API_FCT(sound_stop);
API_FCT(sound_fade);
API_FCT(dynamic_add_media);
API_FCT(get_player_information);
API_FCT(get_player_privs);

@ -70,6 +70,9 @@ private:
// sound_fade(handle, step, gain)
static int l_sound_fade(lua_State *L);
// dynamic_add_media(filepath)
static int l_dynamic_add_media(lua_State *L);
// get_player_privs(name, text)
static int l_get_player_privs(lua_State *L);

@ -2405,9 +2405,87 @@ bool Server::SendBlock(session_t peer_id, const v3s16 &blockpos)
return true;
}
bool Server::addMediaFile(const std::string &filename,
const std::string &filepath, std::string *filedata_to,
std::string *digest_to)
{
// If name contains illegal characters, ignore the file
if (!string_allowed(filename, TEXTURENAME_ALLOWED_CHARS)) {
infostream << "Server: ignoring illegal file name: \""
<< filename << "\"" << std::endl;
return false;
}
// If name is not in a supported format, ignore it
const char *supported_ext[] = {
".png", ".jpg", ".bmp", ".tga",
".pcx", ".ppm", ".psd", ".wal", ".rgb",
".ogg",
".x", ".b3d", ".md2", ".obj",
// Custom translation file format
".tr",
NULL
};
if (removeStringEnd(filename, supported_ext).empty()) {
infostream << "Server: ignoring unsupported file extension: \""
<< filename << "\"" << std::endl;
return false;
}
// Ok, attempt to load the file and add to cache
// Read data
std::ifstream fis(filepath.c_str(), std::ios_base::binary);
if (!fis.good()) {
errorstream << "Server::addMediaFile(): Could not open \""
<< filename << "\" for reading" << std::endl;
return false;
}
std::string filedata;
bool bad = false;
for (;;) {
char buf[1024];
fis.read(buf, sizeof(buf));
std::streamsize len = fis.gcount();
filedata.append(buf, len);
if (fis.eof())
break;
if (!fis.good()) {
bad = true;
break;
}
}
if (bad) {
errorstream << "Server::addMediaFile(): Failed to read \""
<< filename << "\"" << std::endl;
return false;
} else if (filedata.empty()) {
errorstream << "Server::addMediaFile(): Empty file \""
<< filepath << "\"" << std::endl;
return false;
}
SHA1 sha1;
sha1.addBytes(filedata.c_str(), filedata.length());
unsigned char *digest = sha1.getDigest();
std::string sha1_base64 = base64_encode(digest, 20);
std::string sha1_hex = hex_encode((char*) digest, 20);
if (digest_to)
*digest_to = std::string((char*) digest, 20);
free(digest);
// Put in list
m_media[filename] = MediaInfo(filepath, sha1_base64);
verbosestream << "Server: " << sha1_hex << " is " << filename
<< std::endl;
if (filedata_to)
*filedata_to = std::move(filedata);
return true;
}
void Server::fillMediaCache()
{
infostream<<"Server: Calculating media file checksums"<<std::endl;
infostream << "Server: Calculating media file checksums" << std::endl;
// Collect all media file paths
std::vector<std::string> paths;
@ -2419,80 +2497,15 @@ void Server::fillMediaCache()
for (const std::string &mediapath : paths) {
std::vector<fs::DirListNode> dirlist = fs::GetDirListing(mediapath);
for (const fs::DirListNode &dln : dirlist) {
if (dln.dir) // Ignode dirs
if (dln.dir) // Ignore dirs
continue;
std::string filename = dln.name;
// If name contains illegal characters, ignore the file
if (!string_allowed(filename, TEXTURENAME_ALLOWED_CHARS)) {
infostream<<"Server: ignoring illegal file name: \""
<< filename << "\"" << std::endl;
continue;
}
// If name is not in a supported format, ignore it
const char *supported_ext[] = {
".png", ".jpg", ".bmp", ".tga",
".pcx", ".ppm", ".psd", ".wal", ".rgb",
".ogg",
".x", ".b3d", ".md2", ".obj",
// Custom translation file format
".tr",
NULL
};
if (removeStringEnd(filename, supported_ext).empty()){
infostream << "Server: ignoring unsupported file extension: \""
<< filename << "\"" << std::endl;
continue;
}
// Ok, attempt to load the file and add to cache
std::string filepath;
filepath.append(mediapath).append(DIR_DELIM).append(filename);
// Read data
std::ifstream fis(filepath.c_str(), std::ios_base::binary);
if (!fis.good()) {
errorstream << "Server::fillMediaCache(): Could not open \""
<< filename << "\" for reading" << std::endl;
continue;
}
std::ostringstream tmp_os(std::ios_base::binary);
bool bad = false;
for(;;) {
char buf[1024];
fis.read(buf, 1024);
std::streamsize len = fis.gcount();
tmp_os.write(buf, len);
if (fis.eof())
break;
if (!fis.good()) {
bad = true;
break;
}
}
if(bad) {
errorstream<<"Server::fillMediaCache(): Failed to read \""
<< filename << "\"" << std::endl;
continue;
}
if(tmp_os.str().length() == 0) {
errorstream << "Server::fillMediaCache(): Empty file \""
<< filepath << "\"" << std::endl;
continue;
}
SHA1 sha1;
sha1.addBytes(tmp_os.str().c_str(), tmp_os.str().length());
unsigned char *digest = sha1.getDigest();
std::string sha1_base64 = base64_encode(digest, 20);
std::string sha1_hex = hex_encode((char*)digest, 20);
free(digest);
// Put in list
m_media[filename] = MediaInfo(filepath, sha1_base64);
verbosestream << "Server: " << sha1_hex << " is " << filename
<< std::endl;
std::string filepath = mediapath;
filepath.append(DIR_DELIM).append(dln.name);
addMediaFile(dln.name, filepath);
}
}
infostream << "Server: " << m_media.size() << " media files collected" << std::endl;
}
void Server::sendMediaAnnouncement(session_t peer_id, const std::string &lang_code)
@ -3428,6 +3441,44 @@ void Server::deleteParticleSpawner(const std::string &playername, u32 id)
SendDeleteParticleSpawner(peer_id, id);
}
bool Server::dynamicAddMedia(const std::string &filepath)
{
std::string filename = fs::GetFilenameFromPath(filepath.c_str());
if (m_media.find(filename) != m_media.end()) {
errorstream << "Server::dynamicAddMedia(): file \"" << filename
<< "\" already exists in media cache" << std::endl;
return false;
}
// Load the file and add it to our media cache
std::string filedata, raw_hash;
bool ok = addMediaFile(filename, filepath, &filedata, &raw_hash);
if (!ok)
return false;
// Push file to existing clients
NetworkPacket pkt(TOCLIENT_MEDIA_PUSH, 0);
pkt << raw_hash << filename << (bool) true;
pkt.putLongString(filedata);
auto client_ids = m_clients.getClientIDs(CS_DefinitionsSent);
for (session_t client_id : client_ids) {
/*
The network layer only guarantees ordered delivery inside a channel.
Since the very next packet could be one that uses the media, we have
to push the media over ALL channels to ensure it is processed before
it is used.
In practice this means we have to send it twice:
- channel 1 (HUD)
- channel 0 (everything else: e.g. play_sound, object messages)
*/
m_clients.send(client_id, 1, &pkt, true);
m_clients.send(client_id, 0, &pkt, true);
}
return true;
}
// actions: time-reversed list
// Return value: success/failure
bool Server::rollbackRevertActions(const std::list<RollbackAction> &actions,

@ -236,6 +236,8 @@ public:
void deleteParticleSpawner(const std::string &playername, u32 id);
bool dynamicAddMedia(const std::string &filepath);
ServerInventoryManager *getInventoryMgr() const { return m_inventory_mgr.get(); }
void sendDetachedInventory(Inventory *inventory, const std::string &name, session_t peer_id);
@ -435,6 +437,8 @@ private:
// Sends blocks to clients (locks env and con on its own)
void SendBlocks(float dtime);
bool addMediaFile(const std::string &filename, const std::string &filepath,
std::string *filedata = nullptr, std::string *digest = nullptr);
void fillMediaCache();
void sendMediaAnnouncement(session_t peer_id, const std::string &lang_code);
void sendRequestedMedia(session_t peer_id,