Rewrite client media download and support hash-based remote download

Move most of the media-related code in client.cpp into a new class
ClientMediaDownloader (clientmedia.cpp, clientmedia.h). Among other
things, this class does the following things:

- Download [remote_server][sha1] instead of [remote_server][name]. This
is to support servers that provide the same file name with different
contents.
- Initially fetch [remote_server]index.mth. This file should follow the
Minetest Hashset format (currently version 1) and contain a list of SHA1
hashes that exist on the server.
- The list of needed SHA1s is uploaded (via HTTP POST) when index.mth is
requested, so servers can optionally narrow down the list to the needs
of the client.
- If index.mth is missing (HTTP response code 404), we enter compat mode,
fetching [remote_server][name] as before this commit.
- remote_server can now contain multiple servers, separated by commas.
The downloader code attempts to split requests between the different
servers, as permitted by each server's index.mth. If one server claims
to have a file but actually doesn't (or something fails), we ask a
different server that also claims to have it.
- As before, when none of the remote servers provide a particular
file, we download it via the conventional method, i.e. using
the minetest protocol: TOSERVER_REQUEST_MEDIA / TOCLIENT_MEDIA.
- Bugfix: Every downloaded file's SHA1 is now verified against the SHA1
announced by the minetest server (before loading it and inserting it
into the file cache).
- Bugfix: Only send TOSERVER_RECEIVED_MEDIA when we actually have all
media. This should fix #863.
This commit is contained in:
Kahrl 2013-08-29 05:22:18 +02:00
parent 0ea3e6dbe2
commit 0404bbf671
12 changed files with 931 additions and 345 deletions

@ -191,9 +191,16 @@
# right clicks when holding the right mouse button # right clicks when holding the right mouse button
#repeat_rightclick_time = 0.25 #repeat_rightclick_time = 0.25
# will only work for servers which use remote_media setting # Default timeout for cURL, in milliseconds
# and only for clients compiled with cURL # Only has an effect if compiled with cURL
#media_fetch_threads = 8 #curl_timeout = 5000
# Limits number of parallel HTTP requests. Affects:
# - Media fetch if server uses remote_media setting
# - Serverlist download and server announcement
# - Downloads performed by main menu (e.g. mod manager)
# - Downloads performed by mods (minetest.httpfetch)
# Only has an effect if compiled with cURL
#curl_parallel_limit = 8
# Url to the server list displayed in the Multiplayer Tab # Url to the server list displayed in the Multiplayer Tab
#serverlist_url = servers.minetest.net #serverlist_url = servers.minetest.net

@ -360,6 +360,7 @@ set(minetest_SRCS
guiDeathScreen.cpp guiDeathScreen.cpp
guiChatConsole.cpp guiChatConsole.cpp
client.cpp client.cpp
clientmedia.cpp
filecache.cpp filecache.cpp
tile.cpp tile.cpp
shader.cpp shader.cpp

@ -19,6 +19,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
#include "client.h" #include "client.h"
#include <iostream> #include <iostream>
#include <algorithm>
#include "clientserver.h" #include "clientserver.h"
#include "jthread/jmutexautolock.h" #include "jthread/jmutexautolock.h"
#include "main.h" #include "main.h"
@ -37,17 +38,17 @@ with this program; if not, write to the Free Software Foundation, Inc.,
#include "itemdef.h" #include "itemdef.h"
#include "shader.h" #include "shader.h"
#include <IFileSystem.h> #include <IFileSystem.h>
#include "sha1.h"
#include "base64.h" #include "base64.h"
#include "clientmap.h" #include "clientmap.h"
#include "filecache.h" #include "clientmedia.h"
#include "sound.h" #include "sound.h"
#include "util/string.h" #include "util/string.h"
#include "hex.h"
#include "IMeshCache.h" #include "IMeshCache.h"
#include "serialization.h"
#include "util/serialize.h" #include "util/serialize.h"
#include "config.h" #include "config.h"
#include "util/directiontables.h" #include "util/directiontables.h"
#include "util/pointedthing.h"
#include "version.h" #include "version.h"
#if USE_CURL #if USE_CURL
@ -222,46 +223,9 @@ void * MeshUpdateThread::Thread()
return NULL; return NULL;
} }
void * MediaFetchThread::Thread() /*
{ Client
ThreadStarted(); */
log_register_thread("MediaFetchThread");
DSTACK(__FUNCTION_NAME);
BEGIN_DEBUG_EXCEPTION_HANDLER
#if USE_CURL
CURL *curl;
CURLcode res;
for (std::list<MediaRequest>::iterator i = m_file_requests.begin();
i != m_file_requests.end(); ++i) {
curl = curl_easy_init();
assert(curl);
curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1);
curl_easy_setopt(curl, CURLOPT_URL, (m_remote_url + i->name).c_str());
curl_easy_setopt(curl, CURLOPT_FAILONERROR, true);
std::ostringstream stream;
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curl_write_data);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &stream);
curl_easy_setopt(curl, CURLOPT_USERAGENT, (std::string("Minetest ")+minetest_version_hash).c_str());
res = curl_easy_perform(curl);
if (res == CURLE_OK) {
std::string data = stream.str();
m_file_data.push_back(make_pair(i->name, data));
} else {
m_failed.push_back(*i);
infostream << "cURL request failed for " << i->name << " (" << curl_easy_strerror(res) << ")"<< std::endl;
}
curl_easy_cleanup(curl);
}
#endif
END_DEBUG_EXCEPTION_HANDLER(errorstream)
return NULL;
}
Client::Client( Client::Client(
IrrlichtDevice *device, IrrlichtDevice *device,
@ -303,12 +267,9 @@ Client::Client(
m_map_seed(0), m_map_seed(0),
m_password(password), m_password(password),
m_access_denied(false), m_access_denied(false),
m_media_cache(getMediaCacheDir()),
m_media_receive_started(false),
m_media_count(0),
m_media_received_count(0),
m_itemdef_received(false), m_itemdef_received(false),
m_nodedef_received(false), m_nodedef_received(false),
m_media_downloader(new ClientMediaDownloader()),
m_time_of_day_set(false), m_time_of_day_set(false),
m_last_time_of_day_f(-1), m_last_time_of_day_f(-1),
m_time_of_day_update_timer(0), m_time_of_day_update_timer(0),
@ -332,9 +293,6 @@ Client::Client(
m_env.addPlayer(player); m_env.addPlayer(player);
} }
for (size_t i = 0; i < g_settings->getU16("media_fetch_threads"); ++i)
m_media_fetch_threads.push_back(new MediaFetchThread(this));
} }
Client::~Client() Client::~Client()
@ -364,10 +322,6 @@ Client::~Client()
} }
} }
for (std::list<MediaFetchThread*>::iterator i = m_media_fetch_threads.begin();
i != m_media_fetch_threads.end(); ++i)
delete *i;
// cleanup 3d model meshes on client shutdown // cleanup 3d model meshes on client shutdown
while (m_device->getSceneManager()->getMeshCache()->getMeshCount() != 0) { while (m_device->getSceneManager()->getMeshCache()->getMeshCount() != 0) {
scene::IAnimatedMesh * mesh = scene::IAnimatedMesh * mesh =
@ -797,57 +751,11 @@ void Client::step(float dtime)
/* /*
Load fetched media Load fetched media
*/ */
if (m_media_receive_started) { if (m_media_downloader && m_media_downloader->isStarted()) {
bool all_stopped = true; m_media_downloader->step(this);
for (std::list<MediaFetchThread*>::iterator thread = m_media_fetch_threads.begin(); if (m_media_downloader->isDone()) {
thread != m_media_fetch_threads.end(); ++thread) { delete m_media_downloader;
all_stopped &= !(*thread)->IsRunning(); m_media_downloader = NULL;
while (!(*thread)->m_file_data.empty()) {
std::pair <std::string, std::string> out = (*thread)->m_file_data.pop_front();
if(m_media_received_count < m_media_count)
m_media_received_count++;
bool success = loadMedia(out.second, out.first);
if(success){
verbosestream<<"Client: Loaded received media: "
<<"\""<<out.first<<"\". Caching."<<std::endl;
} else{
infostream<<"Client: Failed to load received media: "
<<"\""<<out.first<<"\". Not caching."<<std::endl;
continue;
}
bool did = fs::CreateAllDirs(getMediaCacheDir());
if(!did){
errorstream<<"Could not create media cache directory"
<<std::endl;
}
{
std::map<std::string, std::string>::iterator n;
n = m_media_name_sha1_map.find(out.first);
if(n == m_media_name_sha1_map.end())
errorstream<<"The server sent a file that has not "
<<"been announced."<<std::endl;
else
m_media_cache.update_sha1(out.second);
}
}
}
if (all_stopped) {
std::list<MediaRequest> fetch_failed;
for (std::list<MediaFetchThread*>::iterator thread = m_media_fetch_threads.begin();
thread != m_media_fetch_threads.end(); ++thread) {
for (std::list<MediaRequest>::iterator request = (*thread)->m_failed.begin();
request != (*thread)->m_failed.end(); ++request)
fetch_failed.push_back(*request);
(*thread)->m_failed.clear();
}
if (fetch_failed.size() > 0) {
infostream << "Failed to remote-fetch " << fetch_failed.size() << " files. "
<< "Requesting them the usual way." << std::endl;
request_media(fetch_failed);
}
} }
} }
@ -1048,15 +956,15 @@ void Client::deletingPeer(con::Peer *peer, bool timeout)
string name string name
} }
*/ */
void Client::request_media(const std::list<MediaRequest> &file_requests) void Client::request_media(const std::list<std::string> &file_requests)
{ {
std::ostringstream os(std::ios_base::binary); std::ostringstream os(std::ios_base::binary);
writeU16(os, TOSERVER_REQUEST_MEDIA); writeU16(os, TOSERVER_REQUEST_MEDIA);
writeU16(os, file_requests.size()); writeU16(os, file_requests.size());
for(std::list<MediaRequest>::const_iterator i = file_requests.begin(); for(std::list<std::string>::const_iterator i = file_requests.begin();
i != file_requests.end(); ++i) { i != file_requests.end(); ++i) {
os<<serializeString(i->name); os<<serializeString(*i);
} }
// Make data buffer // Make data buffer
@ -1068,6 +976,19 @@ void Client::request_media(const std::list<MediaRequest> &file_requests)
<<file_requests.size()<<" files)"<<std::endl; <<file_requests.size()<<" files)"<<std::endl;
} }
void Client::received_media()
{
// notify server we received everything
std::ostringstream os(std::ios_base::binary);
writeU16(os, TOSERVER_RECEIVED_MEDIA);
std::string s = os.str();
SharedBuffer<u8> data((u8*)s.c_str(), s.size());
// Send as reliable
Send(0, data, true);
infostream<<"Client: Notifying server that we received all media"
<<std::endl;
}
void Client::ReceiveAll() void Client::ReceiveAll()
{ {
DSTACK(__FUNCTION_NAME); DSTACK(__FUNCTION_NAME);
@ -1660,96 +1581,54 @@ void Client::ProcessData(u8 *data, u32 datasize, u16 sender_peer_id)
std::string datastring((char*)&data[2], datasize-2); std::string datastring((char*)&data[2], datasize-2);
std::istringstream is(datastring, std::ios_base::binary); std::istringstream is(datastring, std::ios_base::binary);
// Mesh update thread must be stopped while
// updating content definitions
assert(!m_mesh_update_thread.IsRunning());
int num_files = readU16(is); int num_files = readU16(is);
infostream<<"Client: Received media announcement: packet size: " infostream<<"Client: Received media announcement: packet size: "
<<datasize<<std::endl; <<datasize<<std::endl;
std::list<MediaRequest> file_requests; if (m_media_downloader == NULL ||
m_media_downloader->isStarted()) {
const char *problem = m_media_downloader ?
"we already saw another announcement" :
"all media has been received already";
errorstream<<"Client: Received media announcement but "
<<problem<<"! "
<<" files="<<num_files
<<" size="<<datasize<<std::endl;
return;
}
// Mesh update thread must be stopped while
// updating content definitions
assert(!m_mesh_update_thread.IsRunning());
for(int i=0; i<num_files; i++) for(int i=0; i<num_files; i++)
{ {
//read file from cache
std::string name = deSerializeString(is); std::string name = deSerializeString(is);
std::string sha1_base64 = deSerializeString(is); std::string sha1_base64 = deSerializeString(is);
// if name contains illegal characters, ignore the file
if(!string_allowed(name, TEXTURENAME_ALLOWED_CHARS)){
errorstream<<"Client: ignoring illegal file name "
<<"sent by server: \""<<name<<"\""<<std::endl;
continue;
}
std::string sha1_raw = base64_decode(sha1_base64); std::string sha1_raw = base64_decode(sha1_base64);
std::string sha1_hex = hex_encode(sha1_raw); m_media_downloader->addFile(name, sha1_raw);
std::ostringstream tmp_os(std::ios_base::binary);
bool found_in_cache = m_media_cache.load_sha1(sha1_raw, tmp_os);
m_media_name_sha1_map[name] = sha1_raw;
// If found in cache, try to load it from there
if(found_in_cache)
{
bool success = loadMedia(tmp_os.str(), name);
if(success){
verbosestream<<"Client: Loaded cached media: "
<<sha1_hex<<" \""<<name<<"\""<<std::endl;
continue;
} else{
infostream<<"Client: Failed to load cached media: "
<<sha1_hex<<" \""<<name<<"\""<<std::endl;
}
}
// Didn't load from cache; queue it to be requested
verbosestream<<"Client: Adding file to request list: \""
<<sha1_hex<<" \""<<name<<"\""<<std::endl;
file_requests.push_back(MediaRequest(name));
} }
std::string remote_media = ""; std::vector<std::string> remote_media;
try { try {
remote_media = deSerializeString(is); Strfnd sf(deSerializeString(is));
while(!sf.atend()) {
std::string baseurl = trim(sf.next(","));
if(baseurl != "")
m_media_downloader->addRemoteServer(baseurl);
}
} }
catch(SerializationError) { catch(SerializationError) {
// not supported by server or turned off // not supported by server or turned off
} }
m_media_count = file_requests.size(); m_media_downloader->step(this);
m_media_receive_started = true; if (m_media_downloader->isDone()) {
// might be done already if all media is in the cache
if (remote_media == "" || !USE_CURL) { delete m_media_downloader;
request_media(file_requests); m_media_downloader = NULL;
} else {
#if USE_CURL
std::list<MediaFetchThread*>::iterator cur = m_media_fetch_threads.begin();
for(std::list<MediaRequest>::iterator i = file_requests.begin();
i != file_requests.end(); ++i) {
(*cur)->m_file_requests.push_back(*i);
cur++;
if (cur == m_media_fetch_threads.end())
cur = m_media_fetch_threads.begin();
} }
for (std::list<MediaFetchThread*>::iterator i = m_media_fetch_threads.begin();
i != m_media_fetch_threads.end(); ++i) {
(*i)->m_remote_url = remote_media;
(*i)->Start();
}
#endif
// notify server we received everything
std::ostringstream os(std::ios_base::binary);
writeU16(os, TOSERVER_RECEIVED_MEDIA);
std::string s = os.str();
SharedBuffer<u8> data((u8*)s.c_str(), s.size());
// Send as reliable
Send(0, data, true);
}
ClientEvent event;
event.type = CE_TEXTURES_UPDATED;
m_client_event_queue.push_back(event);
} }
else if(command == TOCLIENT_MEDIA) else if(command == TOCLIENT_MEDIA)
{ {
@ -1775,67 +1654,37 @@ void Client::ProcessData(u8 *data, u32 datasize, u16 sender_peer_id)
<<num_bunches<<" files="<<num_files <<num_bunches<<" files="<<num_files
<<" size="<<datasize<<std::endl; <<" size="<<datasize<<std::endl;
// Check total and received media count if (num_files == 0)
assert(m_media_received_count <= m_media_count); return;
if (num_files > m_media_count - m_media_received_count) {
errorstream<<"Client: Received more files than requested:" if (m_media_downloader == NULL ||
<<" total count="<<m_media_count !m_media_downloader->isStarted()) {
<<" total received="<<m_media_received_count const char *problem = m_media_downloader ?
"media has not been requested" :
"all media has been received already";
errorstream<<"Client: Received media but "
<<problem<<"! "
<<" bunch "<<bunch_i<<"/"<<num_bunches <<" bunch "<<bunch_i<<"/"<<num_bunches
<<" files="<<num_files <<" files="<<num_files
<<" size="<<datasize<<std::endl; <<" size="<<datasize<<std::endl;
num_files = m_media_count - m_media_received_count;
}
if (num_files == 0)
return; return;
}
// Mesh update thread must be stopped while // Mesh update thread must be stopped while
// updating content definitions // updating content definitions
assert(!m_mesh_update_thread.IsRunning()); assert(!m_mesh_update_thread.IsRunning());
for(u32 i=0; i<num_files; i++){ for(u32 i=0; i<num_files; i++){
assert(m_media_received_count < m_media_count);
m_media_received_count++;
std::string name = deSerializeString(is); std::string name = deSerializeString(is);
std::string data = deSerializeLongString(is); std::string data = deSerializeLongString(is);
m_media_downloader->conventionalTransferDone(
// if name contains illegal characters, ignore the file name, data, this);
if(!string_allowed(name, TEXTURENAME_ALLOWED_CHARS)){
errorstream<<"Client: ignoring illegal file name "
<<"sent by server: \""<<name<<"\""<<std::endl;
continue;
} }
bool success = loadMedia(data, name); if (m_media_downloader->isDone()) {
if(success){ delete m_media_downloader;
verbosestream<<"Client: Loaded received media: " m_media_downloader = NULL;
<<"\""<<name<<"\". Caching."<<std::endl;
} else{
infostream<<"Client: Failed to load received media: "
<<"\""<<name<<"\". Not caching."<<std::endl;
continue;
} }
bool did = fs::CreateAllDirs(getMediaCacheDir());
if(!did){
errorstream<<"Could not create media cache directory"
<<std::endl;
}
{
std::map<std::string, std::string>::iterator n;
n = m_media_name_sha1_map.find(name);
if(n == m_media_name_sha1_map.end())
errorstream<<"The server sent a file that has not "
<<"been announced."<<std::endl;
else
m_media_cache.update_sha1(data);
}
}
ClientEvent event;
event.type = CE_TEXTURES_UPDATED;
m_client_event_queue.push_back(event);
} }
else if(command == TOCLIENT_TOOLDEF) else if(command == TOCLIENT_TOOLDEF)
{ {
@ -2885,6 +2734,14 @@ ClientEvent Client::getClientEvent()
return m_client_event_queue.pop_front(); return m_client_event_queue.pop_front();
} }
float Client::mediaReceiveProgress()
{
if (m_media_downloader)
return m_media_downloader->getProgress();
else
return 1.0; // downloader only exists when not yet done
}
void draw_load_screen(const std::wstring &text, void draw_load_screen(const std::wstring &text,
IrrlichtDevice* device, gui::IGUIFont* font, IrrlichtDevice* device, gui::IGUIFont* font,
float dtime=0 ,int percent=0, bool clouds=true); float dtime=0 ,int percent=0, bool clouds=true);
@ -2893,11 +2750,7 @@ void Client::afterContentReceived(IrrlichtDevice *device, gui::IGUIFont* font)
infostream<<"Client::afterContentReceived() started"<<std::endl; infostream<<"Client::afterContentReceived() started"<<std::endl;
assert(m_itemdef_received); assert(m_itemdef_received);
assert(m_nodedef_received); assert(m_nodedef_received);
assert(texturesReceived()); assert(mediaReceived());
// remove the information about which checksum each texture
// ought to have
m_media_name_sha1_map.clear();
// Rebuild inherited images and recreate textures // Rebuild inherited images and recreate textures
infostream<<"- Rebuilding images and textures"<<std::endl; infostream<<"- Rebuilding images and textures"<<std::endl;

@ -31,32 +31,21 @@ with this program; if not, write to the Free Software Foundation, Inc.,
#include "clientobject.h" #include "clientobject.h"
#include "gamedef.h" #include "gamedef.h"
#include "inventorymanager.h" #include "inventorymanager.h"
#include "filecache.h"
#include "localplayer.h" #include "localplayer.h"
#include "server.h" #include "hud.h"
#include "particles.h" #include "particles.h"
#include "util/pointedthing.h"
#include <algorithm>
struct MeshMakeData; struct MeshMakeData;
class MapBlockMesh; class MapBlockMesh;
class IGameDef;
class IWritableTextureSource; class IWritableTextureSource;
class IWritableShaderSource; class IWritableShaderSource;
class IWritableItemDefManager; class IWritableItemDefManager;
class IWritableNodeDefManager; class IWritableNodeDefManager;
//class IWritableCraftDefManager; //class IWritableCraftDefManager;
class ClientEnvironment; class ClientMediaDownloader;
struct MapDrawControl; struct MapDrawControl;
class MtEventManager; class MtEventManager;
struct PointedThing;
class ClientNotReadyException : public BaseException
{
public:
ClientNotReadyException(const char *s):
BaseException(s)
{}
};
struct QueuedMeshUpdate struct QueuedMeshUpdate
{ {
@ -132,31 +121,12 @@ public:
IGameDef *m_gamedef; IGameDef *m_gamedef;
}; };
class MediaFetchThread : public SimpleThread
{
public:
MediaFetchThread(IGameDef *gamedef):
m_gamedef(gamedef)
{
}
void * Thread();
std::list<MediaRequest> m_file_requests;
MutexedQueue<std::pair<std::string, std::string> > m_file_data;
std::list<MediaRequest> m_failed;
std::string m_remote_url;
IGameDef *m_gamedef;
};
enum ClientEventType enum ClientEventType
{ {
CE_NONE, CE_NONE,
CE_PLAYER_DAMAGE, CE_PLAYER_DAMAGE,
CE_PLAYER_FORCE_MOVE, CE_PLAYER_FORCE_MOVE,
CE_DEATHSCREEN, CE_DEATHSCREEN,
CE_TEXTURES_UPDATED,
CE_SHOW_FORMSPEC, CE_SHOW_FORMSPEC,
CE_SPAWN_PARTICLE, CE_SPAWN_PARTICLE,
CE_ADD_PARTICLESPAWNER, CE_ADD_PARTICLESPAWNER,
@ -426,18 +396,14 @@ public:
std::wstring accessDeniedReason() std::wstring accessDeniedReason()
{ return m_access_denied_reason; } { return m_access_denied_reason; }
float mediaReceiveProgress()
{
if (!m_media_receive_started) return 0;
return 1.0 * m_media_received_count / m_media_count;
}
bool texturesReceived()
{ return m_media_receive_started && m_media_received_count == m_media_count; }
bool itemdefReceived() bool itemdefReceived()
{ return m_itemdef_received; } { return m_itemdef_received; }
bool nodedefReceived() bool nodedefReceived()
{ return m_nodedef_received; } { return m_nodedef_received; }
bool mediaReceived()
{ return m_media_downloader == NULL; }
float mediaReceiveProgress();
void afterContentReceived(IrrlichtDevice *device, gui::IGUIFont* font); void afterContentReceived(IrrlichtDevice *device, gui::IGUIFont* font);
@ -455,12 +421,15 @@ public:
virtual bool checkLocalPrivilege(const std::string &priv) virtual bool checkLocalPrivilege(const std::string &priv)
{ return checkPrivilege(priv); } { return checkPrivilege(priv); }
private: // The following set of functions is used by ClientMediaDownloader
// Insert a media file appropriately into the appropriate manager // Insert a media file appropriately into the appropriate manager
bool loadMedia(const std::string &data, const std::string &filename); bool loadMedia(const std::string &data, const std::string &filename);
// Send a request for conventional media transfer
void request_media(const std::list<std::string> &file_requests);
// Send a notification that no conventional media transfer is needed
void received_media();
void request_media(const std::list<MediaRequest> &file_requests); private:
// Virtual methods from con::PeerHandler // Virtual methods from con::PeerHandler
void peerAdded(con::Peer *peer); void peerAdded(con::Peer *peer);
@ -488,7 +457,6 @@ private:
MtEventManager *m_event; MtEventManager *m_event;
MeshUpdateThread m_mesh_update_thread; MeshUpdateThread m_mesh_update_thread;
std::list<MediaFetchThread*> m_media_fetch_threads;
ClientEnvironment m_env; ClientEnvironment m_env;
con::Connection m_con; con::Connection m_con;
IrrlichtDevice *m_device; IrrlichtDevice *m_device;
@ -514,14 +482,9 @@ private:
bool m_access_denied; bool m_access_denied;
std::wstring m_access_denied_reason; std::wstring m_access_denied_reason;
Queue<ClientEvent> m_client_event_queue; Queue<ClientEvent> m_client_event_queue;
FileCache m_media_cache;
// Mapping from media file name to SHA1 checksum
std::map<std::string, std::string> m_media_name_sha1_map;
bool m_media_receive_started;
u32 m_media_count;
u32 m_media_received_count;
bool m_itemdef_received; bool m_itemdef_received;
bool m_nodedef_received; bool m_nodedef_received;
ClientMediaDownloader *m_media_downloader;
// time_of_day speed approximation for old protocol // time_of_day speed approximation for old protocol
bool m_time_of_day_set; bool m_time_of_day_set;

656
src/clientmedia.cpp Normal file

@ -0,0 +1,656 @@
/*
Minetest
Copyright (C) 2013 celeron55, Perttu Ahola <celeron55@gmail.com>
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation; either version 2.1 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License along
with this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
#include "clientmedia.h"
#include "httpfetch.h"
#include "client.h"
#include "clientserver.h"
#include "filecache.h"
#include "hex.h"
#include "sha1.h"
#include "debug.h"
#include "log.h"
#include "porting.h"
#include "settings.h"
#include "main.h"
#include "util/serialize.h"
#include "util/string.h"
static std::string getMediaCacheDir()
{
return porting::path_user + DIR_DELIM + "cache" + DIR_DELIM + "media";
}
/*
ClientMediaDownloader
*/
ClientMediaDownloader::ClientMediaDownloader():
m_media_cache(getMediaCacheDir())
{
m_initial_step_done = false;
m_name_bound = ""; // works because "" is an invalid file name
m_uncached_count = 0;
m_uncached_received_count = 0;
m_httpfetch_caller = HTTPFETCH_DISCARD;
m_httpfetch_active = 0;
m_httpfetch_active_limit = 0;
m_httpfetch_next_id = 0;
m_httpfetch_timeout = 0;
m_outstanding_hash_sets = 0;
}
ClientMediaDownloader::~ClientMediaDownloader()
{
if (m_httpfetch_caller != HTTPFETCH_DISCARD)
httpfetch_caller_free(m_httpfetch_caller);
for (std::map<std::string, FileStatus*>::iterator it = m_files.begin();
it != m_files.end(); ++it)
delete it->second;
for (u32 i = 0; i < m_remotes.size(); ++i)
delete m_remotes[i];
}
void ClientMediaDownloader::addFile(std::string name, std::string sha1)
{
assert(!m_initial_step_done);
// if name was already announced, ignore the new announcement
if (m_files.count(name) != 0) {
errorstream << "Client: ignoring duplicate media announcement "
<< "sent by server: \"" << name << "\""
<< std::endl;
return;
}
// if name is empty or contains illegal characters, ignore the file
if (name.empty() || !string_allowed(name, TEXTURENAME_ALLOWED_CHARS)) {
errorstream << "Client: ignoring illegal file name "
<< "sent by server: \"" << name << "\""
<< std::endl;
return;
}
// length of sha1 must be exactly 20 (160 bits), else ignore the file
if (sha1.size() != 20) {
errorstream << "Client: ignoring illegal SHA1 sent by server: "
<< hex_encode(sha1) << " \"" << name << "\""
<< std::endl;
return;
}
FileStatus *filestatus = new FileStatus;
filestatus->received = false;
filestatus->sha1 = sha1;
filestatus->current_remote = -1;
m_files.insert(std::make_pair(name, filestatus));
}
void ClientMediaDownloader::addRemoteServer(std::string baseurl)
{
assert(!m_initial_step_done);
#ifdef USE_CURL
infostream << "Client: Adding remote server \""
<< baseurl << "\" for media download" << std::endl;
RemoteServerStatus *remote = new RemoteServerStatus;
remote->baseurl = baseurl;
remote->active_count = 0;
remote->request_by_filename = false;
m_remotes.push_back(remote);
#else
infostream << "Client: Ignoring remote server \""
<< baseurl << "\" because cURL support is not compiled in"
<< std::endl;
#endif
}
void ClientMediaDownloader::step(Client *client)
{
if (!m_initial_step_done) {
initialStep(client);
m_initial_step_done = true;
}
// Remote media: check for completion of fetches
if (m_httpfetch_active) {
bool fetched_something = false;
HTTPFetchResult fetchresult;
while (httpfetch_async_get(m_httpfetch_caller, fetchresult)) {
m_httpfetch_active--;
fetched_something = true;
// Is this a hashset (index.mth) or a media file?
if (fetchresult.request_id < m_remotes.size())
remoteHashSetReceived(fetchresult);
else
remoteMediaReceived(fetchresult, client);
}
if (fetched_something)
startRemoteMediaTransfers();
// Did all remote transfers end and no new ones can be started?
// If so, request still missing files from the minetest server
// (Or report that we have all files.)
if (m_httpfetch_active == 0) {
if (m_uncached_received_count < m_uncached_count) {
infostream << "Client: Failed to remote-fetch "
<< (m_uncached_count-m_uncached_received_count)
<< " files. Requesting them"
<< " the usual way." << std::endl;
}
startConventionalTransfers(client);
}
}
}
void ClientMediaDownloader::initialStep(Client *client)
{
// Check media cache
m_uncached_count = m_files.size();
for (std::map<std::string, FileStatus*>::iterator
it = m_files.begin();
it != m_files.end(); ++it) {
std::string name = it->first;
FileStatus *filestatus = it->second;
const std::string &sha1 = filestatus->sha1;
std::ostringstream tmp_os(std::ios_base::binary);
bool found_in_cache = m_media_cache.load(hex_encode(sha1), tmp_os);
// If found in cache, try to load it from there
if (found_in_cache) {
bool success = checkAndLoad(name, sha1,
tmp_os.str(), true, client);
if (success) {
filestatus->received = true;
m_uncached_count--;
}
}
}
assert(m_uncached_received_count == 0);
// Create the media cache dir if we are likely to write to it
if (m_uncached_count != 0) {
bool did = fs::CreateAllDirs(getMediaCacheDir());
if (!did) {
errorstream << "Client: "
<< "Could not create media cache directory: "
<< getMediaCacheDir()
<< std::endl;
}
}
// If we found all files in the cache, report this fact to the server.
// If the server reported no remote servers, immediately start
// conventional transfers. Note: if cURL support is not compiled in,
// m_remotes is always empty, so "!USE_CURL" is redundant but may
// reduce the size of the compiled code
if (!USE_CURL || m_uncached_count == 0 || m_remotes.empty()) {
startConventionalTransfers(client);
}
else {
// Otherwise start off by requesting each server's sha1 set
// This is the first time we use httpfetch, so alloc a caller ID
m_httpfetch_caller = httpfetch_caller_alloc();
m_httpfetch_timeout = g_settings->getS32("curl_timeout");
// Set the active fetch limit to curl_parallel_limit or 84,
// whichever is greater. This gives us some leeway so that
// inefficiencies in communicating with the httpfetch thread
// don't slow down fetches too much. (We still want some limit
// so that when the first remote server returns its hash set,
// not all files are requested from that server immediately.)
// One such inefficiency is that ClientMediaDownloader::step()
// is only called a couple times per second, while httpfetch
// might return responses much faster than that.
// Note that httpfetch strictly enforces curl_parallel_limit
// but at no inter-thread communication cost. This however
// doesn't help with the aforementioned inefficiencies.
// The signifance of 84 is that it is 2*6*9 in base 13.
m_httpfetch_active_limit = g_settings->getS32("curl_parallel_limit");
m_httpfetch_active_limit = MYMAX(m_httpfetch_active_limit, 84);
// Write a list of hashes that we need. This will be POSTed
// to the server using Content-Type: application/octet-stream
std::string required_hash_set = serializeRequiredHashSet();
// minor fixme: this loop ignores m_httpfetch_active_limit
// another minor fixme, unlikely to matter in normal usage:
// these index.mth fetches do (however) count against
// m_httpfetch_active_limit when starting actual media file
// requests, so if there are lots of remote servers that are
// not responding, those will stall new media file transfers.
for (u32 i = 0; i < m_remotes.size(); ++i) {
assert(m_httpfetch_next_id == i);
RemoteServerStatus *remote = m_remotes[i];
actionstream << "Client: Contacting remote server \""
<< remote->baseurl << "\"" << std::endl;
HTTPFetchRequest fetchrequest;
fetchrequest.url =
remote->baseurl + MTHASHSET_FILE_NAME;
fetchrequest.caller = m_httpfetch_caller;
fetchrequest.request_id = m_httpfetch_next_id; // == i
fetchrequest.timeout = m_httpfetch_timeout;
fetchrequest.connect_timeout = m_httpfetch_timeout;
fetchrequest.post_fields = required_hash_set;
fetchrequest.extra_headers.push_back(
"Content-Type: application/octet-stream");
httpfetch_async(fetchrequest);
m_httpfetch_active++;
m_httpfetch_next_id++;
m_outstanding_hash_sets++;
}
}
}
void ClientMediaDownloader::remoteHashSetReceived(
const HTTPFetchResult &fetchresult)
{
u32 remote_id = fetchresult.request_id;
assert(remote_id < m_remotes.size());
RemoteServerStatus *remote = m_remotes[remote_id];
m_outstanding_hash_sets--;
if (fetchresult.succeeded) {
try {
// Server sent a list of file hashes that are
// available on it, try to parse the list
std::set<std::string> sha1_set;
deSerializeHashSet(fetchresult.data, sha1_set);
// Parsing succeeded: For every file that is
// available on this server, add this server
// to the available_remotes array
for(std::map<std::string, FileStatus*>::iterator
it = m_files.upper_bound(m_name_bound);
it != m_files.end(); ++it) {
FileStatus *f = it->second;
if (!f->received && sha1_set.count(f->sha1))
f->available_remotes.push_back(remote_id);
}
}
catch (SerializationError &e) {
infostream << "Client: Remote server \""
<< remote->baseurl << "\" sent invalid hash set: "
<< e.what() << std::endl;
}
}
// For compatibility: If index.mth is not found, assume that the
// server contains files named like the original files (not their sha1)
if (!fetchresult.succeeded && !fetchresult.timeout &&
fetchresult.response_code == 404) {
infostream << "Client: Enabling compatibility mode for remote "
<< "server \"" << remote->baseurl << "\"" << std::endl;
remote->request_by_filename = true;
// Assume every file is available on this server
for(std::map<std::string, FileStatus*>::iterator
it = m_files.upper_bound(m_name_bound);
it != m_files.end(); ++it) {
FileStatus *f = it->second;
if (!f->received)
f->available_remotes.push_back(remote_id);
}
}
}
void ClientMediaDownloader::remoteMediaReceived(
const HTTPFetchResult &fetchresult,
Client *client)
{
// Some remote server sent us a file.
// -> decrement number of active fetches
// -> mark file as received if fetch succeeded
// -> try to load media
std::string name;
{
std::map<unsigned long, std::string>::iterator it =
m_remote_file_transfers.find(fetchresult.request_id);
assert(it != m_remote_file_transfers.end());
name = it->second;
m_remote_file_transfers.erase(it);
}
assert(m_files.count(name) != 0);
FileStatus *filestatus = m_files[name];
assert(!filestatus->received);
assert(filestatus->current_remote >= 0);
RemoteServerStatus *remote = m_remotes[filestatus->current_remote];
filestatus->current_remote = -1;
remote->active_count--;
// If fetch succeeded, try to load media file
if (fetchresult.succeeded) {
bool success = checkAndLoad(name, filestatus->sha1,
fetchresult.data, false, client);
if (success) {
filestatus->received = true;
assert(m_uncached_received_count < m_uncached_count);
m_uncached_received_count++;
}
}
}
s32 ClientMediaDownloader::selectRemoteServer(FileStatus *filestatus)
{
assert(filestatus != NULL);
assert(!filestatus->received);
assert(filestatus->current_remote < 0);
if (filestatus->available_remotes.empty())
return -1;
else {
// Of all servers that claim to provide the file (and haven't
// been unsuccessfully tried before), find the one with the
// smallest number of currently active transfers
s32 best = 0;
s32 best_remote_id = filestatus->available_remotes[best];
s32 best_active_count = m_remotes[best_remote_id]->active_count;
for (u32 i = 1; i < filestatus->available_remotes.size(); ++i) {
s32 remote_id = filestatus->available_remotes[i];
s32 active_count = m_remotes[remote_id]->active_count;
if (active_count < best_active_count) {
best = i;
best_remote_id = remote_id;
best_active_count = active_count;
}
}
filestatus->available_remotes.erase(
filestatus->available_remotes.begin() + best);
return best_remote_id;
}
}
void ClientMediaDownloader::startRemoteMediaTransfers()
{
bool changing_name_bound = true;
for (std::map<std::string, FileStatus*>::iterator
files_iter = m_files.upper_bound(m_name_bound);
files_iter != m_files.end(); ++files_iter) {
// Abort if active fetch limit is exceeded
if (m_httpfetch_active >= m_httpfetch_active_limit)
break;
const std::string &name = files_iter->first;
FileStatus *filestatus = files_iter->second;
if (!filestatus->received && filestatus->current_remote < 0) {
// File has not been received yet and is not currently
// being transferred. Choose a server for it.
s32 remote_id = selectRemoteServer(filestatus);
if (remote_id >= 0) {
// Found a server, so start fetching
RemoteServerStatus *remote =
m_remotes[remote_id];
std::string url = remote->baseurl +
(remote->request_by_filename ? name :
hex_encode(filestatus->sha1));
verbosestream << "Client: "
<< "Requesting remote media file "
<< "\"" << name << "\" "
<< "\"" << url << "\"" << std::endl;
HTTPFetchRequest fetchrequest;
fetchrequest.url = url;
fetchrequest.caller = m_httpfetch_caller;
fetchrequest.request_id = m_httpfetch_next_id;
fetchrequest.timeout = 0; // no data timeout!
fetchrequest.connect_timeout =
m_httpfetch_timeout;
httpfetch_async(fetchrequest);
m_remote_file_transfers.insert(std::make_pair(
m_httpfetch_next_id,
name));
filestatus->current_remote = remote_id;
remote->active_count++;
m_httpfetch_active++;
m_httpfetch_next_id++;
}
}
if (filestatus->received ||
(filestatus->current_remote < 0 &&
!m_outstanding_hash_sets)) {
// If we arrive here, we conclusively know that we
// won't fetch this file from a remote server in the
// future. So update the name bound if possible.
if (changing_name_bound)
m_name_bound = name;
}
else
changing_name_bound = false;
}
}
void ClientMediaDownloader::startConventionalTransfers(Client *client)
{
assert(m_httpfetch_active == 0);
if (m_uncached_received_count == m_uncached_count) {
// In this case all media was found in the cache or
// has been downloaded from some remote server;
// report this fact to the server
client->received_media();
}
else {
// Some media files have not been received yet, use the
// conventional slow method (minetest protocol) to get them
std::list<std::string> file_requests;
for (std::map<std::string, FileStatus*>::iterator
it = m_files.begin();
it != m_files.end(); ++it) {
if (!it->second->received)
file_requests.push_back(it->first);
}
assert((s32) file_requests.size() ==
m_uncached_count - m_uncached_received_count);
client->request_media(file_requests);
}
}
void ClientMediaDownloader::conventionalTransferDone(
const std::string &name,
const std::string &data,
Client *client)
{
// Check that file was announced
std::map<std::string, FileStatus*>::iterator
file_iter = m_files.find(name);
if (file_iter == m_files.end()) {
errorstream << "Client: server sent media file that was"
<< "not announced, ignoring it: \"" << name << "\""
<< std::endl;
return;
}
FileStatus *filestatus = file_iter->second;
assert(filestatus != NULL);
// Check that file hasn't already been received
if (filestatus->received) {
errorstream << "Client: server sent media file that we already"
<< "received, ignoring it: \"" << name << "\""
<< std::endl;
return;
}
// Mark file as received, regardless of whether loading it works and
// whether the checksum matches (because at this point there is no
// other server that could send a replacement)
filestatus->received = true;
assert(m_uncached_received_count < m_uncached_count);
m_uncached_received_count++;
// Check that received file matches announced checksum
// If so, load it
checkAndLoad(name, filestatus->sha1, data, false, client);
}
bool ClientMediaDownloader::checkAndLoad(
const std::string &name, const std::string &sha1,
const std::string &data, bool is_from_cache, Client *client)
{
const char *cached_or_received = is_from_cache ? "cached" : "received";
const char *cached_or_received_uc = is_from_cache ? "Cached" : "Received";
std::string sha1_hex = hex_encode(sha1);
// Compute actual checksum of data
std::string data_sha1;
{
SHA1 data_sha1_calculator;
data_sha1_calculator.addBytes(data.c_str(), data.size());
unsigned char *data_tmpdigest = data_sha1_calculator.getDigest();
data_sha1.assign((char*) data_tmpdigest, 20);
free(data_tmpdigest);
}
// Check that received file matches announced checksum
if (data_sha1 != sha1) {
std::string data_sha1_hex = hex_encode(data_sha1);
infostream << "Client: "
<< cached_or_received_uc << " media file "
<< sha1_hex << " \"" << name << "\" "
<< "mismatches actual checksum " << data_sha1_hex
<< std::endl;
return false;
}
// Checksum is ok, try loading the file
bool success = client->loadMedia(data, name);
if (!success) {
infostream << "Client: "
<< "Failed to load " << cached_or_received << " media: "
<< sha1_hex << " \"" << name << "\""
<< std::endl;
return false;
}
verbosestream << "Client: "
<< "Loaded " << cached_or_received << " media: "
<< sha1_hex << " \"" << name << "\""
<< std::endl;
// Update cache (unless we just loaded the file from the cache)
if (!is_from_cache)
m_media_cache.update(sha1_hex, data);
return true;
}
/*
Minetest Hashset File Format
All values are stored in big-endian byte order.
[u32] signature: 'MTHS'
[u16] version: 1
For each hash in set:
[u8*20] SHA1 hash
Version changes:
1 - Initial version
*/
std::string ClientMediaDownloader::serializeRequiredHashSet()
{
std::ostringstream os(std::ios::binary);
writeU32(os, MTHASHSET_FILE_SIGNATURE); // signature
writeU16(os, 1); // version
// Write list of hashes of files that have not been
// received (found in cache) yet
for (std::map<std::string, FileStatus*>::iterator
it = m_files.begin();
it != m_files.end(); ++it) {
if (!it->second->received) {
assert(it->second->sha1.size() == 20);
os << it->second->sha1;
}
}
return os.str();
}
void ClientMediaDownloader::deSerializeHashSet(const std::string &data,
std::set<std::string> &result)
{
if (data.size() < 6 || data.size() % 20 != 6) {
throw SerializationError(
"ClientMediaDownloader::deSerializeHashSet: "
"invalid hash set file size");
}
const u8 *data_cstr = (const u8*) data.c_str();
u32 signature = readU32(&data_cstr[0]);
if (signature != MTHASHSET_FILE_SIGNATURE) {
throw SerializationError(
"ClientMediaDownloader::deSerializeHashSet: "
"invalid hash set file signature");
}
u16 version = readU16(&data_cstr[4]);
if (version != 1) {
throw SerializationError(
"ClientMediaDownloader::deSerializeHashSet: "
"unsupported hash set file version");
}
for (u32 pos = 6; pos < data.size(); pos += 20) {
result.insert(data.substr(pos, 20));
}
}

150
src/clientmedia.h Normal file

@ -0,0 +1,150 @@
/*
Minetest
Copyright (C) 2013 celeron55, Perttu Ahola <celeron55@gmail.com>
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation; either version 2.1 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License along
with this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
#ifndef CLIENTMEDIA_HEADER
#define CLIENTMEDIA_HEADER
#include "irrlichttypes.h"
#include "filecache.h"
#include <ostream>
#include <map>
#include <set>
#include <vector>
class Client;
struct HTTPFetchResult;
#define MTHASHSET_FILE_SIGNATURE 0x4d544853 // 'MTHS'
#define MTHASHSET_FILE_NAME "index.mth"
class ClientMediaDownloader
{
public:
ClientMediaDownloader();
~ClientMediaDownloader();
float getProgress() const {
if (m_uncached_count >= 1)
return 1.0 * m_uncached_received_count /
m_uncached_count;
else
return 0.0;
}
bool isStarted() const {
return m_initial_step_done;
}
// If this returns true, the downloader is done and can be deleted
bool isDone() const {
return m_initial_step_done &&
m_uncached_received_count == m_uncached_count;
}
// Add a file to the list of required file (but don't fetch it yet)
void addFile(std::string name, std::string sha1);
// Add a remote server to the list; ignored if not built with cURL
void addRemoteServer(std::string baseurl);
// Steps the media downloader:
// - May load media into client by calling client->loadMedia()
// - May check media cache for files
// - May add files to media cache
// - May start remote transfers by calling httpfetch_async
// - May check for completion of current remote transfers
// - May start conventional transfers by calling client->request_media()
// - May inform server that all media has been loaded
// by calling client->received_media()
// After step has been called once, don't call addFile/addRemoteServer.
void step(Client *client);
// Must be called for each file received through TOCLIENT_MEDIA
void conventionalTransferDone(
const std::string &name,
const std::string &data,
Client *client);
private:
struct FileStatus {
bool received;
std::string sha1;
s32 current_remote;
std::vector<s32> available_remotes;
};
struct RemoteServerStatus {
std::string baseurl;
s32 active_count;
bool request_by_filename;
};
void initialStep(Client *client);
void remoteHashSetReceived(const HTTPFetchResult &fetchresult);
void remoteMediaReceived(const HTTPFetchResult &fetchresult,
Client *client);
s32 selectRemoteServer(FileStatus *filestatus);
void startRemoteMediaTransfers();
void startConventionalTransfers(Client *client);
bool checkAndLoad(const std::string &name, const std::string &sha1,
const std::string &data, bool is_from_cache,
Client *client);
std::string serializeRequiredHashSet();
static void deSerializeHashSet(const std::string &data,
std::set<std::string> &result);
// Maps filename to file status
std::map<std::string, FileStatus*> m_files;
// Array of remote media servers
std::vector<RemoteServerStatus*> m_remotes;
// Filesystem-based media cache
FileCache m_media_cache;
// Has an attempt been made to load media files from the file cache?
// Have hash sets been requested from remote servers?
bool m_initial_step_done;
// Total number of media files to load
s32 m_uncached_count;
// Number of media files that have been received
s32 m_uncached_received_count;
// Status of remote transfers
unsigned long m_httpfetch_caller;
unsigned long m_httpfetch_next_id;
long m_httpfetch_timeout;
s32 m_httpfetch_active;
s32 m_httpfetch_active_limit;
s32 m_outstanding_hash_sets;
std::map<unsigned long, std::string> m_remote_file_transfers;
// All files up to this name have either been received from a
// remote server or failed on all remote servers, so those files
// don't need to be looked at again
// (use m_files.upper_bound(m_name_bound) to get an iterator)
std::string m_name_bound;
};
#endif // !CLIENTMEDIA_HEADER

@ -139,7 +139,7 @@ void set_default_settings(Settings *settings)
settings->setDefault("repeat_rightclick_time", "0.25"); settings->setDefault("repeat_rightclick_time", "0.25");
settings->setDefault("enable_particles", "true"); settings->setDefault("enable_particles", "true");
settings->setDefault("media_fetch_threads", "8"); settings->setDefault("curl_timeout", "5000");
settings->setDefault("curl_parallel_limit", "8"); settings->setDefault("curl_parallel_limit", "8");
settings->setDefault("serverlist_url", "servers.minetest.net"); settings->setDefault("serverlist_url", "servers.minetest.net");
@ -284,7 +284,6 @@ void set_default_settings(Settings *settings)
settings->setDefault("mgmath_generator", "mandelbox"); settings->setDefault("mgmath_generator", "mandelbox");
settings->setDefault("curl_timeout", "5000");
// IPv6 // IPv6
settings->setDefault("enable_ipv6", "true"); settings->setDefault("enable_ipv6", "true");

@ -23,12 +23,9 @@ with this program; if not, write to the Free Software Foundation, Inc.,
#include "clientserver.h" #include "clientserver.h"
#include "log.h" #include "log.h"
#include "filesys.h" #include "filesys.h"
#include "hex.h"
#include "sha1.h"
#include <string> #include <string>
#include <iostream> #include <iostream>
#include <fstream> #include <fstream>
#include <sstream>
#include <stdlib.h> #include <stdlib.h>
bool FileCache::loadByPath(const std::string &path, std::ostream &os) bool FileCache::loadByPath(const std::string &path, std::ostream &os)
@ -85,36 +82,8 @@ bool FileCache::update(const std::string &name, const std::string &data)
std::string path = m_dir + DIR_DELIM + name; std::string path = m_dir + DIR_DELIM + name;
return updateByPath(path, data); return updateByPath(path, data);
} }
bool FileCache::update_sha1(const std::string &data)
{
SHA1 sha1;
sha1.addBytes(data.c_str(), data.size());
unsigned char *digest = sha1.getDigest();
std::string sha1_raw((char*)digest, 20);
free(digest);
std::string sha1_hex = hex_encode(sha1_raw);
return update(sha1_hex, data);
}
bool FileCache::load(const std::string &name, std::ostream &os) bool FileCache::load(const std::string &name, std::ostream &os)
{ {
std::string path = m_dir + DIR_DELIM + name; std::string path = m_dir + DIR_DELIM + name;
return loadByPath(path, os); return loadByPath(path, os);
} }
bool FileCache::load_sha1(const std::string &sha1_raw, std::ostream &os)
{
std::ostringstream tmp_os(std::ios_base::binary);
if(!load(hex_encode(sha1_raw), tmp_os))
return false;
SHA1 sha1;
sha1.addBytes(tmp_os.str().c_str(), tmp_os.str().length());
unsigned char *digest = sha1.getDigest();
std::string sha1_real_raw((char*)digest, 20);
free(digest);
if(sha1_real_raw != sha1_raw){
verbosestream<<"FileCache["<<m_dir<<"]: filename "<<sha1_real_raw
<<" mismatches actual checksum"<<std::endl;
return false;
}
os<<tmp_os.str();
return true;
}

@ -36,9 +36,7 @@ public:
} }
bool update(const std::string &name, const std::string &data); bool update(const std::string &name, const std::string &data);
bool update_sha1(const std::string &data);
bool load(const std::string &name, std::ostream &os); bool load(const std::string &name, std::ostream &os);
bool load_sha1(const std::string &sha1_raw, std::ostream &os);
private: private:
std::string m_dir; std::string m_dir;

@ -69,6 +69,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
#include <iomanip> #include <iomanip>
#include <list> #include <list>
#include "util/directiontables.h" #include "util/directiontables.h"
#include "util/pointedthing.h"
/* /*
Text input system Text input system
@ -1207,7 +1208,7 @@ void the_game(
server->step(dtime); server->step(dtime);
// End condition // End condition
if(client.texturesReceived() && if(client.mediaReceived() &&
client.itemdefReceived() && client.itemdefReceived() &&
client.nodedefReceived()){ client.nodedefReceived()){
got_content = true; got_content = true;
@ -1422,7 +1423,7 @@ void the_game(
bool invert_mouse = g_settings->getBool("invert_mouse"); bool invert_mouse = g_settings->getBool("invert_mouse");
bool respawn_menu_active = false; bool respawn_menu_active = false;
bool update_wielded_item_trigger = false; bool update_wielded_item_trigger = true;
bool show_hud = true; bool show_hud = true;
bool show_chat = true; bool show_chat = true;
@ -2290,10 +2291,6 @@ void the_game(
delete(event.show_formspec.formspec); delete(event.show_formspec.formspec);
delete(event.show_formspec.formname); delete(event.show_formspec.formname);
} }
else if(event.type == CE_TEXTURES_UPDATED)
{
update_wielded_item_trigger = true;
}
else if(event.type == CE_SPAWN_PARTICLE) else if(event.type == CE_SPAWN_PARTICLE)
{ {
LocalPlayer* player = client.getEnv().getLocalPlayer(); LocalPlayer* player = client.getEnv().getLocalPlayer();

@ -2745,7 +2745,7 @@ void Server::ProcessData(u8 *data, u32 datasize, u16 peer_id)
std::string datastring((char*)&data[2], datasize-2); std::string datastring((char*)&data[2], datasize-2);
std::istringstream is(datastring, std::ios_base::binary); std::istringstream is(datastring, std::ios_base::binary);
std::list<MediaRequest> tosend; std::list<std::string> tosend;
u16 numfiles = readU16(is); u16 numfiles = readU16(is);
infostream<<"Sending "<<numfiles<<" files to " infostream<<"Sending "<<numfiles<<" files to "
@ -2754,7 +2754,7 @@ void Server::ProcessData(u8 *data, u32 datasize, u16 peer_id)
for(int i = 0; i < numfiles; i++) { for(int i = 0; i < numfiles; i++) {
std::string name = deSerializeString(is); std::string name = deSerializeString(is);
tosend.push_back(MediaRequest(name)); tosend.push_back(name);
verbosestream<<"TOSERVER_REQUEST_MEDIA: requested file " verbosestream<<"TOSERVER_REQUEST_MEDIA: requested file "
<<name<<std::endl; <<name<<std::endl;
} }
@ -4453,7 +4453,7 @@ struct SendableMedia
}; };
void Server::sendRequestedMedia(u16 peer_id, void Server::sendRequestedMedia(u16 peer_id,
const std::list<MediaRequest> &tosend) const std::list<std::string> &tosend)
{ {
DSTACK(__FUNCTION_NAME); DSTACK(__FUNCTION_NAME);
@ -4470,17 +4470,19 @@ void Server::sendRequestedMedia(u16 peer_id,
u32 file_size_bunch_total = 0; u32 file_size_bunch_total = 0;
for(std::list<MediaRequest>::const_iterator i = tosend.begin(); for(std::list<std::string>::const_iterator i = tosend.begin();
i != tosend.end(); ++i) i != tosend.end(); ++i)
{ {
if(m_media.find(i->name) == m_media.end()){ const std::string &name = *i;
if(m_media.find(name) == m_media.end()){
errorstream<<"Server::sendRequestedMedia(): Client asked for " errorstream<<"Server::sendRequestedMedia(): Client asked for "
<<"unknown file \""<<(i->name)<<"\""<<std::endl; <<"unknown file \""<<(name)<<"\""<<std::endl;
continue; continue;
} }
//TODO get path + name //TODO get path + name
std::string tpath = m_media[(*i).name].path; std::string tpath = m_media[name].path;
// Read data // Read data
std::ifstream fis(tpath.c_str(), std::ios_base::binary); std::ifstream fis(tpath.c_str(), std::ios_base::binary);
@ -4506,14 +4508,14 @@ void Server::sendRequestedMedia(u16 peer_id,
} }
if(bad){ if(bad){
errorstream<<"Server::sendRequestedMedia(): Failed to read \"" errorstream<<"Server::sendRequestedMedia(): Failed to read \""
<<(*i).name<<"\""<<std::endl; <<name<<"\""<<std::endl;
continue; continue;
} }
/*infostream<<"Server::sendRequestedMedia(): Loaded \"" /*infostream<<"Server::sendRequestedMedia(): Loaded \""
<<tname<<"\""<<std::endl;*/ <<tname<<"\""<<std::endl;*/
// Put in list // Put in list
file_bunches[file_bunches.size()-1].push_back( file_bunches[file_bunches.size()-1].push_back(
SendableMedia((*i).name, tpath, tmp_os.str())); SendableMedia(name, tpath, tmp_os.str()));
// Start next bunch if got enough data // Start next bunch if got enough data
if(file_size_bunch_total >= bytes_per_bunch){ if(file_size_bunch_total >= bytes_per_bunch){

@ -153,15 +153,6 @@ struct PrioritySortedBlockTransfer
u16 peer_id; u16 peer_id;
}; };
struct MediaRequest
{
std::string name;
MediaRequest(const std::string &name_=""):
name(name_)
{}
};
struct MediaInfo struct MediaInfo
{ {
std::string path; std::string path;
@ -569,7 +560,7 @@ private:
void fillMediaCache(); void fillMediaCache();
void sendMediaAnnouncement(u16 peer_id); void sendMediaAnnouncement(u16 peer_id);
void sendRequestedMedia(u16 peer_id, void sendRequestedMedia(u16 peer_id,
const std::list<MediaRequest> &tosend); const std::list<std::string> &tosend);
void sendDetachedInventory(const std::string &name, u16 peer_id); void sendDetachedInventory(const std::string &name, u16 peer_id);
void sendDetachedInventoryToAll(const std::string &name); void sendDetachedInventoryToAll(const std::string &name);