mirror of
https://github.com/minetest/minetest.git
synced 2024-08-18 20:44:16 +02:00
Added ability to fetch media from remote server (using cURL library)
This commit is contained in:
parent
aa46e5c5e7
commit
3578e1d4a7
17
cmake/Modules/FindCURL.cmake
Normal file
17
cmake/Modules/FindCURL.cmake
Normal file
@ -0,0 +1,17 @@
|
||||
# - Find curl
|
||||
# Find the native CURL headers and libraries.
|
||||
#
|
||||
# CURL_INCLUDE_DIR - where to find curl/curl.h, etc.
|
||||
# CURL_LIBRARY - List of libraries when using curl.
|
||||
# CURL_FOUND - True if curl found.
|
||||
|
||||
# Look for the header file.
|
||||
FIND_PATH(CURL_INCLUDE_DIR NAMES curl/curl.h)
|
||||
|
||||
# Look for the library.
|
||||
FIND_LIBRARY(CURL_LIBRARY NAMES curl)
|
||||
|
||||
# handle the QUIETLY and REQUIRED arguments and set CURL_FOUND to TRUE if
|
||||
# all listed variables are TRUE
|
||||
INCLUDE(FindPackageHandleStandardArgs)
|
||||
FIND_PACKAGE_HANDLE_STANDARD_ARGS(CURL DEFAULT_MSG CURL_LIBRARY CURL_INCLUDE_DIR)
|
@ -141,6 +141,10 @@
|
||||
# 2: enable high level shaders
|
||||
#enable_shaders = 2
|
||||
|
||||
# will only work for servers which use remote_media setting
|
||||
# and only for clients compiled with cURL
|
||||
#media_fetch_threads = 8
|
||||
|
||||
#
|
||||
# Server stuff
|
||||
#
|
||||
@ -217,3 +221,4 @@
|
||||
#congestion_control_aim_rtt = 0.2
|
||||
#congestion_control_max_rate = 400
|
||||
#congestion_control_min_rate = 10
|
||||
#remote_media =
|
||||
|
@ -6,6 +6,19 @@ mark_as_advanced(EXECUTABLE_OUTPUT_PATH LIBRARY_OUTPUT_PATH)
|
||||
mark_as_advanced(JTHREAD_INCLUDE_DIR JTHREAD_LIBRARY)
|
||||
mark_as_advanced(SQLITE3_INCLUDE_DIR SQLITE3_LIBRARY)
|
||||
|
||||
option(ENABLE_CURL "Enable cURL support for fetching media" 1)
|
||||
|
||||
if (NOT ENABLE_CURL)
|
||||
mark_as_advanced(CLEAR CURL_LIBRARY CURL_INCLUDE_DIR)
|
||||
endif(NOT ENABLE_CURL)
|
||||
|
||||
find_package(CURL)
|
||||
set(USE_CURL 0)
|
||||
if (CURL_FOUND AND ENABLE_CURL)
|
||||
message(STATUS "cURL support enabled")
|
||||
set(USE_CURL 1)
|
||||
endif(CURL_FOUND AND ENABLE_CURL)
|
||||
|
||||
# user-visible option to enable/disable gettext usage
|
||||
OPTION(ENABLE_GETTEXT "Use GetText for internationalization" 0)
|
||||
|
||||
@ -307,6 +320,16 @@ if(BUILD_CLIENT)
|
||||
${PLATFORM_LIBS}
|
||||
${CLIENT_PLATFORM_LIBS}
|
||||
)
|
||||
|
||||
if(USE_CURL)
|
||||
target_link_libraries(
|
||||
${PROJECT_NAME}
|
||||
${CURL_LIBRARY}
|
||||
)
|
||||
include_directories(
|
||||
${CURL_INCLUDE_DIR}
|
||||
)
|
||||
endif(USE_CURL)
|
||||
endif(BUILD_CLIENT)
|
||||
|
||||
if(BUILD_SERVER)
|
||||
|
221
src/client.cpp
221
src/client.cpp
@ -44,21 +44,17 @@ with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
#include "hex.h"
|
||||
#include "IMeshCache.h"
|
||||
#include "util/serialize.h"
|
||||
#include "config.h"
|
||||
|
||||
#if USE_CURL
|
||||
#include <curl/curl.h>
|
||||
#endif
|
||||
|
||||
static std::string getMediaCacheDir()
|
||||
{
|
||||
return porting::path_user + DIR_DELIM + "cache" + DIR_DELIM + "media";
|
||||
}
|
||||
|
||||
struct MediaRequest
|
||||
{
|
||||
std::string name;
|
||||
|
||||
MediaRequest(const std::string &name_=""):
|
||||
name(name_)
|
||||
{}
|
||||
};
|
||||
|
||||
/*
|
||||
QueuedMeshUpdate
|
||||
*/
|
||||
@ -223,6 +219,45 @@ void * MeshUpdateThread::Thread()
|
||||
return NULL;
|
||||
}
|
||||
|
||||
void * MediaFetchThread::Thread()
|
||||
{
|
||||
ThreadStarted();
|
||||
|
||||
log_register_thread("MediaFetchThread");
|
||||
|
||||
DSTACK(__FUNCTION_NAME);
|
||||
|
||||
BEGIN_DEBUG_EXCEPTION_HANDLER
|
||||
|
||||
#if USE_CURL
|
||||
CURL *curl;
|
||||
CURLcode res;
|
||||
for (core::list<MediaRequest>::Iterator i = m_file_requests.begin();
|
||||
i != m_file_requests.end(); i++) {
|
||||
curl = curl_easy_init();
|
||||
assert(curl);
|
||||
curl_easy_setopt(curl, CURLOPT_URL, (m_remote_url + i->name).c_str());
|
||||
curl_easy_setopt(curl, CURLOPT_FAILONERROR, true);
|
||||
std::ostringstream stream;
|
||||
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curl_write_data);
|
||||
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &stream);
|
||||
res = curl_easy_perform(curl);
|
||||
if (res == CURLE_OK) {
|
||||
std::string data = stream.str();
|
||||
m_file_data.push_back(make_pair(i->name, data));
|
||||
} else {
|
||||
m_failed.push_back(*i);
|
||||
infostream << "cURL request failed for " << i->name << std::endl;
|
||||
}
|
||||
curl_easy_cleanup(curl);
|
||||
}
|
||||
#endif
|
||||
|
||||
END_DEBUG_EXCEPTION_HANDLER(errorstream)
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
||||
Client::Client(
|
||||
IrrlichtDevice *device,
|
||||
const char *playername,
|
||||
@ -263,8 +298,9 @@ Client::Client(
|
||||
m_password(password),
|
||||
m_access_denied(false),
|
||||
m_media_cache(getMediaCacheDir()),
|
||||
m_media_receive_progress(0),
|
||||
m_media_received(false),
|
||||
m_media_receive_started(false),
|
||||
m_media_count(0),
|
||||
m_media_received_count(0),
|
||||
m_itemdef_received(false),
|
||||
m_nodedef_received(false),
|
||||
m_time_of_day_set(false),
|
||||
@ -730,6 +766,63 @@ void Client::step(float dtime)
|
||||
g_profiler->graphAdd("num_processed_meshes", num_processed_meshes);
|
||||
}
|
||||
|
||||
/*
|
||||
Load fetched media
|
||||
*/
|
||||
if (m_media_receive_started) {
|
||||
bool all_stopped = true;
|
||||
for (core::list<MediaFetchThread>::Iterator thread = m_media_fetch_threads.begin();
|
||||
thread != m_media_fetch_threads.end(); thread++) {
|
||||
all_stopped &= !thread->IsRunning();
|
||||
while (thread->m_file_data.size() > 0) {
|
||||
std::pair <std::string, std::string> out = thread->m_file_data.pop_front();
|
||||
++m_media_received_count;
|
||||
|
||||
bool success = loadMedia(out.second, out.first);
|
||||
if(success){
|
||||
verbosestream<<"Client: Loaded received media: "
|
||||
<<"\""<<out.first<<"\". Caching."<<std::endl;
|
||||
} else{
|
||||
infostream<<"Client: Failed to load received media: "
|
||||
<<"\""<<out.first<<"\". Not caching."<<std::endl;
|
||||
continue;
|
||||
}
|
||||
|
||||
bool did = fs::CreateAllDirs(getMediaCacheDir());
|
||||
if(!did){
|
||||
errorstream<<"Could not create media cache directory"
|
||||
<<std::endl;
|
||||
}
|
||||
|
||||
{
|
||||
core::map<std::string, std::string>::Node *n;
|
||||
n = m_media_name_sha1_map.find(out.first);
|
||||
if(n == NULL)
|
||||
errorstream<<"The server sent a file that has not "
|
||||
<<"been announced."<<std::endl;
|
||||
else
|
||||
m_media_cache.update_sha1(out.second);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (all_stopped) {
|
||||
core::list<MediaRequest> fetch_failed;
|
||||
for (core::list<MediaFetchThread>::Iterator thread = m_media_fetch_threads.begin();
|
||||
thread != m_media_fetch_threads.end(); thread++) {
|
||||
for (core::list<MediaRequest>::Iterator request = thread->m_failed.begin();
|
||||
request != thread->m_failed.end(); request++)
|
||||
fetch_failed.push_back(*request);
|
||||
thread->m_failed.clear();
|
||||
}
|
||||
if (fetch_failed.size() > 0) {
|
||||
infostream << "Failed to remote-fetch " << fetch_failed.size() << " files. "
|
||||
<< "Requesting them the usual way." << std::endl;
|
||||
request_media(fetch_failed);
|
||||
}
|
||||
m_media_fetch_threads.clear();
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
If the server didn't update the inventory in a while, revert
|
||||
the local inventory (so the player notices the lag problem
|
||||
@ -907,6 +1000,34 @@ void Client::deletingPeer(con::Peer *peer, bool timeout)
|
||||
<<"(timeout="<<timeout<<")"<<std::endl;
|
||||
}
|
||||
|
||||
/*
|
||||
u16 command
|
||||
u16 number of files requested
|
||||
for each file {
|
||||
u16 length of name
|
||||
string name
|
||||
}
|
||||
*/
|
||||
void Client::request_media(const core::list<MediaRequest> &file_requests)
|
||||
{
|
||||
std::ostringstream os(std::ios_base::binary);
|
||||
writeU16(os, TOSERVER_REQUEST_MEDIA);
|
||||
writeU16(os, file_requests.size());
|
||||
|
||||
for(core::list<MediaRequest>::ConstIterator i = file_requests.begin();
|
||||
i != file_requests.end(); i++) {
|
||||
os<<serializeString(i->name);
|
||||
}
|
||||
|
||||
// Make data buffer
|
||||
std::string s = os.str();
|
||||
SharedBuffer<u8> data((u8*)s.c_str(), s.size());
|
||||
// Send as reliable
|
||||
Send(0, data, true);
|
||||
infostream<<"Client: Sending media request list to server ("
|
||||
<<file_requests.size()<<" files)"<<std::endl;
|
||||
}
|
||||
|
||||
void Client::ReceiveAll()
|
||||
{
|
||||
DSTACK(__FUNCTION_NAME);
|
||||
@ -1514,37 +1635,56 @@ void Client::ProcessData(u8 *data, u32 datasize, u16 sender_peer_id)
|
||||
file_requests.push_back(MediaRequest(name));
|
||||
}
|
||||
|
||||
std::string remote_media = "";
|
||||
try {
|
||||
remote_media = deSerializeString(is);
|
||||
}
|
||||
catch(SerializationError) {
|
||||
// not supported by server or turned off
|
||||
}
|
||||
|
||||
m_media_count = file_requests.size();
|
||||
m_media_receive_started = true;
|
||||
|
||||
if (remote_media == "" || !USE_CURL) {
|
||||
request_media(file_requests);
|
||||
} else {
|
||||
#if USE_CURL
|
||||
for (size_t i = 0; i < g_settings->getU16("media_fetch_threads"); ++i) {
|
||||
m_media_fetch_threads.push_back(MediaFetchThread(this));
|
||||
}
|
||||
|
||||
core::list<MediaFetchThread>::Iterator cur = m_media_fetch_threads.begin();
|
||||
for(core::list<MediaRequest>::Iterator i = file_requests.begin();
|
||||
i != file_requests.end(); i++) {
|
||||
cur->m_file_requests.push_back(*i);
|
||||
cur++;
|
||||
if (cur == m_media_fetch_threads.end())
|
||||
cur = m_media_fetch_threads.begin();
|
||||
}
|
||||
for (core::list<MediaFetchThread>::Iterator i = m_media_fetch_threads.begin();
|
||||
i != m_media_fetch_threads.end(); i++) {
|
||||
i->m_remote_url = remote_media;
|
||||
i->Start();
|
||||
}
|
||||
#endif
|
||||
|
||||
// notify server we received everything
|
||||
std::ostringstream os(std::ios_base::binary);
|
||||
writeU16(os, TOSERVER_RECEIVED_MEDIA);
|
||||
std::string s = os.str();
|
||||
SharedBuffer<u8> data((u8*)s.c_str(), s.size());
|
||||
// Send as reliable
|
||||
Send(0, data, true);
|
||||
}
|
||||
ClientEvent event;
|
||||
event.type = CE_TEXTURES_UPDATED;
|
||||
m_client_event_queue.push_back(event);
|
||||
|
||||
/*
|
||||
u16 command
|
||||
u16 number of files requested
|
||||
for each file {
|
||||
u16 length of name
|
||||
string name
|
||||
}
|
||||
*/
|
||||
std::ostringstream os(std::ios_base::binary);
|
||||
writeU16(os, TOSERVER_REQUEST_MEDIA);
|
||||
writeU16(os, file_requests.size());
|
||||
|
||||
for(core::list<MediaRequest>::Iterator i = file_requests.begin();
|
||||
i != file_requests.end(); i++) {
|
||||
os<<serializeString(i->name);
|
||||
}
|
||||
|
||||
// Make data buffer
|
||||
std::string s = os.str();
|
||||
SharedBuffer<u8> data((u8*)s.c_str(), s.size());
|
||||
// Send as reliable
|
||||
Send(0, data, true);
|
||||
infostream<<"Client: Sending media request list to server ("
|
||||
<<file_requests.size()<<" files)"<<std::endl;
|
||||
}
|
||||
else if(command == TOCLIENT_MEDIA)
|
||||
{
|
||||
if (m_media_count == 0)
|
||||
return;
|
||||
std::string datastring((char*)&data[2], datasize-2);
|
||||
std::istringstream is(datastring, std::ios_base::binary);
|
||||
|
||||
@ -1566,17 +1706,12 @@ void Client::ProcessData(u8 *data, u32 datasize, u16 sender_peer_id)
|
||||
*/
|
||||
int num_bunches = readU16(is);
|
||||
int bunch_i = readU16(is);
|
||||
if(num_bunches >= 2)
|
||||
m_media_receive_progress = (float)bunch_i / (float)(num_bunches - 1);
|
||||
else
|
||||
m_media_receive_progress = 1.0;
|
||||
if(bunch_i == num_bunches - 1)
|
||||
m_media_received = true;
|
||||
int num_files = readU32(is);
|
||||
infostream<<"Client: Received files: bunch "<<bunch_i<<"/"
|
||||
<<num_bunches<<" files="<<num_files
|
||||
<<" size="<<datasize<<std::endl;
|
||||
for(int i=0; i<num_files; i++){
|
||||
m_media_received_count++;
|
||||
std::string name = deSerializeString(is);
|
||||
std::string data = deSerializeLongString(is);
|
||||
|
||||
@ -2458,7 +2593,7 @@ void Client::afterContentReceived()
|
||||
infostream<<"Client::afterContentReceived() started"<<std::endl;
|
||||
assert(m_itemdef_received);
|
||||
assert(m_nodedef_received);
|
||||
assert(m_media_received);
|
||||
assert(texturesReceived());
|
||||
|
||||
// remove the information about which checksum each texture
|
||||
// ought to have
|
||||
|
36
src/client.h
36
src/client.h
@ -33,6 +33,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
#include "filesys.h"
|
||||
#include "filecache.h"
|
||||
#include "localplayer.h"
|
||||
#include "server.h"
|
||||
#include "util/pointedthing.h"
|
||||
|
||||
struct MeshMakeData;
|
||||
@ -129,6 +130,24 @@ public:
|
||||
IGameDef *m_gamedef;
|
||||
};
|
||||
|
||||
class MediaFetchThread : public SimpleThread
|
||||
{
|
||||
public:
|
||||
|
||||
MediaFetchThread(IGameDef *gamedef):
|
||||
m_gamedef(gamedef)
|
||||
{
|
||||
}
|
||||
|
||||
void * Thread();
|
||||
|
||||
core::list<MediaRequest> m_file_requests;
|
||||
MutexedQueue<std::pair<std::string, std::string> > m_file_data;
|
||||
core::list<MediaRequest> m_failed;
|
||||
std::string m_remote_url;
|
||||
IGameDef *m_gamedef;
|
||||
};
|
||||
|
||||
enum ClientEventType
|
||||
{
|
||||
CE_NONE,
|
||||
@ -289,10 +308,13 @@ public:
|
||||
{ return m_access_denied_reason; }
|
||||
|
||||
float mediaReceiveProgress()
|
||||
{ return m_media_receive_progress; }
|
||||
{
|
||||
if (!m_media_receive_started) return 0;
|
||||
return 1.0 * m_media_received_count / m_media_count;
|
||||
}
|
||||
|
||||
bool texturesReceived()
|
||||
{ return m_media_received; }
|
||||
{ return m_media_receive_started && m_media_received_count == m_media_count; }
|
||||
bool itemdefReceived()
|
||||
{ return m_itemdef_received; }
|
||||
bool nodedefReceived()
|
||||
@ -318,7 +340,9 @@ private:
|
||||
|
||||
// Insert a media file appropriately into the appropriate manager
|
||||
bool loadMedia(const std::string &data, const std::string &filename);
|
||||
|
||||
|
||||
void request_media(const core::list<MediaRequest> &file_requests);
|
||||
|
||||
// Virtual methods from con::PeerHandler
|
||||
void peerAdded(con::Peer *peer);
|
||||
void deletingPeer(con::Peer *peer, bool timeout);
|
||||
@ -347,6 +371,7 @@ private:
|
||||
MtEventManager *m_event;
|
||||
|
||||
MeshUpdateThread m_mesh_update_thread;
|
||||
core::list<MediaFetchThread> m_media_fetch_threads;
|
||||
ClientEnvironment m_env;
|
||||
con::Connection m_con;
|
||||
IrrlichtDevice *m_device;
|
||||
@ -375,8 +400,9 @@ private:
|
||||
FileCache m_media_cache;
|
||||
// Mapping from media file name to SHA1 checksum
|
||||
core::map<std::string, std::string> m_media_name_sha1_map;
|
||||
float m_media_receive_progress;
|
||||
bool m_media_received;
|
||||
bool m_media_receive_started;
|
||||
u32 m_media_count;
|
||||
u32 m_media_received_count;
|
||||
bool m_itemdef_received;
|
||||
bool m_nodedef_received;
|
||||
friend class FarMesh;
|
||||
|
@ -267,6 +267,8 @@ enum ToClientCommand
|
||||
u32 length of data
|
||||
data
|
||||
}
|
||||
u16 length of remote media server url (if applicable)
|
||||
string url
|
||||
*/
|
||||
|
||||
TOCLIENT_TOOLDEF = 0x39,
|
||||
@ -571,6 +573,10 @@ enum ToServerCommand
|
||||
}
|
||||
*/
|
||||
|
||||
TOSERVER_RECEIVED_MEDIA = 0x41,
|
||||
/*
|
||||
u16 command
|
||||
*/
|
||||
};
|
||||
|
||||
#endif
|
||||
|
@ -7,6 +7,7 @@
|
||||
#define CMAKE_VERSION_STRING "@VERSION_STRING@"
|
||||
#define CMAKE_RUN_IN_PLACE @RUN_IN_PLACE@
|
||||
#define CMAKE_USE_GETTEXT @USE_GETTEXT@
|
||||
#define CMAKE_USE_CURL @USE_CURL@
|
||||
#define CMAKE_USE_SOUND @USE_SOUND@
|
||||
#define CMAKE_STATIC_SHAREDIR "@SHAREDIR@"
|
||||
|
||||
|
@ -11,6 +11,7 @@
|
||||
#define RUN_IN_PLACE 0
|
||||
#define USE_GETTEXT 0
|
||||
#define USE_SOUND 0
|
||||
#define USE_CURL 0
|
||||
#define STATIC_SHAREDIR ""
|
||||
#define BUILD_INFO "non-cmake"
|
||||
|
||||
@ -26,6 +27,8 @@
|
||||
#define USE_GETTEXT CMAKE_USE_GETTEXT
|
||||
#undef USE_SOUND
|
||||
#define USE_SOUND CMAKE_USE_SOUND
|
||||
#undef USE_CURL
|
||||
#define USE_CURL CMAKE_USE_CURL
|
||||
#undef STATIC_SHAREDIR
|
||||
#define STATIC_SHAREDIR CMAKE_STATIC_SHAREDIR
|
||||
#undef BUILD_INFO
|
||||
|
@ -119,6 +119,8 @@ void set_default_settings(Settings *settings)
|
||||
settings->setDefault("preload_item_visuals", "true");
|
||||
settings->setDefault("enable_shaders", "2");
|
||||
|
||||
settings->setDefault("media_fetch_threads", "8");
|
||||
|
||||
// Server stuff
|
||||
// "map-dir" doesn't exist by default.
|
||||
settings->setDefault("default_game", "minetest");
|
||||
@ -158,5 +160,6 @@ void set_default_settings(Settings *settings)
|
||||
settings->setDefault("congestion_control_aim_rtt", "0.2");
|
||||
settings->setDefault("congestion_control_max_rate", "400");
|
||||
settings->setDefault("congestion_control_min_rate", "10");
|
||||
settings->setDefault("remote_media", "");
|
||||
}
|
||||
|
||||
|
@ -2885,6 +2885,9 @@ void Server::ProcessData(u8 *data, u32 datasize, u16 peer_id)
|
||||
// (definitions and files)
|
||||
getClient(peer_id)->definitions_sent = true;
|
||||
}
|
||||
else if(command == TOSERVER_RECEIVED_MEDIA) {
|
||||
getClient(peer_id)->definitions_sent = true;
|
||||
}
|
||||
else if(command == TOSERVER_INTERACT)
|
||||
{
|
||||
std::string datastring((char*)&data[2], datasize-2);
|
||||
@ -4217,6 +4220,7 @@ void Server::sendMediaAnnouncement(u16 peer_id)
|
||||
os<<serializeString(j->name);
|
||||
os<<serializeString(j->sha1_digest);
|
||||
}
|
||||
os<<serializeString(g_settings->get("remote_media"));
|
||||
|
||||
// Make data buffer
|
||||
std::string s = os.str();
|
||||
@ -4224,7 +4228,6 @@ void Server::sendMediaAnnouncement(u16 peer_id)
|
||||
|
||||
// Send as reliable
|
||||
m_con.Send(peer_id, 0, data, true);
|
||||
|
||||
}
|
||||
|
||||
struct SendableMedia
|
||||
|
@ -41,3 +41,9 @@ std::string translatePassword(std::string playername, std::wstring password)
|
||||
return pwd;
|
||||
}
|
||||
|
||||
size_t curl_write_data(char *ptr, size_t size, size_t nmemb, void *userdata) {
|
||||
std::ostringstream *stream = (std::ostringstream*)userdata;
|
||||
size_t count = size * nmemb;
|
||||
stream->write(ptr, count);
|
||||
return count;
|
||||
}
|
||||
|
@ -282,6 +282,7 @@ inline std::string wrap_rows(const std::string &from, u32 rowlen)
|
||||
}
|
||||
|
||||
std::string translatePassword(std::string playername, std::wstring password);
|
||||
size_t curl_write_data(char *ptr, size_t size, size_t nmemb, void *userdata);
|
||||
|
||||
#endif
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user