Added ability to fetch media from remote server (using cURL library)

cmake/Modules/FindCURL.cmake

@@ -0,0 +1,17 @@
+# - Find curl
+# Find the native CURL headers and libraries.
+#
+#  CURL_INCLUDE_DIR - where to find curl/curl.h, etc.
+#  CURL_LIBRARY    - List of libraries when using curl.
+#  CURL_FOUND        - True if curl found.
+
+# Look for the header file.
+FIND_PATH(CURL_INCLUDE_DIR NAMES curl/curl.h)
+
+# Look for the library.
+FIND_LIBRARY(CURL_LIBRARY NAMES curl)
+
+# handle the QUIETLY and REQUIRED arguments and set CURL_FOUND to TRUE if
+# all listed variables are TRUE
+INCLUDE(FindPackageHandleStandardArgs)
+FIND_PACKAGE_HANDLE_STANDARD_ARGS(CURL DEFAULT_MSG CURL_LIBRARY CURL_INCLUDE_DIR)

minetest.conf.example

@@ -141,6 +141,10 @@
 # 2: enable high level shaders
 #enable_shaders = 2
 
+# will only work for servers which use remote_media setting
+# and only for clients compiled with cURL
+#media_fetch_threads = 8
+
 #
 # Server stuff
 #
@@ -217,3 +221,4 @@
 #congestion_control_aim_rtt = 0.2
 #congestion_control_max_rate = 400
 #congestion_control_min_rate = 10
+#remote_media =

src/CMakeLists.txt

@@ -6,6 +6,19 @@ mark_as_advanced(EXECUTABLE_OUTPUT_PATH LIBRARY_OUTPUT_PATH)
 mark_as_advanced(JTHREAD_INCLUDE_DIR JTHREAD_LIBRARY)
 mark_as_advanced(SQLITE3_INCLUDE_DIR SQLITE3_LIBRARY)
 
+option(ENABLE_CURL "Enable cURL support for fetching media" 1)
+
+if (NOT ENABLE_CURL)
+	mark_as_advanced(CLEAR CURL_LIBRARY CURL_INCLUDE_DIR)
+endif(NOT ENABLE_CURL)
+
+find_package(CURL)
+set(USE_CURL 0)
+if (CURL_FOUND AND ENABLE_CURL)
+	message(STATUS "cURL support enabled")
+	set(USE_CURL 1)
+endif(CURL_FOUND AND ENABLE_CURL)
+
 # user-visible option to enable/disable gettext usage
 OPTION(ENABLE_GETTEXT "Use GetText for internationalization" 0)
 
@@ -307,6 +320,16 @@ if(BUILD_CLIENT)
 		${PLATFORM_LIBS}
 		${CLIENT_PLATFORM_LIBS}
 	)
+
+	if(USE_CURL)
+		target_link_libraries(
+			${PROJECT_NAME}
+			${CURL_LIBRARY}
+		)
+		include_directories(
+			${CURL_INCLUDE_DIR}
+		)
+	endif(USE_CURL)
 endif(BUILD_CLIENT)
 
 if(BUILD_SERVER)

src/client.cpp

@@ -44,21 +44,17 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "hex.h"
 #include "IMeshCache.h"
 #include "util/serialize.h"
+#include "config.h"
+
+#if USE_CURL
+#include <curl/curl.h>
+#endif
 
 static std::string getMediaCacheDir()
 {
 	return porting::path_user + DIR_DELIM + "cache" + DIR_DELIM + "media";
 }
 
-struct MediaRequest
-{
-	std::string name;
-
-	MediaRequest(const std::string &name_=""):
-		name(name_)
-	{}
-};
-
 /*
 	QueuedMeshUpdate
 */
@@ -223,6 +219,45 @@ void * MeshUpdateThread::Thread()
 	return NULL;
 }
 
+void * MediaFetchThread::Thread()
+{
+	ThreadStarted();
+
+	log_register_thread("MediaFetchThread");
+
+	DSTACK(__FUNCTION_NAME);
+
+	BEGIN_DEBUG_EXCEPTION_HANDLER
+
+	#if USE_CURL
+	CURL *curl;
+	CURLcode res;
+	for (core::list<MediaRequest>::Iterator i = m_file_requests.begin();
+			i != m_file_requests.end(); i++) {
+		curl = curl_easy_init();
+		assert(curl);
+		curl_easy_setopt(curl, CURLOPT_URL, (m_remote_url + i->name).c_str());
+		curl_easy_setopt(curl, CURLOPT_FAILONERROR, true);
+		std::ostringstream stream;
+		curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curl_write_data);
+		curl_easy_setopt(curl, CURLOPT_WRITEDATA, &stream);
+		res = curl_easy_perform(curl);
+		if (res == CURLE_OK) {
+			std::string data = stream.str();
+			m_file_data.push_back(make_pair(i->name, data));
+		} else {
+			m_failed.push_back(*i);
+			infostream << "cURL request failed for " << i->name << std::endl;
+		}
+		curl_easy_cleanup(curl);
+	}
+	#endif
+
+	END_DEBUG_EXCEPTION_HANDLER(errorstream)
+
+	return NULL;
+}
+
 Client::Client(
 		IrrlichtDevice *device,
 		const char *playername,
@@ -263,8 +298,9 @@ Client::Client(
 	m_password(password),
 	m_access_denied(false),
 	m_media_cache(getMediaCacheDir()),
-	m_media_receive_progress(0),
-	m_media_received(false),
+	m_media_receive_started(false),
+	m_media_count(0),
+	m_media_received_count(0),
 	m_itemdef_received(false),
 	m_nodedef_received(false),
 	m_time_of_day_set(false),
@@ -730,6 +766,63 @@ void Client::step(float dtime)
 			g_profiler->graphAdd("num_processed_meshes", num_processed_meshes);
 	}
 
+	/*
+		Load fetched media
+	*/
+	if (m_media_receive_started) {
+		bool all_stopped = true;
+		for (core::list<MediaFetchThread>::Iterator thread = m_media_fetch_threads.begin();
+				thread != m_media_fetch_threads.end(); thread++) {
+			all_stopped &= !thread->IsRunning();
+			while (thread->m_file_data.size() > 0) {
+				std::pair <std::string, std::string> out = thread->m_file_data.pop_front();
+				++m_media_received_count;
+
+				bool success = loadMedia(out.second, out.first);
+				if(success){
+					verbosestream<<"Client: Loaded received media: "
+							<<"\""<<out.first<<"\". Caching."<<std::endl;
+				} else{
+					infostream<<"Client: Failed to load received media: "
+							<<"\""<<out.first<<"\". Not caching."<<std::endl;
+					continue;
+				}
+
+				bool did = fs::CreateAllDirs(getMediaCacheDir());
+				if(!did){
+					errorstream<<"Could not create media cache directory"
+							<<std::endl;
+				}
+
+				{
+					core::map<std::string, std::string>::Node *n;
+					n = m_media_name_sha1_map.find(out.first);
+					if(n == NULL)
+						errorstream<<"The server sent a file that has not "
+								<<"been announced."<<std::endl;
+					else
+						m_media_cache.update_sha1(out.second);
+				}
+			}
+		}
+		if (all_stopped) {
+			core::list<MediaRequest> fetch_failed;
+			for (core::list<MediaFetchThread>::Iterator thread = m_media_fetch_threads.begin();
+					thread != m_media_fetch_threads.end(); thread++) {
+				for (core::list<MediaRequest>::Iterator request = thread->m_failed.begin();
+						request != thread->m_failed.end(); request++)
+					fetch_failed.push_back(*request);
+				thread->m_failed.clear();
+			}
+			if (fetch_failed.size() > 0) {
+				infostream << "Failed to remote-fetch " << fetch_failed.size() << " files. "
+						<< "Requesting them the usual way." << std::endl;
+				request_media(fetch_failed);
+			}
+			m_media_fetch_threads.clear();
+		}
+	}
+
 	/*
 		If the server didn't update the inventory in a while, revert
 		the local inventory (so the player notices the lag problem
@@ -907,6 +1000,34 @@ void Client::deletingPeer(con::Peer *peer, bool timeout)
 			<<"(timeout="<<timeout<<")"<<std::endl;
 }
 
+/*
+	u16 command
+	u16 number of files requested
+	for each file {
+		u16 length of name
+		string name
+	}
+*/
+void Client::request_media(const core::list<MediaRequest> &file_requests)
+{
+	std::ostringstream os(std::ios_base::binary);
+	writeU16(os, TOSERVER_REQUEST_MEDIA);
+	writeU16(os, file_requests.size());
+
+	for(core::list<MediaRequest>::ConstIterator i = file_requests.begin();
+			i != file_requests.end(); i++) {
+		os<<serializeString(i->name);
+	}
+
+	// Make data buffer
+	std::string s = os.str();
+	SharedBuffer<u8> data((u8*)s.c_str(), s.size());
+	// Send as reliable
+	Send(0, data, true);
+	infostream<<"Client: Sending media request list to server ("
+			<<file_requests.size()<<" files)"<<std::endl;
+}
+
 void Client::ReceiveAll()
 {
 	DSTACK(__FUNCTION_NAME);
@@ -1514,37 +1635,56 @@ void Client::ProcessData(u8 *data, u32 datasize, u16 sender_peer_id)
 			file_requests.push_back(MediaRequest(name));
 		}
 
+		std::string remote_media = "";
+		try {
+			remote_media = deSerializeString(is);
+		}
+		catch(SerializationError) {
+			// not supported by server or turned off
+		}
+
+		m_media_count = file_requests.size();
+		m_media_receive_started = true;
+
+		if (remote_media == "" || !USE_CURL) {
+			request_media(file_requests);
+		} else {
+			#if USE_CURL
+			for (size_t i = 0; i < g_settings->getU16("media_fetch_threads"); ++i) {
+				m_media_fetch_threads.push_back(MediaFetchThread(this));
+			}
+
+			core::list<MediaFetchThread>::Iterator cur = m_media_fetch_threads.begin();
+			for(core::list<MediaRequest>::Iterator i = file_requests.begin();
+					i != file_requests.end(); i++) {
+				cur->m_file_requests.push_back(*i);
+				cur++;
+				if (cur == m_media_fetch_threads.end())
+					cur = m_media_fetch_threads.begin();
+			}
+			for (core::list<MediaFetchThread>::Iterator i = m_media_fetch_threads.begin();
+					i != m_media_fetch_threads.end(); i++) {
+				i->m_remote_url = remote_media;
+				i->Start();
+			}
+			#endif
+
+			// notify server we received everything
+			std::ostringstream os(std::ios_base::binary);
+			writeU16(os, TOSERVER_RECEIVED_MEDIA);
+			std::string s = os.str();
+			SharedBuffer<u8> data((u8*)s.c_str(), s.size());
+			// Send as reliable
+			Send(0, data, true);
+		}
 		ClientEvent event;
 		event.type = CE_TEXTURES_UPDATED;
 		m_client_event_queue.push_back(event);
-
-		/*
-			u16 command
-			u16 number of files requested
-			for each file {
-				u16 length of name
-				string name
-			}
-		*/
-		std::ostringstream os(std::ios_base::binary);
-		writeU16(os, TOSERVER_REQUEST_MEDIA);
-		writeU16(os, file_requests.size());
-
-		for(core::list<MediaRequest>::Iterator i = file_requests.begin();
-				i != file_requests.end(); i++) {
-			os<<serializeString(i->name);
-		}
-
-		// Make data buffer
-		std::string s = os.str();
-		SharedBuffer<u8> data((u8*)s.c_str(), s.size());
-		// Send as reliable
-		Send(0, data, true);
-		infostream<<"Client: Sending media request list to server ("
-				<<file_requests.size()<<" files)"<<std::endl;
 	}
 	else if(command == TOCLIENT_MEDIA)
 	{
+		if (m_media_count == 0)
+			return;
 		std::string datastring((char*)&data[2], datasize-2);
 		std::istringstream is(datastring, std::ios_base::binary);
 
@@ -1566,17 +1706,12 @@ void Client::ProcessData(u8 *data, u32 datasize, u16 sender_peer_id)
 		*/
 		int num_bunches = readU16(is);
 		int bunch_i = readU16(is);
-		if(num_bunches >= 2)
-			m_media_receive_progress = (float)bunch_i / (float)(num_bunches - 1);
-		else
-			m_media_receive_progress = 1.0;
-		if(bunch_i == num_bunches - 1)
-			m_media_received = true;
 		int num_files = readU32(is);
 		infostream<<"Client: Received files: bunch "<<bunch_i<<"/"
 				<<num_bunches<<" files="<<num_files
 				<<" size="<<datasize<<std::endl;
 		for(int i=0; i<num_files; i++){
+			m_media_received_count++;
 			std::string name = deSerializeString(is);
 			std::string data = deSerializeLongString(is);
 
@@ -2458,7 +2593,7 @@ void Client::afterContentReceived()
 	infostream<<"Client::afterContentReceived() started"<<std::endl;
 	assert(m_itemdef_received);
 	assert(m_nodedef_received);
-	assert(m_media_received);
+	assert(texturesReceived());
 	
 	// remove the information about which checksum each texture
 	// ought to have

src/client.h

@@ -33,6 +33,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "filesys.h"
 #include "filecache.h"
 #include "localplayer.h"
+#include "server.h"
 #include "util/pointedthing.h"
 
 struct MeshMakeData;
@@ -129,6 +130,24 @@ public:
 	IGameDef *m_gamedef;
 };
 
+class MediaFetchThread : public SimpleThread
+{
+public:
+
+	MediaFetchThread(IGameDef *gamedef):
+		m_gamedef(gamedef)
+	{
+	}
+
+	void * Thread();
+
+	core::list<MediaRequest> m_file_requests;
+	MutexedQueue<std::pair<std::string, std::string> > m_file_data;
+	core::list<MediaRequest> m_failed;
+	std::string m_remote_url;
+	IGameDef *m_gamedef;
+};
+
 enum ClientEventType
 {
 	CE_NONE,
@@ -289,10 +308,13 @@ public:
 	{ return m_access_denied_reason; }
 
 	float mediaReceiveProgress()
-	{ return m_media_receive_progress; }
+	{
+		if (!m_media_receive_started) return 0;
+		return 1.0 * m_media_received_count / m_media_count;
+	}
 
 	bool texturesReceived()
-	{ return m_media_received; }
+	{ return m_media_receive_started && m_media_received_count == m_media_count; }
 	bool itemdefReceived()
 	{ return m_itemdef_received; }
 	bool nodedefReceived()
@@ -319,6 +341,8 @@ private:
 	// Insert a media file appropriately into the appropriate manager
 	bool loadMedia(const std::string &data, const std::string &filename);
 
+	void request_media(const core::list<MediaRequest> &file_requests);
+
 	// Virtual methods from con::PeerHandler
 	void peerAdded(con::Peer *peer);
 	void deletingPeer(con::Peer *peer, bool timeout);
@@ -347,6 +371,7 @@ private:
 	MtEventManager *m_event;
 
 	MeshUpdateThread m_mesh_update_thread;
+	core::list<MediaFetchThread> m_media_fetch_threads;
 	ClientEnvironment m_env;
 	con::Connection m_con;
 	IrrlichtDevice *m_device;
@@ -375,8 +400,9 @@ private:
 	FileCache m_media_cache;
 	// Mapping from media file name to SHA1 checksum
 	core::map<std::string, std::string> m_media_name_sha1_map;
-	float m_media_receive_progress;
-	bool m_media_received;
+	bool m_media_receive_started;
+	u32 m_media_count;
+	u32 m_media_received_count;
 	bool m_itemdef_received;
 	bool m_nodedef_received;
 	friend class FarMesh;

src/clientserver.h

@@ -267,6 +267,8 @@ enum ToClientCommand
 			u32 length of data
 			data
 		}
+		u16 length of remote media server url (if applicable)
+		string url
 	*/
 	
 	TOCLIENT_TOOLDEF = 0x39,
@@ -571,6 +573,10 @@ enum ToServerCommand
 		}
 	 */
 
+	TOSERVER_RECEIVED_MEDIA = 0x41,
+	/*
+		u16 command
+	*/
 };
 
 #endif

src/cmake_config.h.in

@@ -7,6 +7,7 @@
 #define CMAKE_VERSION_STRING "@VERSION_STRING@"
 #define CMAKE_RUN_IN_PLACE @RUN_IN_PLACE@
 #define CMAKE_USE_GETTEXT @USE_GETTEXT@
+#define CMAKE_USE_CURL @USE_CURL@
 #define CMAKE_USE_SOUND @USE_SOUND@
 #define CMAKE_STATIC_SHAREDIR "@SHAREDIR@"
 

src/config.h

@@ -11,6 +11,7 @@
 #define RUN_IN_PLACE 0
 #define USE_GETTEXT 0
 #define USE_SOUND 0
+#define USE_CURL 0
 #define STATIC_SHAREDIR ""
 #define BUILD_INFO "non-cmake"
 
@@ -26,6 +27,8 @@
 	#define USE_GETTEXT CMAKE_USE_GETTEXT
 	#undef USE_SOUND
 	#define USE_SOUND CMAKE_USE_SOUND
+	#undef USE_CURL
+	#define USE_CURL CMAKE_USE_CURL
 	#undef STATIC_SHAREDIR
 	#define STATIC_SHAREDIR CMAKE_STATIC_SHAREDIR
 	#undef BUILD_INFO

src/defaultsettings.cpp

@@ -119,6 +119,8 @@ void set_default_settings(Settings *settings)
 	settings->setDefault("preload_item_visuals", "true");
 	settings->setDefault("enable_shaders", "2");
 
+	settings->setDefault("media_fetch_threads", "8");
+
 	// Server stuff
 	// "map-dir" doesn't exist by default.
 	settings->setDefault("default_game", "minetest");
@@ -158,5 +160,6 @@ void set_default_settings(Settings *settings)
 	settings->setDefault("congestion_control_aim_rtt", "0.2");
 	settings->setDefault("congestion_control_max_rate", "400");
 	settings->setDefault("congestion_control_min_rate", "10");
+	settings->setDefault("remote_media", "");
 }
 

src/server.cpp

@@ -2885,6 +2885,9 @@ void Server::ProcessData(u8 *data, u32 datasize, u16 peer_id)
 		// (definitions and files)
 		getClient(peer_id)->definitions_sent = true;
 	}
+	else if(command == TOSERVER_RECEIVED_MEDIA) {
+		getClient(peer_id)->definitions_sent = true;
+	}
 	else if(command == TOSERVER_INTERACT)
 	{
 		std::string datastring((char*)&data[2], datasize-2);
@@ -4217,6 +4220,7 @@ void Server::sendMediaAnnouncement(u16 peer_id)
 		os<<serializeString(j->name);
 		os<<serializeString(j->sha1_digest);
 	}
+	os<<serializeString(g_settings->get("remote_media"));
 
 	// Make data buffer
 	std::string s = os.str();
@@ -4224,7 +4228,6 @@ void Server::sendMediaAnnouncement(u16 peer_id)
 
 	// Send as reliable
 	m_con.Send(peer_id, 0, data, true);
-
 }
 
 struct SendableMedia

src/util/string.cpp

@@ -41,3 +41,9 @@ std::string translatePassword(std::string playername, std::wstring password)
 	return pwd;
 }
 
+size_t curl_write_data(char *ptr, size_t size, size_t nmemb, void *userdata) {
+    std::ostringstream *stream = (std::ostringstream*)userdata;
+    size_t count = size * nmemb;
+    stream->write(ptr, count);
+    return count;
+}

src/util/string.h

@@ -282,6 +282,7 @@ inline std::string wrap_rows(const std::string &from, u32 rowlen)
 }
 
 std::string translatePassword(std::string playername, std::wstring password);
+size_t curl_write_data(char *ptr, size_t size, size_t nmemb, void *userdata);
 
 #endif