Fix modstore/favourites hang by adding asynchronous lua job support

builtin/async_env.lua

@@ -0,0 +1,19 @@
+engine.log("info","Initializing Asynchronous environment")
+
+dofile(SCRIPTDIR .. DIR_DELIM .. "misc_helpers.lua")
+
+function engine.job_processor(serialized_function, serialized_data)
+
+	local fct = marshal.decode(serialized_function)
+	local params = marshal.decode(serialized_data)
+	local retval = marshal.encode(nil)
+
+	if fct ~= nil and type(fct) == "function" then
+		local result = fct(params)
+		retval = marshal.encode(result)
+	else
+		engine.log("error","ASYNC WORKER: unable to deserialize function")
+	end
+
+	return retval,retval:len()
+end

builtin/async_event.lua

@@ -0,0 +1,59 @@
+local tbl = engine or minetest
+
+tbl.async_jobs = {}
+
+if engine ~= nil then
+	function tbl.async_event_handler(jobid, serialized_retval)
+		local retval = nil
+		if serialized_retval ~= "ERROR" then
+			retval= marshal.decode(serialized_retval)
+		else
+			tbl.log("error","Error fetching async result")
+		end
+
+		assert(type(tbl.async_jobs[jobid]) == "function")
+		tbl.async_jobs[jobid](retval)
+		tbl.async_jobs[jobid] = nil
+	end
+else
+
+	minetest.register_globalstep(
+		function(dtime)
+			local list = tbl.get_finished_jobs()
+
+			for i=1,#list,1 do
+				local retval = marshal.decode(list[i].retval)
+
+				assert(type(tbl.async_jobs[jobid]) == "function")
+				tbl.async_jobs[list[i].jobid](retval)
+				tbl.async_jobs[list[i].jobid] = nil
+			end
+		end)
+end
+
+function tbl.handle_async(fct, parameters, callback)
+
+	--serialize fct
+	local serialized_fct = marshal.encode(fct)
+
+	assert(marshal.decode(serialized_fct) ~= nil)
+
+	--serialize parameters
+	local serialized_params = marshal.encode(parameters)
+
+	if serialized_fct == nil or
+		serialized_params == nil or
+		serialized_fct:len() == 0 or
+		serialized_params:len() == 0 then
+		return false
+	end
+
+	local jobid = tbl.do_async_callback(	serialized_fct,
+											serialized_fct:len(),
+											serialized_params,
+											serialized_params:len())
+
+	tbl.async_jobs[jobid] = callback
+
+	return true
+end

builtin/mainmenu.lua

@@ -24,6 +24,7 @@ dofile(scriptpath .. DIR_DELIM .. "modstore.lua")
 dofile(scriptpath .. DIR_DELIM .. "gamemgr.lua")
 dofile(scriptpath .. DIR_DELIM .. "mm_textures.lua")
 dofile(scriptpath .. DIR_DELIM .. "mm_menubar.lua")
+dofile(scriptpath .. DIR_DELIM .. "async_event.lua")
 
 menu = {}
 local tabbuilder = {}
@@ -196,6 +197,21 @@ function menu.render_texture_pack_list(list)
 	return retval
 end
 
+--------------------------------------------------------------------------------
+function menu.asyncOnlineFavourites()
+	menu.favorites = {}
+	engine.handle_async(
+		function(param)
+			return engine.get_favorites("online")
+		end,
+		nil,
+		function(result)
+			menu.favorites = result
+			engine.event_handler("Refresh")
+		end
+		)
+end
+
 --------------------------------------------------------------------------------
 function menu.init()
 	--init menu data
@@ -208,7 +224,7 @@ function menu.init()
 	end
 
 	if engine.setting_getbool("public_serverlist") then
-		menu.favorites = engine.get_favorites("online")
+		menu.asyncOnlineFavourites()
 	else
 		menu.favorites = engine.get_favorites("local")
 	end
@@ -489,7 +505,7 @@ function tabbuilder.handle_multiplayer_buttons(fields)
 		engine.setting_set("public_serverlist", fields["cb_public_serverlist"])
 
 		if engine.setting_getbool("public_serverlist") then
-			menu.favorites = engine.get_favorites("online")
+			menu.asyncOnlineFavourites()
 		else
 			menu.favorites = engine.get_favorites("local")
 		end
@@ -583,6 +599,7 @@ function tabbuilder.handle_server_buttons(fields)
 			gamedata.selected_world	= filterlist.get_raw_index(worldlist,selected)
 
 			engine.setting_set("port",gamedata.port)
+
 			menu.update_last_game(gamedata.selected_world)
 			engine.start()
 		end
@@ -1124,6 +1141,12 @@ function tabbuilder.checkretval(retval)
 		if retval.skipformupdate ~= nil then
 			tabbuilder.skipformupdate = retval.skipformupdate
 		end
+
+		if retval.ignore_menu_quit == true then
+			tabbuilder.ignore_menu_quit = true
+		else
+			tabbuilder.ignore_menu_quit = false
+		end
 	end
 end
 
@@ -1194,6 +1217,10 @@ end
 engine.event_handler = function(event)
 	if event == "MenuQuit" then
 		if tabbuilder.is_dialog then
+			if tabbuilder.ignore_menu_quit then
+				return
+			end
+
 			tabbuilder.is_dialog = false
 			tabbuilder.show_buttons = true
 			tabbuilder.current_tab = engine.setting_get("main_menu_tab")
@@ -1203,6 +1230,10 @@ engine.event_handler = function(event)
 			engine.close()
 		end
 	end
+
+	if event == "Refresh" then
+		update_menu()
+	end
 end
 
 --------------------------------------------------------------------------------

builtin/modmgr.lua

@@ -54,7 +54,7 @@ function modmgr.extract(modfile)
 		local tempfolder = os.tempfolder()
 
 		if tempfolder ~= nil and
-			tempfodler ~= "" then
+			tempfolder ~= "" then
 			engine.create_dir(tempfolder)
 			engine.extract_zip(modfile.name,tempfolder)
 			return tempfolder

builtin/modstore.lua

@@ -32,10 +32,11 @@ function modstore.init()
 	modstore.basetexturedir = engine.get_texturepath() .. DIR_DELIM .. "base" ..
 						DIR_DELIM .. "pack" .. DIR_DELIM
 
-	modstore.current_list = nil
+	modstore.lastmodtitle = ""
 
-	modstore.details_cache = {}
+	modstore.current_list = nil
 end
+
 --------------------------------------------------------------------------------
 function modstore.nametoindex(name)
 
@@ -60,8 +61,7 @@ function modstore.gettab(tabname)
 	end
 
 	if tabname == "dialog_modstore_search" then
-	
+		retval = modstore.getsearchpage()
-	
 		is_modstore_tab = true
 	end
 
@@ -70,11 +70,16 @@ function modstore.gettab(tabname)
 	end
 
 	if tabname == "modstore_mod_installed" then
-		return "size[6,2]label[0.25,0.25;Mod: " .. modstore.lastmodtitle .. 
+		return "size[6,2]label[0.25,0.25;Mod(s): " .. modstore.lastmodtitle ..
 				" installed successfully]" ..
 				"button[2.5,1.5;1,0.5;btn_confirm_mod_successfull;ok]"
 	end
 
+	if tabname == "modstore_downloading" then
+		return "size[6,2]label[0.25,0.25;Dowloading " .. modstore.lastmodtitle ..
+				" please wait]"
+	end
+
 	return ""
 end
 
@@ -91,8 +96,6 @@ end
 --------------------------------------------------------------------------------
 function modstore.handle_buttons(current_tab,fields)
 
-	modstore.lastmodtitle = ""
-	
 	if fields["modstore_tab"] then
 		local index = tonumber(fields["modstore_tab"])
 
@@ -121,7 +124,16 @@ function modstore.handle_buttons(current_tab,fields)
 		end
 	end
 
+	if fields["btn_hidden_close_download"] then
+		return {
+					current_tab = "modstore_mod_installed",
+					is_dialog = true,
+					show_buttons = false
+			}
+	end
+
 	if fields["btn_confirm_mod_successfull"] then
+		modstore.lastmodtitle = ""
 		return {
 					current_tab = modstore.tabnames[1],
 					is_dialog = true,
@@ -136,27 +148,61 @@ function modstore.handle_buttons(current_tab,fields)
 			local modlistentry =
 				modstore.current_list.page * modstore.modsperpage + i
 
-			local moddetails = modstore.get_details(modstore.current_list.data[modlistentry].id)
+			if modstore.modlist_unsorted.data[modlistentry] ~= nil and
+				modstore.modlist_unsorted.data[modlistentry].details ~= nil then
 
+				local moddetails = modstore.modlist_unsorted.data[modlistentry].details
+
+				if modstore.lastmodtitle ~= "" then
+					modstore.lastmodtitle = modstore.lastmodtitle .. ", "
+				end
+
+				modstore.lastmodtitle = modstore.lastmodtitle .. moddetails.title
+
+				engine.handle_async(
+					function(param)
 						local fullurl = engine.setting_get("modstore_download_url") ..
-								moddetails.download_url
+										param.moddetails.download_url
-			local modfilename = os.tempfolder() .. ".zip"
-			
-			if engine.download_file(fullurl,modfilename) then
-			
-				modmgr.installmod(modfilename,moddetails.basename)
-				
-				os.remove(modfilename)
-				modstore.lastmodtitle = modstore.current_list.data[modlistentry].title
 
+						if engine.download_file(fullurl,param.filename) then
 							return {
-					current_tab = "modstore_mod_installed",
+								moddetails = param.moddetails,
-					is_dialog = true,
+								filename = param.filename,
-					show_buttons = false
+								successfull = true
 							}
 						else
-				gamedata.errormessage = "Unable to download " .. 
+							return {
-					moddetails.download_url .. " (internet connection?)"
+								modtitle = param.title,
+								successfull = false
+							}
+						end
+					end,
+					{
+						moddetails = moddetails,
+						filename = os.tempfolder() .. ".zip"
+					},
+					function(result)
+						if result.successfull then
+							modmgr.installmod(result.filename,result.moddetails.basename)
+							os.remove(result.filename)
+						else
+							gamedata.errormessage = "Failed to download " .. result.moddetails.title
+						end
+
+						engine.button_handler({btn_hidden_close_download=true})
+					end
+				)
+
+				return {
+					current_tab = "modstore_downloading",
+					is_dialog = true,
+					show_buttons = false,
+					ignore_menu_quit = true
+				}
+
+			else
+				gamedata.errormessage =
+					"Internal modstore error please leave modstore and reopen! (Sorry)"
 			end
 		end
 	end
@@ -165,7 +211,19 @@ end
 --------------------------------------------------------------------------------
 function modstore.update_modlist()
 	modstore.modlist_unsorted = {}
-	modstore.modlist_unsorted.data = engine.get_modstore_list()
+	modstore.modlist_unsorted.data = {}
+	modstore.modlist_unsorted.pagecount = 1
+	modstore.modlist_unsorted.page = 0
+
+	engine.handle_async(
+		function(param)
+			return engine.get_modstore_list()
+		end,
+		nil,
+		function(result)
+			if result ~= nil then
+				modstore.modlist_unsorted = {}
+				modstore.modlist_unsorted.data = result
 
 				if modstore.modlist_unsorted.data ~= nil then
 					modstore.modlist_unsorted.pagecount =
@@ -175,8 +233,42 @@ function modstore.update_modlist()
 					modstore.modlist_unsorted.pagecount = 1
 				end
 				modstore.modlist_unsorted.page = 0
+				modstore.fetchdetails()
+				engine.event_handler("Refresh")
+			end
+		end
+	)
 end
 
+--------------------------------------------------------------------------------
+function modstore.fetchdetails()
+
+	for i=1,#modstore.modlist_unsorted.data,1 do
+		engine.handle_async(
+		function(param)
+			param.details = engine.get_modstore_details(tostring(param.modid))
+			return param
+		end,
+		{
+			modid=modstore.modlist_unsorted.data[i].id,
+			listindex=i
+		},
+		function(result)
+			if result ~= nil and
+				modstore.modlist_unsorted ~= nil
+				and modstore.modlist_unsorted.data ~= nil and
+				modstore.modlist_unsorted.data[result.listindex] ~= nil and
+				modstore.modlist_unsorted.data[result.listindex].id ~= nil then
+
+				modstore.modlist_unsorted.data[result.listindex].details = result.details
+				engine.event_handler("Refresh")
+			end
+		end
+		)
+	end
+end
+--------------------------------------------------------------------------------
+
 --------------------------------------------------------------------------------
 function modstore.getmodlist(list)
 	local retval = ""
@@ -201,33 +293,72 @@ function modstore.getmodlist(list)
 
 	for i=(list.page * modstore.modsperpage) +1, endmod, 1 do
 		--getmoddetails
-		local details = modstore.get_details(list.data[i].id)
+		local details = list.data[i].details
+
+--		if details == nil then
+--			details = modstore.get_details(list.data[i].id)
+--		end
+
+		if details == nil then
+			details = {}
+			details.title = list.data[i].title
+			details.author = ""
+			details.rating = -1
+			details.description = ""
+		end
 
 		if details ~= nil then
 			local screenshot_ypos = (i-1 - (list.page * modstore.modsperpage))*1.9 +0.2
 
 			retval = retval .. "box[0," .. screenshot_ypos .. ";11.4,1.75;#FFFFFF]"
 
+			if details.basename then
 				--screenshot
 				if details.screenshot_url ~= nil and
 					details.screenshot_url ~= "" then
 					if list.data[i].texturename == nil then
 						local fullurl = engine.setting_get("modstore_download_url") ..
 									details.screenshot_url
-					local filename = os.tempfolder()
+						local filename = os.tempfolder() .. "_MID_" .. list.data[i].id
-					
+						list.data[i].texturename = "in progress"
-					if engine.download_file(fullurl,filename) then
+						engine.handle_async(
-						list.data[i].texturename = filename
+							function(param)
+								param.successfull = engine.download_file(param.fullurl,param.filename)
+								return param
+							end,
+							{
+								fullurl = fullurl,
+								filename = filename,
+								listindex = i,
+								modid = list.data[i].id
+							},
+							function(result)
+								if modstore.modlist_unsorted and
+									modstore.modlist_unsorted.data and
+									#modstore.modlist_unsorted.data >= result.listindex and
+									modstore.modlist_unsorted.data[result.listindex].id == result.modid then
+									if result.successfull then
+										modstore.modlist_unsorted.data[result.listindex].texturename = result.filename
+									else
+										modstore.modlist_unsorted.data[result.listindex].texturename = modstore.basetexturedir .. "no_screenshot.png"
+									end
+									engine.event_handler("Refresh")
 								end
 							end
+						)
 					end
-			
+				else
 					if list.data[i].texturename == nil then
 						list.data[i].texturename = modstore.basetexturedir .. "no_screenshot.png"
 					end
+				end
 
+				if list.data[i].texturename ~= nil and
+					list.data[i].texturename ~= "in progress" then
 					retval = retval .. "image[0,".. screenshot_ypos .. ";3,2;" ..
 						engine.formspec_escape(list.data[i].texturename) .. "]"
+				end
+			end
 
 			--title + author
 			retval = retval .."label[2.75," .. screenshot_ypos .. ";" ..
@@ -239,13 +370,15 @@ function modstore.getmodlist(list)
 				engine.formspec_escape(details.description) .. ";]"
 			--rating
 			local ratingy = screenshot_ypos + 0.6
-			retval = retval .."label[10.1," .. ratingy .. ";" .. 
+			retval = retval .."label[9.1," .. ratingy .. ";" ..
-							fgettext("Rating") .. ": " .. details.rating .."]"
+							fgettext("Rating") .. ":]"
+			retval = retval .. "label[11.1," .. ratingy .. ";" .. details.rating .."]"
 
+			if details.basename then
 				--install button
 				local buttony = screenshot_ypos + 1.2
 				local buttonnumber = (i - (list.page * modstore.modsperpage))
-			retval = retval .."button[9.6," .. buttony .. ";2,0.5;btn_install_mod_" .. buttonnumber .. ";"
+				retval = retval .."button[9.1," .. buttony .. ";2.5,0.5;btn_install_mod_" .. buttonnumber .. ";"
 
 				if modmgr.mod_exists(details.basename) then
 					retval = retval .. fgettext("re-Install") .."]"
@@ -254,6 +387,7 @@ function modstore.getmodlist(list)
 				end
 			end
 		end
+	end
 
 	modstore.current_list = list
 
@@ -261,14 +395,11 @@ function modstore.getmodlist(list)
 end
 
 --------------------------------------------------------------------------------
-function modstore.get_details(modid) 
+function modstore.getsearchpage()
+	local retval = ""
 
-	if modstore.details_cache[modid] ~= nil then
+	--TODO implement search!
-		return modstore.details_cache[modid]
-	end
 
-	local retval = engine.get_modstore_details(tostring(modid))
+	return retval;
-	modstore.details_cache[modid] = retval
-	return retval
 end
 

doc/menu_lua_api.txt

@@ -33,9 +33,9 @@ engine.close()
 Filesystem:
 engine.get_scriptdir()
 ^ returns directory of script
-engine.get_modpath()
+engine.get_modpath() (possible in async calls)
 ^ returns path to global modpath
-engine.get_modstore_details(modid)
+engine.get_modstore_details(modid) (possible in async calls)
 ^ modid numeric id of mod in modstore
 ^ returns {
 	id			= <numeric id of mod in modstore>,
@@ -47,7 +47,7 @@ engine.get_modstore_details(modid)
 	license		= <short description of license>,
 	rating		= <float value of current rating>
 }
-engine.get_modstore_list()
+engine.get_modstore_list() (possible in async calls)
 ^ returns {
 	[1] = {
 		id		 = <numeric id of mod in modstore>,
@@ -55,19 +55,21 @@ engine.get_modstore_list()
 		basename = <basename for mod>
 	}
 }
-engine.get_gamepath()
+engine.get_gamepath() (possible in async calls)
 ^ returns path to global gamepath
-engine.get_dirlist(path,onlydirs)
+engine.get_texturepath() (possible in async calls)
+^ returns path to default textures
+engine.get_dirlist(path,onlydirs) (possible in async calls)
 ^ path to get subdirs from
 ^ onlydirs should result contain only dirs?
 ^ returns list of folders within path
-engine.create_dir(absolute_path)
+engine.create_dir(absolute_path) (possible in async calls)
 ^ absolute_path to directory to create (needs to be absolute)
 ^ returns true/false
-engine.delete_dir(absolute_path)
+engine.delete_dir(absolute_path) (possible in async calls)
 ^ absolute_path to directory to delete (needs to be absolute)
 ^ returns true/false
-engine.copy_dir(source,destination,keep_soure)
+engine.copy_dir(source,destination,keep_soure) (possible in async calls)
 ^ source folder
 ^ destination folder
 ^ keep_source DEFAULT true --> if set to false source is deleted after copying
@@ -76,11 +78,11 @@ engine.extract_zip(zipfile,destination) [unzip within path required]
 ^ zipfile to extract
 ^ destination folder to extract to
 ^ returns true/false
-engine.download_file(url,target)
+engine.download_file(url,target) (possible in async calls)
 ^ url to download
 ^ target to store to
 ^ returns true/false
-engine.get_version()
+engine.get_version() (possible in async calls)
 ^ returns current minetest version
 engine.sound_play(spec, looped) -> handle
 ^ spec = SimpleSoundSpec (see lua-api.txt)
@@ -105,10 +107,10 @@ engine.get_game(index)
 	DEPRECATED:
 	addon_mods_paths = {[1] = <path>,},
 }
-engine.get_games() -> table of all games in upper format
+engine.get_games() -> table of all games in upper format (possible in async calls)
 
 Favorites:
-engine.get_favorites(location) -> list of favorites
+engine.get_favorites(location) -> list of favorites (possible in async calls)
 ^ location: "local" or "online"
 ^ returns {
 	[1] = {
@@ -128,21 +130,21 @@ engine.get_favorites(location) -> list of favorites
 engine.delete_favorite(id, location) -> success
 
 Logging:
-engine.debug(line)
+engine.debug(line) (possible in async calls)
 ^ Always printed to stderr and logfile (print() is redirected here)
-engine.log(line)
+engine.log(line) (possible in async calls)
-engine.log(loglevel, line)
+engine.log(loglevel, line) (possible in async calls)
 ^ loglevel one of "error", "action", "info", "verbose"
 
 Settings:
 engine.setting_set(name, value)
-engine.setting_get(name) -> string or nil
+engine.setting_get(name) -> string or nil (possible in async calls)
 engine.setting_setbool(name, value)
-engine.setting_getbool(name) -> bool or nil
+engine.setting_getbool(name) -> bool or nil (possible in async calls)
 engine.setting_save() -> nil, save all settings to config file
 
 Worlds:
-engine.get_worlds() -> list of worlds
+engine.get_worlds() -> list of worlds (possible in async calls)
 ^ returns {
 	[1] = {
 	path   = <full path to world>,
@@ -174,7 +176,7 @@ engine.gettext(string) -> string
 fgettext(string, ...) -> string
 ^ call engine.gettext(string), replace "$1"..."$9" with the given
 ^ extra arguments, call engine.formspec_escape and return the result
-engine.parse_json(string[, nullvalue]) -> something
+engine.parse_json(string[, nullvalue]) -> something (possible in async calls)
 ^ see minetest.parse_json (lua_api.txt)
 dump(obj, dumped={})
 ^ Return object serialized as a string
@@ -182,9 +184,24 @@ string:split(separator)
 ^ eg. string:split("a,b", ",") == {"a","b"}
 string:trim()
 ^ eg. string.trim("\n \t\tfoo bar\t ") == "foo bar"
-minetest.is_yes(arg)
+minetest.is_yes(arg) (possible in async calls)
 ^ returns whether arg can be interpreted as yes
 
+Async:
+engine.handle_async(async_job,parameters,finished)
+^ execute a function asynchronously
+^ async_job is a function receiving one parameter and returning one parameter
+^ parameters parameter table passed to async_job
+^ finished function to be called once async_job has finished 
+^    the result of async_job is passed to this function
+
+Limitations of Async operations
+ -No access to global lua variables, don't even try
+ -Limited set of available functions
+	e.g. No access to functions modifying menu like engine.start,engine.close,
+	engine.file_open_dialog
+			
+
 Class reference
 ----------------
 Settings: see lua_api.txt

src/guiEngine.cpp

@@ -286,6 +286,8 @@ void GUIEngine::run()
 			cloudPostProcess();
 		else
 			sleep_ms(25);
+
+		m_script->Step();
 	}
 }
 
@@ -576,3 +578,9 @@ void GUIEngine::stopSound(s32 handle)
 {
 	m_sound_manager->stopSound(handle);
 }
+
+/******************************************************************************/
+unsigned int GUIEngine::DoAsync(std::string serialized_fct,
+		std::string serialized_params) {
+	return m_script->DoAsync(serialized_fct,serialized_params);
+}

src/guiEngine.h

@@ -166,6 +166,9 @@ public:
 		return m_scriptdir;
 	}
 
+	/** pass async callback to scriptengine **/
+	unsigned int DoAsync(std::string serialized_fct,std::string serialized_params);
+
 private:
 
 	/** find and run the main menu script */
@@ -244,7 +247,7 @@ private:
 	 * @param url url to download
 	 * @param target file to store to
 	 */
-	bool downloadFile(std::string url,std::string target);
+	static bool downloadFile(std::string url,std::string target);
 
 	/** array containing pointers to current specified texture layers */
 	video::ITexture*		m_textures[TEX_LAYER_MAX];

src/jthread/CMakeLists.txt

@@ -2,10 +2,12 @@ if( UNIX )
 	set(JTHREAD_SRCS
 		${CMAKE_CURRENT_SOURCE_DIR}/pthread/jmutex.cpp
 		${CMAKE_CURRENT_SOURCE_DIR}/pthread/jthread.cpp
+		${CMAKE_CURRENT_SOURCE_DIR}/pthread/jsemaphore.cpp
 		PARENT_SCOPE)
 else( UNIX )
 	set(JTHREAD_SRCS
 		${CMAKE_CURRENT_SOURCE_DIR}/win32/jmutex.cpp
 		${CMAKE_CURRENT_SOURCE_DIR}/win32/jthread.cpp
+		${CMAKE_CURRENT_SOURCE_DIR}/win32/jsemaphore.cpp
 		PARENT_SCOPE)
 endif( UNIX )

src/jthread/jsemaphore.h

@@ -0,0 +1,50 @@
+/*
+Minetest
+Copyright (C) 2013 sapier, < sapier AT gmx DOT net >
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU Lesser General Public License as published by
+the Free Software Foundation; either version 2.1 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+*/
+
+#ifndef JSEMAPHORE_H_
+#define JSEMAPHORE_H_
+
+#if defined(WIN32)
+#include <windows.h>
+#define MAX_SEMAPHORE_COUNT 1024
+#else
+#include <pthread.h>
+#include <semaphore.h>
+#endif
+
+class JSemaphore {
+public:
+	JSemaphore();
+	~JSemaphore();
+	JSemaphore(int initval);
+
+	void Post();
+	void Wait();
+
+	int GetValue();
+
+private:
+#if defined(WIN32)
+	HANDLE m_hSemaphore;
+#else
+	sem_t m_semaphore;
+#endif
+};
+
+#endif /* JSEMAPHORE_H_ */

src/jthread/jthread.h

@@ -43,6 +43,7 @@ public:
 	JThread();
 	virtual ~JThread();
 	int Start();
+	void Stop();
 	int Kill();
 	virtual void *Thread() = 0;
 	bool IsRunning();

src/jthread/pthread/jsemaphore.cpp

@@ -0,0 +1,48 @@
+/*
+Minetest
+Copyright (C) 2013 sapier, < sapier AT gmx DOT net >
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU Lesser General Public License as published by
+the Free Software Foundation; either version 2.1 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+*/
+#include "jthread/jsemaphore.h"
+
+JSemaphore::JSemaphore() {
+	sem_init(&m_semaphore,0,0);
+}
+
+JSemaphore::~JSemaphore() {
+	sem_destroy(&m_semaphore);
+}
+
+JSemaphore::JSemaphore(int initval) {
+	sem_init(&m_semaphore,0,initval);
+}
+
+void JSemaphore::Post() {
+	sem_post(&m_semaphore);
+}
+
+void JSemaphore::Wait() {
+	sem_wait(&m_semaphore);
+}
+
+int JSemaphore::GetValue() {
+
+	int retval = 0;
+	sem_getvalue(&m_semaphore,&retval);
+
+	return retval;
+}
+

src/jthread/pthread/jthread.cpp

@@ -42,6 +42,12 @@ JThread::~JThread()
 	Kill();
 }
 
+void JThread::Stop() {
+	runningmutex.Lock();
+	running = false;
+	runningmutex.Unlock();
+}
+
 int JThread::Start()
 {
 	int status;

src/jthread/win32/jsemaphore.cpp

@@ -0,0 +1,64 @@
+/*
+Minetest
+Copyright (C) 2013 sapier, < sapier AT gmx DOT net >
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU Lesser General Public License as published by
+the Free Software Foundation; either version 2.1 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+*/
+#include "jthread/jsemaphore.h"
+
+JSemaphore::JSemaphore() {
+	m_hSemaphore = CreateSemaphore(
+			0,
+			0,
+			MAX_SEMAPHORE_COUNT,
+			0);
+}
+
+JSemaphore::~JSemaphore() {
+	CloseHandle(&m_hSemaphore);
+}
+
+JSemaphore::JSemaphore(int initval) {
+	m_hSemaphore = CreateSemaphore(
+			0,
+			initval,
+			MAX_SEMAPHORE_COUNT,
+			0);
+}
+
+void JSemaphore::Post() {
+	ReleaseSemaphore(
+			m_hSemaphore,
+			1,
+			0);
+}
+
+void JSemaphore::Wait() {
+	WaitForSingleObject(
+			m_hSemaphore,
+			INFINITE);
+}
+
+int JSemaphore::GetValue() {
+
+	long int retval = 0;
+	ReleaseSemaphore(
+			m_hSemaphore,
+			0,
+			&retval);
+
+	return retval;
+}
+

src/jthread/win32/jthread.cpp

@@ -43,6 +43,12 @@ JThread::~JThread()
 	Kill();
 }
 
+void JThread::Stop() {
+	runningmutex.Lock();
+	running = false;
+	runningmutex.Unlock();
+}
+
 int JThread::Start()
 {
 	if (!mutexinit)

src/main.cpp

@@ -85,6 +85,10 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "database-leveldb.h"
 #endif
 
+#if USE_CURL
+#include "curl.h"
+#endif
+
 /*
 	Settings.
 	These are loaded from the config file.
@@ -993,6 +997,11 @@ int main(int argc, char *argv[])
 	srand(time(0));
 	mysrand(time(0));
 
+#if USE_CURL
+	CURLcode res = curl_global_init(CURL_GLOBAL_DEFAULT);
+	assert(res == CURLE_OK);
+#endif
+
 	/*
 		Run unit tests
 	*/

src/script/lua_api/CMakeLists.txt

@@ -16,6 +16,8 @@ set(common_SCRIPT_LUA_API_SRCS
 	${CMAKE_CURRENT_SOURCE_DIR}/l_util.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/l_vmanip.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/l_settings.cpp
+	${CMAKE_CURRENT_SOURCE_DIR}/l_async_events.cpp
+	${CMAKE_CURRENT_SOURCE_DIR}/marshall.c
 	PARENT_SCOPE)
 
 # Used by client only

src/script/lua_api/l_async_events.cpp

@@ -0,0 +1,396 @@
+/*
+Minetest
+Copyright (C) 2013 sapier, <sapier AT gmx DOT net>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU Lesser General Public License as published by
+the Free Software Foundation; either version 2.1 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+*/
+
+extern "C" {
+#include "lua.h"
+#include "lauxlib.h"
+#include "lualib.h"
+int luaopen_marshal(lua_State *L);
+}
+#include <stdio.h>
+
+#include "l_async_events.h"
+#include "log.h"
+#include "filesys.h"
+#include "porting.h"
+
+//TODO replace by ShadowNinja version not yet merged to master
+static int script_error_handler(lua_State *L) {
+  lua_getfield(L, LUA_GLOBALSINDEX, "debug");
+  if (!lua_istable(L, -1)) {
+    lua_pop(L, 1);
+    return 1;
+  }
+  lua_getfield(L, -1, "traceback");
+  if (!lua_isfunction(L, -1)) {
+    lua_pop(L, 2);
+    return 1;
+  }
+  lua_pushvalue(L, 1);
+  lua_pushinteger(L, 2);
+  lua_call(L, 2, 1);
+  return 1;
+}
+
+/******************************************************************************/
+static void scriptError(const char *fmt, ...)
+{
+	va_list argp;
+	va_start(argp, fmt);
+	char buf[10000];
+	vsnprintf(buf, 10000, fmt, argp);
+	va_end(argp);
+	errorstream<<"ERROR: "<<buf;
+	fprintf(stderr,"ERROR: %s\n",buf);
+}
+
+/******************************************************************************/
+AsyncEngine::AsyncEngine() :
+	m_initDone(false),
+	m_JobIdCounter(0)
+{
+	assert(m_JobQueueMutex.Init() == 0);
+	assert(m_ResultQueueMutex.Init() == 0);
+}
+
+/******************************************************************************/
+AsyncEngine::~AsyncEngine()
+{
+	/** request all threads to stop **/
+	for (std::vector<AsyncWorkerThread*>::iterator i= m_WorkerThreads.begin();
+			i != m_WorkerThreads.end();i++) {
+		(*i)->Stop();
+	}
+
+
+	/** wakeup all threads **/
+	for (std::vector<AsyncWorkerThread*>::iterator i= m_WorkerThreads.begin();
+				i != m_WorkerThreads.end();i++) {
+		m_JobQueueCounter.Post();
+	}
+
+	/** wait for threads to finish **/
+	for (std::vector<AsyncWorkerThread*>::iterator i= m_WorkerThreads.begin();
+			i != m_WorkerThreads.end();i++) {
+		(*i)->Wait();
+	}
+
+	/** force kill all threads **/
+	for (std::vector<AsyncWorkerThread*>::iterator i= m_WorkerThreads.begin();
+			i != m_WorkerThreads.end();i++) {
+		(*i)->Kill();
+		delete *i;
+	}
+
+	m_JobQueueMutex.Lock();
+	m_JobQueue.clear();
+	m_JobQueueMutex.Unlock();
+	m_WorkerThreads.clear();
+}
+
+/******************************************************************************/
+bool AsyncEngine::registerFunction(const char* name, lua_CFunction fct) {
+
+	if (m_initDone) return false;
+	m_FunctionList[name] = fct;
+	return true;
+}
+
+/******************************************************************************/
+void AsyncEngine::Initialize(unsigned int numengines) {
+	m_initDone = true;
+
+	for (unsigned int i=0; i < numengines; i ++) {
+
+		AsyncWorkerThread* toadd = new AsyncWorkerThread(this,i);
+		m_WorkerThreads.push_back(toadd);
+		toadd->Start();
+	}
+}
+
+/******************************************************************************/
+unsigned int AsyncEngine::doAsyncJob(std::string fct, std::string params) {
+
+	m_JobQueueMutex.Lock();
+	LuaJobInfo toadd;
+	toadd.JobId = m_JobIdCounter++;
+	toadd.serializedFunction = fct;
+	toadd.serializedParams = params;
+
+	m_JobQueue.push_back(toadd);
+
+	m_JobQueueCounter.Post();
+
+	m_JobQueueMutex.Unlock();
+
+	return toadd.JobId;
+}
+
+/******************************************************************************/
+LuaJobInfo AsyncEngine::getJob() {
+
+	m_JobQueueCounter.Wait();
+	m_JobQueueMutex.Lock();
+
+	LuaJobInfo retval;
+
+	if (m_JobQueue.size() != 0) {
+		retval = m_JobQueue.front();
+		m_JobQueue.erase((m_JobQueue.begin()));
+	}
+	m_JobQueueMutex.Unlock();
+
+	return retval;
+}
+
+/******************************************************************************/
+void AsyncEngine::putJobResult(LuaJobInfo result) {
+	m_ResultQueueMutex.Lock();
+	m_ResultQueue.push_back(result);
+	m_ResultQueueMutex.Unlock();
+}
+
+/******************************************************************************/
+void AsyncEngine::Step(lua_State *L) {
+	m_ResultQueueMutex.Lock();
+	while(!m_ResultQueue.empty()) {
+
+		LuaJobInfo jobdone = m_ResultQueue.front();
+		m_ResultQueue.erase(m_ResultQueue.begin());
+		lua_getglobal(L, "engine");
+
+		lua_getfield(L, -1, "async_event_handler");
+
+		if(lua_isnil(L, -1))
+			assert("Someone managed to destroy a async callback in engine!" == 0);
+
+		luaL_checktype(L, -1, LUA_TFUNCTION);
+
+		lua_pushinteger(L, jobdone.JobId);
+		lua_pushlstring(L, jobdone.serializedResult.c_str(),
+							jobdone.serializedResult.length());
+
+		if(lua_pcall(L, 2, 0, 0)) {
+			scriptError("Async ENGINE step: %s", lua_tostring(L, -1));
+		}
+
+		lua_pop(L,1);
+	}
+	m_ResultQueueMutex.Unlock();
+}
+
+/******************************************************************************/
+void AsyncEngine::PushFinishedJobs(lua_State* L) {
+	//Result Table
+	m_ResultQueueMutex.Lock();
+
+	unsigned int index=1;
+	lua_newtable(L);
+	int top = lua_gettop(L);
+
+	while(!m_ResultQueue.empty()) {
+
+		LuaJobInfo jobdone = m_ResultQueue.front();
+		m_ResultQueue.erase(m_ResultQueue.begin());
+
+		lua_pushnumber(L,index);
+
+		lua_newtable(L);
+		int top_lvl2 = lua_gettop(L);
+
+		lua_pushstring(L,"jobid");
+		lua_pushnumber(L,jobdone.JobId);
+		lua_settable(L, top_lvl2);
+
+		lua_pushstring(L,"retval");
+		lua_pushstring(L, jobdone.serializedResult.c_str());
+		lua_settable(L, top_lvl2);
+
+		lua_settable(L, top);
+		index++;
+	}
+
+	m_ResultQueueMutex.Unlock();
+
+}
+/******************************************************************************/
+void AsyncEngine::PrepareEnvironment(lua_State* L, int top) {
+	for(std::map<std::string,lua_CFunction>::iterator i = m_FunctionList.begin();
+			i != m_FunctionList.end(); i++) {
+
+		lua_pushstring(L,i->first.c_str());
+		lua_pushcfunction(L,i->second);
+		lua_settable(L, top);
+
+	}
+}
+
+/******************************************************************************/
+int async_worker_ErrorHandler(lua_State *L) {
+	lua_getfield(L, LUA_GLOBALSINDEX, "debug");
+	if (!lua_istable(L, -1)) {
+	lua_pop(L, 1);
+	return 1;
+	}
+	lua_getfield(L, -1, "traceback");
+	if (!lua_isfunction(L, -1)) {
+	lua_pop(L, 2);
+	return 1;
+	}
+	lua_pushvalue(L, 1);
+	lua_pushinteger(L, 2);
+	lua_call(L, 2, 1);
+	return 1;
+}
+
+/******************************************************************************/
+AsyncWorkerThread::AsyncWorkerThread(AsyncEngine* jobdispatcher,
+									unsigned int numthreadnumber) :
+	m_JobDispatcher(jobdispatcher),
+	m_luaerrorhandler(-1),
+	m_threadnum(numthreadnumber)
+{
+	// create luastack
+	m_LuaStack = luaL_newstate();
+
+	// load basic lua modules
+	luaL_openlibs(m_LuaStack);
+
+	// load serialization functions
+	luaopen_marshal(m_LuaStack);
+}
+
+/******************************************************************************/
+AsyncWorkerThread::~AsyncWorkerThread() {
+
+	assert(IsRunning() == false);
+	lua_close(m_LuaStack);
+}
+
+/******************************************************************************/
+void* AsyncWorkerThread::worker_thread_main() {
+
+	//register thread for error logging
+	char number[21];
+	snprintf(number,sizeof(number),"%d",m_threadnum);
+	log_register_thread(std::string("AsyncWorkerThread_") + number);
+
+	/** prepare job lua environment **/
+	lua_newtable(m_LuaStack);
+	lua_setglobal(m_LuaStack, "engine");
+
+	lua_getglobal(m_LuaStack, "engine");
+	int top = lua_gettop(m_LuaStack);
+
+	lua_pushstring(m_LuaStack, DIR_DELIM);
+	lua_setglobal(m_LuaStack, "DIR_DELIM");
+
+	lua_pushstring(m_LuaStack,
+			std::string(porting::path_share + DIR_DELIM + "builtin").c_str());
+	lua_setglobal(m_LuaStack, "SCRIPTDIR");
+
+
+	m_JobDispatcher->PrepareEnvironment(m_LuaStack,top);
+
+	std::string asyncscript =
+	porting::path_share + DIR_DELIM + "builtin"
+			+ DIR_DELIM + "async_env.lua";
+
+	lua_pushcfunction(m_LuaStack, async_worker_ErrorHandler);
+	m_luaerrorhandler = lua_gettop(m_LuaStack);
+
+	if(!runScript(asyncscript)) {
+		infostream
+			<< "AsyncWorkderThread::worker_thread_main execution of async base environment failed!"
+			<< std::endl;
+			assert("no future with broken builtin async environment scripts" == 0);
+	}
+	/** main loop **/
+	while(IsRunning()) {
+		//wait for job
+		LuaJobInfo toprocess = m_JobDispatcher->getJob();
+
+		if (!IsRunning()) { continue; }
+
+		//first push error handler
+		lua_pushcfunction(m_LuaStack, script_error_handler);
+		int errorhandler = lua_gettop(m_LuaStack);
+
+		lua_getglobal(m_LuaStack, "engine");
+		if(lua_isnil(m_LuaStack, -1))
+			assert("unable to find engine within async environment" == 0);
+
+		lua_getfield(m_LuaStack, -1, "job_processor");
+		if(lua_isnil(m_LuaStack, -1))
+			assert("Someone managed to destroy a async worker engine!" == 0);
+
+		luaL_checktype(m_LuaStack, -1, LUA_TFUNCTION);
+
+		//call it
+		lua_pushlstring(m_LuaStack,
+						toprocess.serializedFunction.c_str(),
+						toprocess.serializedFunction.length());
+		lua_pushlstring(m_LuaStack,
+						toprocess.serializedParams.c_str(),
+						toprocess.serializedParams.length());
+
+		if (!IsRunning()) { continue; }
+		if(lua_pcall(m_LuaStack, 2, 2, errorhandler)) {
+			scriptError("Async WORKER thread: %s\n", lua_tostring(m_LuaStack, -1));
+			toprocess.serializedResult="ERROR";
+		}
+		else {
+			//fetch result
+			const char *retval = lua_tostring(m_LuaStack, -2);
+			unsigned int lenght = lua_tointeger(m_LuaStack,-1);
+			toprocess.serializedResult = std::string(retval,lenght);
+		}
+
+		if (!IsRunning()) { continue; }
+		//put job result
+		m_JobDispatcher->putJobResult(toprocess);
+	}
+	log_deregister_thread();
+	return 0;
+}
+
+/******************************************************************************/
+bool AsyncWorkerThread::runScript(std::string script) {
+
+	int ret = 	luaL_loadfile(m_LuaStack, script.c_str()) ||
+				lua_pcall(m_LuaStack, 0, 0, m_luaerrorhandler);
+	if(ret){
+		errorstream<<"==== ERROR FROM LUA WHILE INITIALIZING ASYNC ENVIRONMENT ====="<<std::endl;
+		errorstream<<"Failed to load and run script from "<<std::endl;
+		errorstream<<script<<":"<<std::endl;
+		errorstream<<std::endl;
+		errorstream<<lua_tostring(m_LuaStack, -1)<<std::endl;
+		errorstream<<std::endl;
+		errorstream<<"=================== END OF ERROR FROM LUA ===================="<<std::endl;
+		lua_pop(m_LuaStack, 1); // Pop error message from stack
+		lua_pop(m_LuaStack, 1); // Pop the error handler from stack
+		return false;
+	}
+	return true;
+}
+
+/******************************************************************************/
+void* AsyncWorkerThread::worker_thread_wrapper(void* thread) {
+	return ((AsyncWorkerThread*) thread)->worker_thread_main();
+}

src/script/lua_api/l_async_events.h

@@ -0,0 +1,228 @@
+/*
+Minetest
+Copyright (C) 2013 sapier, <sapier AT gmx DOT net>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU Lesser General Public License as published by
+the Free Software Foundation; either version 2.1 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+*/
+
+#ifndef C_ASYNC_EVENTS_H_
+#define C_ASYNC_EVENTS_H_
+
+#include <vector>
+#include <map>
+#include <unistd.h>
+
+/******************************************************************************/
+/* Includes                                                                   */
+/******************************************************************************/
+#include "jthread/jthread.h"
+#include "jthread/jmutex.h"
+#include "jthread/jsemaphore.h"
+#include "debug.h"
+#include "lua.h"
+
+/******************************************************************************/
+/* Typedefs and macros                                                        */
+/******************************************************************************/
+#define MAINMENU_NUMBER_OF_ASYNC_THREADS                   4
+
+/******************************************************************************/
+/* forward declarations                                                       */
+/******************************************************************************/
+class AsyncEngine;
+
+/******************************************************************************/
+/* declarations                                                               */
+/******************************************************************************/
+
+/** a struct to encapsulate data required to queue a job **/
+struct LuaJobInfo {
+	/** function to be called in async environment **/
+	std::string serializedFunction;
+	/** parameter table to be passed to function **/
+	std::string serializedParams;
+	/** result of function call **/
+	std::string serializedResult;
+	/** jobid used to identify a job and match it to callback **/
+	unsigned int JobId;
+};
+
+/** class encapsulating a asynchronous working environment **/
+class AsyncWorkerThread : public JThread {
+public:
+	/**
+	 * default constructor
+	 * @param pointer to job dispatcher
+	 */
+	AsyncWorkerThread(AsyncEngine* jobdispatcher, unsigned int threadnumber);
+
+	/**
+	 * default destructor
+	 */
+	virtual ~AsyncWorkerThread();
+
+	/**
+	 * thread function
+	 */
+	void* Thread() {
+			ThreadStarted();
+			return worker_thread_wrapper(this);
+	}
+
+	/**
+	 * wait for thread to stop
+	 */
+	void Wait() {
+		while(IsRunning()) {
+			sleep(1);
+		}
+	}
+
+private:
+	/**
+	 * helper function to run a lua script
+	 * @param path of script
+	 */
+	bool runScript(std::string script);
+
+	/**
+	 * main function of thread
+	 */
+	void* worker_thread_main();
+
+	/**
+	 * static wrapper for thread creation
+	 * @param this pointer to the thread to be created
+	 */
+	static void* worker_thread_wrapper(void* thread);
+
+	/**
+	 * pointer to job dispatcher
+	 */
+	AsyncEngine* m_JobDispatcher;
+
+	/**
+	 * the lua stack to run at
+	 */
+	lua_State*   m_LuaStack;
+
+	/**
+	 * lua internal stack number of error handler
+	 */
+	int m_luaerrorhandler;
+
+	/**
+	 * thread number used for debug output
+	 */
+	unsigned int m_threadnum;
+
+};
+
+/** asynchornous thread and job management **/
+class AsyncEngine {
+	friend AsyncWorkerThread;
+public:
+	/**
+	 * default constructor
+	 */
+	AsyncEngine();
+	/**
+	 * default destructor
+	 */
+	~AsyncEngine();
+
+	/**
+	 * register function to be used within engines
+	 * @param name function name to be used within lua environment
+	 * @param fct c-function to be called
+	 */
+	bool registerFunction(const char* name, lua_CFunction fct);
+
+	/**
+	 * create async engine tasks and lock function registration
+	 * @param numengines number of async threads to be started
+	 */
+	void Initialize(unsigned int numengines);
+
+	/**
+	 * queue/run a async job
+	 * @param fct serialized lua function
+	 * @param params serialized parameters
+	 * @return jobid the job is queued
+	 */
+	unsigned int doAsyncJob(std::string fct, std::string params);
+
+	/**
+	 * engine step to process finished jobs
+	 *   the engine step is one way to pass events back, PushFinishedJobs another
+	 * @param L the lua environment to do the step in
+	 */
+	void Step(lua_State *L);
+
+
+	void PushFinishedJobs(lua_State* L);
+
+protected:
+	/**
+	 * Get a Job from queue to be processed
+	 *  this function blocks until a job is ready
+	 * @return a job to be processed
+	 */
+	LuaJobInfo getJob();
+
+	/**
+	 * put a Job result back to result queue
+	 * @param result result of completed job
+	 */
+	void putJobResult(LuaJobInfo result);
+
+	/**
+	 * initialize environment with current registred functions
+	 *  this function adds all functions registred by registerFunction to the
+	 *  passed lua stack
+	 * @param L lua stack to initialize
+	 * @param top stack position
+	 */
+	void PrepareEnvironment(lua_State* L, int top);
+
+private:
+
+	/** variable locking the engine against further modification **/
+	bool m_initDone;
+
+	/** internal store for registred functions **/
+	std::map<std::string,lua_CFunction> m_FunctionList;
+
+	/** internal counter to create job id's **/
+	unsigned int m_JobIdCounter;
+
+	/** mutex to protect job queue **/
+	JMutex m_JobQueueMutex;
+	/** job queue **/
+	std::vector<LuaJobInfo> m_JobQueue;
+
+	/** mutext to protect result queue **/
+	JMutex m_ResultQueueMutex;
+	/** result queue **/
+	std::vector<LuaJobInfo> m_ResultQueue;
+
+	/** list of current worker threads **/
+	std::vector<AsyncWorkerThread*> m_WorkerThreads;
+
+	/** counter semaphore for job dispatching **/
+	JSemaphore m_JobQueueCounter;
+};
+
+#endif /* C_ASYNC_EVENTS_H_ */

src/script/lua_api/l_internal.h

@@ -30,14 +30,16 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "common/c_internal.h"
 
 #define luamethod(class, name) {#name, class::l_##name}
-#define API_FCT(name) registerFunction(L,#name,l_##name,top)
+#define API_FCT(name) registerFunction(L, #name, l_##name,top)
+#define ASYNC_API_FCT(name) engine.registerFunction(#name, l_##name)
 
 #if (defined(WIN32) || defined(_WIN32_WCE))
 #define NO_MAP_LOCK_REQUIRED
 #else
 #include "main.h"
 #include "profiler.h"
-#define NO_MAP_LOCK_REQUIRED ScopeProfiler nolocktime(g_profiler,"Scriptapi: unlockable time",SPT_ADD)
+#define NO_MAP_LOCK_REQUIRED \
+	ScopeProfiler nolocktime(g_profiler,"Scriptapi: unlockable time",SPT_ADD)
 #endif
 
 #endif /* L_INTERNAL_H_ */

src/script/lua_api/l_mainmenu.cpp

@@ -20,6 +20,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "lua_api/l_mainmenu.h"
 #include "lua_api/l_internal.h"
 #include "common/c_content.h"
+#include "lua_api/l_async_events.h"
 #include "guiEngine.h"
 #include "guiMainMenu.h"
 #include "guiKeyChangeMenu.h"
@@ -200,9 +201,6 @@ int ModApiMainMenu::l_get_textlist_index(lua_State *L)
 /******************************************************************************/
 int ModApiMainMenu::l_get_worlds(lua_State *L)
 {
-	GUIEngine* engine = getGuiEngine(L);
-	assert(engine != 0);
-
 	std::vector<WorldSpec> worlds = getAvailableWorlds();
 
 	lua_newtable(L);
@@ -237,9 +235,6 @@ int ModApiMainMenu::l_get_worlds(lua_State *L)
 /******************************************************************************/
 int ModApiMainMenu::l_get_games(lua_State *L)
 {
-	GUIEngine* engine = getGuiEngine(L);
-	assert(engine != 0);
-
 	std::vector<SubgameSpec> games = getAvailableGames();
 
 	lua_newtable(L);
@@ -365,9 +360,6 @@ int ModApiMainMenu::l_get_modstore_details(lua_State *L)
 /******************************************************************************/
 int ModApiMainMenu::l_get_modstore_list(lua_State *L)
 {
-	GUIEngine* engine = getGuiEngine(L);
-	assert(engine != 0);
-
 	std::string listtype = "local";
 
 	if (!lua_isnone(L,1)) {
@@ -421,9 +413,6 @@ int ModApiMainMenu::l_get_modstore_list(lua_State *L)
 /******************************************************************************/
 int ModApiMainMenu::l_get_favorites(lua_State *L)
 {
-	GUIEngine* engine = getGuiEngine(L);
-	assert(engine != 0);
-
 	std::string listtype = "local";
 
 	if (!lua_isnone(L,1)) {
@@ -545,9 +534,6 @@ int ModApiMainMenu::l_get_favorites(lua_State *L)
 /******************************************************************************/
 int ModApiMainMenu::l_delete_favorite(lua_State *L)
 {
-	GUIEngine* engine = getGuiEngine(L);
-	assert(engine != 0);
-
 	std::vector<ServerListSpec> servers;
 
 	std::string listtype = "local";
@@ -599,9 +585,6 @@ int ModApiMainMenu::l_show_keys_menu(lua_State *L)
 /******************************************************************************/
 int ModApiMainMenu::l_create_world(lua_State *L)
 {
-	GUIEngine* engine = getGuiEngine(L);
-	assert(engine != 0);
-
 	const char *name	= luaL_checkstring(L, 1);
 	int gameidx			= luaL_checkinteger(L,2) -1;
 
@@ -632,9 +615,6 @@ int ModApiMainMenu::l_create_world(lua_State *L)
 /******************************************************************************/
 int ModApiMainMenu::l_delete_world(lua_State *L)
 {
-	GUIEngine* engine = getGuiEngine(L);
-	assert(engine != 0);
-
 	int worldidx	= luaL_checkinteger(L,1) -1;
 
 	std::vector<WorldSpec> worlds = getAvailableWorlds();
@@ -962,9 +942,6 @@ int ModApiMainMenu::l_sound_stop(lua_State *L)
 /******************************************************************************/
 int ModApiMainMenu::l_download_file(lua_State *L)
 {
-	GUIEngine* engine = getGuiEngine(L);
-	assert(engine != 0);
-
 	const char *url    = luaL_checkstring(L, 1);
 	const char *target = luaL_checkstring(L, 2);
 
@@ -972,7 +949,7 @@ int ModApiMainMenu::l_download_file(lua_State *L)
 	std::string absolute_destination = fs::RemoveRelativePathComponents(target);
 
 	if (ModApiMainMenu::isMinetestPath(absolute_destination)) {
-		if (engine->downloadFile(url,absolute_destination)) {
+		if (GUIEngine::downloadFile(url,absolute_destination)) {
 			lua_pushboolean(L,true);
 			return 1;
 		}
@@ -990,6 +967,28 @@ int ModApiMainMenu::l_gettext(lua_State *L)
 	return 1;
 }
 
+/******************************************************************************/
+int ModApiMainMenu::l_do_async_callback(lua_State *L)
+{
+	GUIEngine* engine = getGuiEngine(L);
+
+	const char* serialized_fct_raw = luaL_checkstring(L, 1);
+	unsigned int lenght_fct = luaL_checkint(L, 2);
+
+	const char* serialized_params_raw = luaL_checkstring(L, 3);
+	unsigned int lenght_params = luaL_checkint(L, 4);
+
+	assert(serialized_fct_raw != 0);
+	assert(serialized_params_raw != 0);
+
+	std::string serialized_fct = std::string(serialized_fct_raw,lenght_fct);
+	std::string serialized_params = std::string(serialized_params_raw,lenght_params);
+
+	lua_pushinteger(L,engine->DoAsync(serialized_fct,serialized_params));
+
+	return 1;
+}
+
 /******************************************************************************/
 void ModApiMainMenu::Initialize(lua_State *L, int top)
 {
@@ -1024,4 +1023,27 @@ void ModApiMainMenu::Initialize(lua_State *L, int top)
 	API_FCT(sound_play);
 	API_FCT(sound_stop);
 	API_FCT(gettext);
+	API_FCT(do_async_callback);
+}
+
+/******************************************************************************/
+void ModApiMainMenu::InitializeAsync(AsyncEngine& engine)
+{
+
+	ASYNC_API_FCT(get_worlds);
+	ASYNC_API_FCT(get_games);
+	ASYNC_API_FCT(get_favorites);
+	ASYNC_API_FCT(get_modpath);
+	ASYNC_API_FCT(get_gamepath);
+	ASYNC_API_FCT(get_texturepath);
+	ASYNC_API_FCT(get_dirlist);
+	ASYNC_API_FCT(create_dir);
+	ASYNC_API_FCT(delete_dir);
+	ASYNC_API_FCT(copy_dir);
+	//ASYNC_API_FCT(extract_zip); //TODO remove dependency to GuiEngine
+	ASYNC_API_FCT(get_version);
+	ASYNC_API_FCT(download_file);
+	ASYNC_API_FCT(get_modstore_details);
+	ASYNC_API_FCT(get_modstore_list);
+	//ASYNC_API_FCT(gettext); (gettext lib isn't threadsafe)
 }

src/script/lua_api/l_mainmenu.h

@@ -22,6 +22,8 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 
 #include "lua_api/l_base.h"
 
+class AsyncEngine;
+
 /** Implementation of lua api support for mainmenu */
 class ModApiMainMenu : public ModApiBase {
 
@@ -125,6 +127,8 @@ private:
 
 	static int l_download_file(lua_State *L);
 
+	// async
+	static int l_do_async_callback(lua_State *L);
 
 public:
 	/**
@@ -134,6 +138,8 @@ public:
 	 */
 	static void Initialize(lua_State *L, int top);
 
+	static void InitializeAsync(AsyncEngine& engine);
+
 };
 
 #endif /* L_MAINMENU_H_ */

src/script/lua_api/l_util.cpp

@@ -21,6 +21,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "lua_api/l_internal.h"
 #include "common/c_converter.h"
 #include "common/c_content.h"
+#include "lua_api/l_async_events.h"
 #include "debug.h"
 #include "log.h"
 #include "tool.h"
@@ -257,3 +258,18 @@ void ModApiUtil::Initialize(lua_State *L, int top)
 	API_FCT(is_yes);
 }
 
+void ModApiUtil::InitializeAsync(AsyncEngine& engine)
+{
+	ASYNC_API_FCT(debug);
+	ASYNC_API_FCT(log);
+
+	//ASYNC_API_FCT(setting_set);
+	ASYNC_API_FCT(setting_get);
+	//ASYNC_API_FCT(setting_setbool);
+	ASYNC_API_FCT(setting_getbool);
+	//ASYNC_API_FCT(setting_save);
+
+	ASYNC_API_FCT(parse_json);
+
+	ASYNC_API_FCT(is_yes);
+}

src/script/lua_api/l_util.h

@@ -22,6 +22,8 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 
 #include "lua_api/l_base.h"
 
+class AsyncEngine;
+
 class ModApiUtil : public ModApiBase {
 private:
 	/*
@@ -77,6 +79,8 @@ private:
 public:
 	static void Initialize(lua_State *L, int top);
 
+	static void InitializeAsync(AsyncEngine& engine);
+
 };
 
 #endif /* L_UTIL_H_ */

src/script/lua_api/marshall.c

@@ -0,0 +1,551 @@
+/*
+* lmarshal.c
+* A Lua library for serializing and deserializing Lua values
+* Richard Hundt <richardhundt@gmail.com>
+*
+* License: MIT
+*
+* Copyright (c) 2010 Richard Hundt
+*
+* Permission is hereby granted, free of charge, to any person
+* obtaining a copy of this software and associated documentation
+* files (the "Software"), to deal in the Software without
+* restriction, including without limitation the rights to use,
+* copy, modify, merge, publish, distribute, sublicense, and/or sell
+* copies of the Software, and to permit persons to whom the
+* Software is furnished to do so, subject to the following
+* conditions:
+*
+* The above copyright notice and this permission notice shall be
+* included in all copies or substantial portions of the Software.
+*
+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+* OTHER DEALINGS IN THE SOFTWARE.
+*/
+
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+
+#include "lua.h"
+#include "lualib.h"
+#include "lauxlib.h"
+
+
+#define MAR_TREF 1
+#define MAR_TVAL 2
+#define MAR_TUSR 3
+
+#define MAR_CHR 1
+#define MAR_I32 4
+#define MAR_I64 8
+
+#define MAR_MAGIC 0x8e
+#define SEEN_IDX  3
+
+typedef struct mar_Buffer {
+    size_t size;
+    size_t seek;
+    size_t head;
+    char*  data;
+} mar_Buffer;
+
+static int mar_encode_table(lua_State *L, mar_Buffer *buf, size_t *idx);
+static int mar_decode_table(lua_State *L, const char* buf, size_t len, size_t *idx);
+
+static void buf_init(lua_State *L, mar_Buffer *buf)
+{
+    buf->size = 128;
+    buf->seek = 0;
+    buf->head = 0;
+    if (!(buf->data = malloc(buf->size))) luaL_error(L, "Out of memory!");
+}
+
+static void buf_done(lua_State* L, mar_Buffer *buf)
+{
+    free(buf->data);
+}
+
+static int buf_write(lua_State* L, const char* str, size_t len, mar_Buffer *buf)
+{
+    if (len > UINT32_MAX) luaL_error(L, "buffer too long");
+    if (buf->size - buf->head < len) {
+        size_t new_size = buf->size << 1;
+        size_t cur_head = buf->head;
+        while (new_size - cur_head <= len) {
+            new_size = new_size << 1;
+        }
+        if (!(buf->data = realloc(buf->data, new_size))) {
+            luaL_error(L, "Out of memory!");
+        }
+        buf->size = new_size;
+    }
+    memcpy(&buf->data[buf->head], str, len);
+    buf->head += len;
+    return 0;
+}
+
+static const char* buf_read(lua_State *L, mar_Buffer *buf, size_t *len)
+{
+    if (buf->seek < buf->head) {
+        buf->seek = buf->head;
+        *len = buf->seek;
+        return buf->data;
+    }
+    *len = 0;
+    return NULL;
+}
+
+static void mar_encode_value(lua_State *L, mar_Buffer *buf, int val, size_t *idx)
+{
+    size_t l;
+    int val_type = lua_type(L, val);
+    lua_pushvalue(L, val);
+
+    buf_write(L, (void*)&val_type, MAR_CHR, buf);
+    switch (val_type) {
+    case LUA_TBOOLEAN: {
+        int int_val = lua_toboolean(L, -1);
+        buf_write(L, (void*)&int_val, MAR_CHR, buf);
+        break;
+    }
+    case LUA_TSTRING: {
+        const char *str_val = lua_tolstring(L, -1, &l);
+        buf_write(L, (void*)&l, MAR_I32, buf);
+        buf_write(L, str_val, l, buf);
+        break;
+    }
+    case LUA_TNUMBER: {
+        lua_Number num_val = lua_tonumber(L, -1);
+        buf_write(L, (void*)&num_val, MAR_I64, buf);
+        break;
+    }
+    case LUA_TTABLE: {
+        int tag, ref;
+        lua_pushvalue(L, -1);
+        lua_rawget(L, SEEN_IDX);
+        if (!lua_isnil(L, -1)) {
+            ref = lua_tointeger(L, -1);
+            tag = MAR_TREF;
+            buf_write(L, (void*)&tag, MAR_CHR, buf);
+            buf_write(L, (void*)&ref, MAR_I32, buf);
+            lua_pop(L, 1);
+        }
+        else {
+            mar_Buffer rec_buf;
+            lua_pop(L, 1); /* pop nil */
+            if (luaL_getmetafield(L, -1, "__persist")) {
+                tag = MAR_TUSR;
+
+                lua_pushvalue(L, -2); /* self */
+                lua_call(L, 1, 1);
+                if (!lua_isfunction(L, -1)) {
+                    luaL_error(L, "__persist must return a function");
+                }
+
+                lua_remove(L, -2); /* __persist */
+
+                lua_newtable(L);
+                lua_pushvalue(L, -2); /* callback */
+                lua_rawseti(L, -2, 1);
+
+                buf_init(L, &rec_buf);
+                mar_encode_table(L, &rec_buf, idx);
+
+                buf_write(L, (void*)&tag, MAR_CHR, buf);
+                buf_write(L, (void*)&rec_buf.head, MAR_I32, buf);
+                buf_write(L, rec_buf.data, rec_buf.head, buf);
+                buf_done(L, &rec_buf);
+                lua_pop(L, 1);
+            }
+            else {
+                tag = MAR_TVAL;
+
+                lua_pushvalue(L, -1);
+                lua_pushinteger(L, (*idx)++);
+                lua_rawset(L, SEEN_IDX);
+
+                lua_pushvalue(L, -1);
+                buf_init(L, &rec_buf);
+                mar_encode_table(L, &rec_buf, idx);
+                lua_pop(L, 1);
+
+                buf_write(L, (void*)&tag, MAR_CHR, buf);
+                buf_write(L, (void*)&rec_buf.head, MAR_I32, buf);
+                buf_write(L, rec_buf.data,rec_buf.head, buf);
+                buf_done(L, &rec_buf);
+            }
+        }
+        break;
+    }
+    case LUA_TFUNCTION: {
+        int tag, ref;
+        lua_pushvalue(L, -1);
+        lua_rawget(L, SEEN_IDX);
+        if (!lua_isnil(L, -1)) {
+            ref = lua_tointeger(L, -1);
+            tag = MAR_TREF;
+            buf_write(L, (void*)&tag, MAR_CHR, buf);
+            buf_write(L, (void*)&ref, MAR_I32, buf);
+            lua_pop(L, 1);
+        }
+        else {
+            mar_Buffer rec_buf;
+            int i;
+            lua_Debug ar;
+            lua_pop(L, 1); /* pop nil */
+
+            lua_pushvalue(L, -1);
+            lua_getinfo(L, ">nuS", &ar);
+            if (ar.what[0] != 'L') {
+                luaL_error(L, "attempt to persist a C function '%s'", ar.name);
+            }
+            tag = MAR_TVAL;
+            lua_pushvalue(L, -1);
+            lua_pushinteger(L, (*idx)++);
+            lua_rawset(L, SEEN_IDX);
+
+            lua_pushvalue(L, -1);
+            buf_init(L, &rec_buf);
+            lua_dump(L, (lua_Writer)buf_write, &rec_buf);
+
+            buf_write(L, (void*)&tag, MAR_CHR, buf);
+            buf_write(L, (void*)&rec_buf.head, MAR_I32, buf);
+            buf_write(L, rec_buf.data, rec_buf.head, buf);
+            buf_done(L, &rec_buf);
+            lua_pop(L, 1);
+
+            lua_newtable(L);
+            for (i=1; i <= ar.nups; i++) {
+                lua_getupvalue(L, -2, i);
+                lua_rawseti(L, -2, i);
+            }
+
+            buf_init(L, &rec_buf);
+            mar_encode_table(L, &rec_buf, idx);
+
+            buf_write(L, (void*)&rec_buf.head, MAR_I32, buf);
+            buf_write(L, rec_buf.data, rec_buf.head, buf);
+            buf_done(L, &rec_buf);
+            lua_pop(L, 1);
+        }
+
+        break;
+    }
+    case LUA_TUSERDATA: {
+        int tag, ref;
+        lua_pushvalue(L, -1);
+        lua_rawget(L, SEEN_IDX);
+        if (!lua_isnil(L, -1)) {
+            ref = lua_tointeger(L, -1);
+            tag = MAR_TREF;
+            buf_write(L, (void*)&tag, MAR_CHR, buf);
+            buf_write(L, (void*)&ref, MAR_I32, buf);
+            lua_pop(L, 1);
+        }
+        else {
+            mar_Buffer rec_buf;
+            lua_pop(L, 1); /* pop nil */
+            if (luaL_getmetafield(L, -1, "__persist")) {
+                tag = MAR_TUSR;
+
+                lua_pushvalue(L, -2);
+                lua_pushinteger(L, (*idx)++);
+                lua_rawset(L, SEEN_IDX);
+
+                lua_pushvalue(L, -2);
+                lua_call(L, 1, 1);
+                if (!lua_isfunction(L, -1)) {
+                    luaL_error(L, "__persist must return a function");
+                }
+                lua_newtable(L);
+                lua_pushvalue(L, -2);
+                lua_rawseti(L, -2, 1);
+                lua_remove(L, -2);
+
+                buf_init(L, &rec_buf);
+                mar_encode_table(L, &rec_buf, idx);
+
+                buf_write(L, (void*)&tag, MAR_CHR, buf);
+                buf_write(L, (void*)&rec_buf.head, MAR_I32, buf);
+		buf_write(L, rec_buf.data, rec_buf.head, buf);
+		buf_done(L, &rec_buf);
+            }
+            else {
+                luaL_error(L, "attempt to encode userdata (no __persist hook)");
+            }
+            lua_pop(L, 1);
+        }
+        break;
+    }
+    case LUA_TNIL: break;
+    default:
+        luaL_error(L, "invalid value type (%s)", lua_typename(L, val_type));
+    }
+    lua_pop(L, 1);
+}
+
+static int mar_encode_table(lua_State *L, mar_Buffer *buf, size_t *idx)
+{
+    lua_pushnil(L);
+    while (lua_next(L, -2) != 0) {
+        mar_encode_value(L, buf, -2, idx);
+        mar_encode_value(L, buf, -1, idx);
+        lua_pop(L, 1);
+    }
+    return 1;
+}
+
+#define mar_incr_ptr(l) \
+    if (((*p)-buf)+(l) > len) luaL_error(L, "bad code"); (*p) += (l);
+
+#define mar_next_len(l,T) \
+    if (((*p)-buf)+sizeof(T) > len) luaL_error(L, "bad code"); \
+    l = *(T*)*p; (*p) += sizeof(T);
+
+static void mar_decode_value
+    (lua_State *L, const char *buf, size_t len, const char **p, size_t *idx)
+{
+    size_t l;
+    char val_type = **p;
+    mar_incr_ptr(MAR_CHR);
+    switch (val_type) {
+    case LUA_TBOOLEAN:
+        lua_pushboolean(L, *(char*)*p);
+        mar_incr_ptr(MAR_CHR);
+        break;
+    case LUA_TNUMBER:
+        lua_pushnumber(L, *(lua_Number*)*p);
+        mar_incr_ptr(MAR_I64);
+        break;
+    case LUA_TSTRING:
+        mar_next_len(l, uint32_t);
+        lua_pushlstring(L, *p, l);
+        mar_incr_ptr(l);
+        break;
+    case LUA_TTABLE: {
+        char tag = *(char*)*p;
+        mar_incr_ptr(MAR_CHR);
+        if (tag == MAR_TREF) {
+            int ref;
+            mar_next_len(ref, int);
+            lua_rawgeti(L, SEEN_IDX, ref);
+        }
+        else if (tag == MAR_TVAL) {
+            mar_next_len(l, uint32_t);
+            lua_newtable(L);
+            lua_pushvalue(L, -1);
+            lua_rawseti(L, SEEN_IDX, (*idx)++);
+            mar_decode_table(L, *p, l, idx);
+            mar_incr_ptr(l);
+        }
+        else if (tag == MAR_TUSR) {
+            mar_next_len(l, uint32_t);
+            lua_newtable(L);
+            mar_decode_table(L, *p, l, idx);
+            lua_rawgeti(L, -1, 1);
+            lua_call(L, 0, 1);
+            lua_remove(L, -2);
+            lua_pushvalue(L, -1);
+            lua_rawseti(L, SEEN_IDX, (*idx)++);
+            mar_incr_ptr(l);
+        }
+        else {
+            luaL_error(L, "bad encoded data");
+        }
+        break;
+    }
+    case LUA_TFUNCTION: {
+        size_t nups;
+        int i;
+        mar_Buffer dec_buf;
+        char tag = *(char*)*p;
+        mar_incr_ptr(1);
+        if (tag == MAR_TREF) {
+            int ref;
+            mar_next_len(ref, int);
+            lua_rawgeti(L, SEEN_IDX, ref);
+        }
+        else {
+            mar_next_len(l, uint32_t);
+            dec_buf.data = (char*)*p;
+            dec_buf.size = l;
+            dec_buf.head = l;
+            dec_buf.seek = 0;
+            lua_load(L, (lua_Reader)buf_read, &dec_buf, "=marshal");
+            mar_incr_ptr(l);
+
+            lua_pushvalue(L, -1);
+            lua_rawseti(L, SEEN_IDX, (*idx)++);
+
+            mar_next_len(l, uint32_t);
+            lua_newtable(L);
+            mar_decode_table(L, *p, l, idx);
+            nups = lua_objlen(L, -1);
+            for (i=1; i <= nups; i++) {
+                lua_rawgeti(L, -1, i);
+                lua_setupvalue(L, -3, i);
+            }
+            lua_pop(L, 1);
+            mar_incr_ptr(l);
+        }
+        break;
+    }
+    case LUA_TUSERDATA: {
+        char tag = *(char*)*p;
+        mar_incr_ptr(MAR_CHR);
+        if (tag == MAR_TREF) {
+            int ref;
+            mar_next_len(ref, int);
+            lua_rawgeti(L, SEEN_IDX, ref);
+        }
+        else if (tag == MAR_TUSR) {
+            mar_next_len(l, uint32_t);
+            lua_newtable(L);
+            mar_decode_table(L, *p, l, idx);
+            lua_rawgeti(L, -1, 1);
+            lua_call(L, 0, 1);
+            lua_remove(L, -2);
+            lua_pushvalue(L, -1);
+            lua_rawseti(L, SEEN_IDX, (*idx)++);
+            mar_incr_ptr(l);
+        }
+        else { /* tag == MAR_TVAL */
+            lua_pushnil(L);
+        }
+        break;
+    }
+    case LUA_TNIL:
+    case LUA_TTHREAD:
+        lua_pushnil(L);
+        break;
+    default:
+        luaL_error(L, "bad code");
+    }
+}
+
+static int mar_decode_table(lua_State *L, const char* buf, size_t len, size_t *idx)
+{
+    const char* p;
+    p = buf;
+    while (p - buf < len) {
+        mar_decode_value(L, buf, len, &p, idx);
+        mar_decode_value(L, buf, len, &p, idx);
+        lua_settable(L, -3);
+    }
+    return 1;
+}
+
+static int mar_encode(lua_State* L)
+{
+    const unsigned char m = MAR_MAGIC;
+    size_t idx, len;
+    mar_Buffer buf;
+
+    if (lua_isnone(L, 1)) {
+        lua_pushnil(L);
+    }
+    if (lua_isnoneornil(L, 2)) {
+        lua_newtable(L);
+    }
+    else if (!lua_istable(L, 2)) {
+        luaL_error(L, "bad argument #2 to encode (expected table)");
+    }
+    lua_settop(L, 2);
+
+    len = lua_objlen(L, 2);
+    lua_newtable(L);
+    for (idx = 1; idx <= len; idx++) {
+        lua_rawgeti(L, 2, idx);
+        if (lua_isnil(L, -1)) {
+            lua_pop(L, 1);
+            continue;
+        }
+        lua_pushinteger(L, idx);
+        lua_rawset(L, SEEN_IDX);
+    }
+    lua_pushvalue(L, 1);
+
+    buf_init(L, &buf);
+    buf_write(L, (void*)&m, 1, &buf);
+
+    mar_encode_value(L, &buf, -1, &idx);
+
+    lua_pop(L, 1);
+
+    lua_pushlstring(L, buf.data, buf.head);
+
+    buf_done(L, &buf);
+
+    lua_remove(L, SEEN_IDX);
+
+    return 1;
+}
+
+static int mar_decode(lua_State* L)
+{
+    size_t l, idx, len;
+    const char *p;
+    const char *s = luaL_checklstring(L, 1, &l);
+
+    if (l < 1) luaL_error(L, "bad header");
+    if (*(unsigned char *)s++ != MAR_MAGIC) luaL_error(L, "bad magic");
+    l -= 1;
+
+    if (lua_isnoneornil(L, 2)) {
+        lua_newtable(L);
+    }
+    else if (!lua_istable(L, 2)) {
+        luaL_error(L, "bad argument #2 to decode (expected table)");
+    }
+    lua_settop(L, 2);
+
+    len = lua_objlen(L, 2);
+    lua_newtable(L);
+    for (idx = 1; idx <= len; idx++) {
+        lua_rawgeti(L, 2, idx);
+        lua_rawseti(L, SEEN_IDX, idx);
+    }
+
+    p = s;
+    mar_decode_value(L, s, l, &p, &idx);
+
+    lua_remove(L, SEEN_IDX);
+    lua_remove(L, 2);
+
+    return 1;
+}
+
+static int mar_clone(lua_State* L)
+{
+    mar_encode(L);
+    lua_replace(L, 1);
+    mar_decode(L);
+    return 1;
+}
+
+static const luaL_reg R[] =
+{
+    {"encode",      mar_encode},
+    {"decode",      mar_decode},
+    {"clone",       mar_clone},
+    {NULL,	    NULL}
+};
+
+int luaopen_marshal(lua_State *L)
+{
+    lua_newtable(L);
+    luaL_register(L, "marshal", R);
+    return 1;
+}
+
+
+
+
+

src/script/scripting_mainmenu.cpp

@@ -28,8 +28,9 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 
 extern "C" {
 #include "lualib.h"
+	int luaopen_marshal(lua_State *L);
 }
-
+/******************************************************************************/
 MainMenuScripting::MainMenuScripting(GUIEngine* guiengine)
 {
 	setGuiEngine(guiengine);
@@ -37,6 +38,7 @@ MainMenuScripting::MainMenuScripting(GUIEngine* guiengine)
 	//TODO add security
 
 	luaL_openlibs(getStack());
+	luaopen_marshal(getStack());
 
 	SCRIPTAPI_PRECHECKHEADER
 
@@ -58,6 +60,7 @@ MainMenuScripting::MainMenuScripting(GUIEngine* guiengine)
 	infostream << "SCRIPTAPI: initialized mainmenu modules" << std::endl;
 }
 
+/******************************************************************************/
 void MainMenuScripting::InitializeModApi(lua_State *L, int top)
 {
 	// Initialize mod api modules
@@ -66,4 +69,23 @@ void MainMenuScripting::InitializeModApi(lua_State *L, int top)
 
 	// Register reference classes (userdata)
 	LuaSettings::Register(L);
+
+	// Register functions to async environment
+	ModApiMainMenu::InitializeAsync(m_AsyncEngine);
+	ModApiUtil::InitializeAsync(m_AsyncEngine);
+
+	// Initialize async environment
+	//TODO possibly make number of async threads configurable
+	m_AsyncEngine.Initialize(MAINMENU_NUMBER_OF_ASYNC_THREADS);
+}
+
+/******************************************************************************/
+void MainMenuScripting::Step() {
+	m_AsyncEngine.Step(getStack());
+}
+
+/******************************************************************************/
+unsigned int MainMenuScripting::DoAsync(std::string serialized_fct,
+		std::string serialized_params) {
+	return m_AsyncEngine.doAsyncJob(serialized_fct,serialized_params);
 }

src/script/scripting_mainmenu.h

@@ -22,6 +22,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 
 #include "cpp_api/s_base.h"
 #include "cpp_api/s_mainmenu.h"
+#include "lua_api/l_async_events.h"
 
 /*****************************************************************************/
 /* Scripting <-> Main Menu Interface                                         */
@@ -37,8 +38,16 @@ public:
 	// use ScriptApiBase::loadMod() or ScriptApiBase::loadScript()
 	// to load scripts
 
+	/* global step handler to pass back async events */
+	void Step();
+
+	/* pass async events from engine to async threads */
+	unsigned int DoAsync(std::string serialized_fct,
+			std::string serialized_params);
 private:
 	void InitializeModApi(lua_State *L, int top);
+
+	AsyncEngine m_AsyncEngine;
 };