Add Lua interface to HTTPFetchRequest

This allows mods to perform both asynchronous and synchronous HTTP
requests. Mods are only granted access to HTTP APIs if either mod
security is disabled or if they are whitelisted in any of the
the secure.http_mods and secure.trusted_mods settings.

Adds httpfetch_caller_alloc_secure to generate random, non-predictable
caller IDs so that lua mods cannot spy on each others HTTP queries.
This commit is contained in:
Jeija 2016-02-18 11:38:47 +01:00 committed by est31
parent a3892f5a66
commit 31e0667a4a
12 changed files with 357 additions and 1 deletions

@ -178,3 +178,22 @@ function core.raillike_group(name)
end
return id
end
-- HTTP callback interface
function core.http_add_fetch(httpenv)
httpenv.fetch = function(req, callback)
local handle = httpenv.fetch_async(req)
local function update_http_status()
local res = httpenv.fetch_async_get(handle)
if res.completed then
callback(res)
else
core.after(0, update_http_status)
end
end
core.after(0, update_http_status)
end
return httpenv
end

@ -1100,6 +1100,10 @@ secure.enable_security (Enable mod security) bool false
# functions even when mod security is on (via request_insecure_environment()).
secure.trusted_mods (Trusted mods) string
# Comma-seperated list of mods that are allowed to access HTTP APIs, which
# allow them to upload and download data to/from the internet.
secure.http_mods (HTTP Mods) string
[Client and Server]
# Name of the player.

@ -2330,6 +2330,26 @@ These functions return the leftover itemstack.
* If `lua_num_indent_spaces` is a nonzero number and `format` is "lua", the Lua code generated
* will use that number of spaces as indentation instead of a tab character.
### HTTP Requests:
* `minetest.request_http_api()`:
* returns `HTTPApiTable` containing http functions if the calling mod has been granted
access by being listed in the `secure.http_mods` or `secure.trusted_mods` setting,
otherwise returns `nil`.
* The returned table contains the functions `fetch`, `fetch_async` and `fetch_async_get`
described below.
* Only works at init time.
* Function only exists if minetest server was built with cURL support.
* **DO NOT ALLOW ANY OTHER MODS TO ACCESS THE RETURNED TABLE, STORE IT IN
A LOCAL VARIABLE!**
* `HTTPApiTable.fetch(HTTPRequest req, callback)`
* Performs given request asynchronously and calls callback upon completion
* callback: `function(HTTPRequestResult res)`
* Use this HTTP function if you are unsure, the others are for advanced use.
* `HTTPApiTable.fetch_async(HTTPRequest req)`: returns handle
* Performs given request asynchronously and returns handle for `minetest.http_fetch_async_get`
* `HTTPApiTable.fetch_async_get(handle)`: returns HTTPRequestResult
* Return response data for given asynchronous HTTP request
### Misc.
* `minetest.get_connected_players()`: returns list of `ObjectRefs`
* `minetest.hash_node_position({x=,y=,z=})`: returns an 48-bit integer
@ -3837,3 +3857,37 @@ Definition tables
playername = "singleplayer"
-- ^ Playername is optional, if specified spawns particle only on the player's client
}
### `HTTPRequest` definition (`http_fetch`, `http_fetch_async`)
{
url = "http://example.org",
timeout = 10,
-- ^ Timeout for connection in seconds. Default is 3 seconds.
post_data = "Raw POST request data string" OR { field1 = "data1", field2 = "data2" },
-- ^ Optional, if specified a POST request with post_data is performed.
-- ^ Accepts both a string and a table. If a table is specified, encodes table
-- ^ as x-www-form-urlencoded key-value pairs.
-- ^ If post_data ist not specified, a GET request is performed instead.
user_agent = "ExampleUserAgent",
-- ^ Optional, if specified replaces the default minetest user agent with given string
extra_headers = { "Accept-Language: en-us", "Accept-Charset: utf-8" },
-- ^ Optional, if specified adds additional headers to the HTTP request. You must make sure
-- ^ that the header strings follow HTTP specification ("Key: Value").
multipart = boolean
-- ^ Optional, if true performs a multipart HTTP request. Default is false.
}
### `HTTPRequestResult` definition (`http_fetch` callback, `http_fetch_async_get`)
{
completed = true,
-- ^ If true, the request has finished (either succeeded, failed or timed out)
succeeded = true,
-- ^ If true, the request was succesful
timeout = false,
-- ^ If true, the request timed out
code = 200,
-- ^ HTTP status code
data = "response"
}

@ -285,6 +285,7 @@ void set_default_settings(Settings *settings)
settings->setDefault("num_emerge_threads", "1");
settings->setDefault("secure.enable_security", "false");
settings->setDefault("secure.trusted_mods", "");
settings->setDefault("secure.http_mods", "");
// physics stuff
settings->setDefault("movement_acceleration_default", "3");

@ -18,7 +18,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
*/
#include "socket.h" // for select()
#include "porting.h" // for sleep_ms(), get_sysinfo()
#include "porting.h" // for sleep_ms(), get_sysinfo(), secure_rand_fill_buf()
#include "httpfetch.h"
#include <iostream>
#include <sstream>
@ -34,9 +34,11 @@ with this program; if not, write to the Free Software Foundation, Inc.,
#include "util/thread.h"
#include "version.h"
#include "settings.h"
#include "noise.h"
Mutex g_httpfetch_mutex;
std::map<unsigned long, std::queue<HTTPFetchResult> > g_httpfetch_results;
PcgRandom g_callerid_randomness;
HTTPFetchRequest::HTTPFetchRequest()
{
@ -84,6 +86,34 @@ unsigned long httpfetch_caller_alloc()
return discard;
}
unsigned long httpfetch_caller_alloc_secure()
{
MutexAutoLock lock(g_httpfetch_mutex);
// Generate random caller IDs and make sure they're not
// already used or equal to HTTPFETCH_DISCARD
// Give up after 100 tries to prevent infinite loop
u8 tries = 100;
unsigned long caller;
do {
caller = (((u64) g_callerid_randomness.next()) << 32) |
g_callerid_randomness.next();
if (--tries < 1) {
FATAL_ERROR("httpfetch_caller_alloc_secure: ran out of caller IDs");
return HTTPFETCH_DISCARD;
}
} while (g_httpfetch_results.find(caller) != g_httpfetch_results.end());
verbosestream << "httpfetch_caller_alloc_secure: allocating "
<< caller << std::endl;
// Access element to create it
g_httpfetch_results[caller];
return caller;
}
void httpfetch_caller_free(unsigned long caller)
{
verbosestream<<"httpfetch_caller_free: freeing "
@ -710,6 +740,11 @@ void httpfetch_init(int parallel_limit)
FATAL_ERROR_IF(res != CURLE_OK, "CURL init failed");
g_httpfetch_thread = new CurlFetchThread(parallel_limit);
// Initialize g_callerid_randomness for httpfetch_caller_alloc_secure
u64 randbuf[2];
porting::secure_rand_fill_buf(randbuf, sizeof(u64) * 2);
g_callerid_randomness = PcgRandom(randbuf[0], randbuf[1]);
}
void httpfetch_cleanup()

@ -116,6 +116,9 @@ bool httpfetch_async_get(unsigned long caller, HTTPFetchResult &fetch_result);
// Not required if you want to set caller = HTTPFETCH_DISCARD
unsigned long httpfetch_caller_alloc();
// Allocates a non-predictable caller ID for httpfetch_async
unsigned long httpfetch_caller_alloc_secure();
// Frees a caller ID allocated with httpfetch_caller_alloc
// Note: This can be expensive, because the httpfetch thread is told
// to stop any ongoing fetches for the given caller.

@ -517,6 +517,15 @@ bool getboolfield_default(lua_State *L, int table,
return result;
}
void setstringfield(lua_State *L, int table,
const char *fieldname, const char *value)
{
lua_pushstring(L, value);
if(table < 0)
table -= 1;
lua_setfield(L, table, fieldname);
}
void setintfield(lua_State *L, int table,
const char *fieldname, int value)
{

@ -69,6 +69,8 @@ bool getfloatfield(lua_State *L, int table,
std::string checkstringfield(lua_State *L, int table,
const char *fieldname);
void setstringfield(lua_State *L, int table,
const char *fieldname, const char *value);
void setintfield(lua_State *L, int table,
const char *fieldname, int value);
void setfloatfield(lua_State *L, int table,

@ -16,6 +16,7 @@ set(common_SCRIPT_LUA_API_SRCS
${CMAKE_CURRENT_SOURCE_DIR}/l_util.cpp
${CMAKE_CURRENT_SOURCE_DIR}/l_vmanip.cpp
${CMAKE_CURRENT_SOURCE_DIR}/l_settings.cpp
${CMAKE_CURRENT_SOURCE_DIR}/l_http.cpp
PARENT_SCOPE)
set(client_SCRIPT_LUA_API_SRCS

@ -0,0 +1,176 @@
/*
Minetest
Copyright (C) 2013 celeron55, Perttu Ahola <celeron55@gmail.com>
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation; either version 2.1 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License along
with this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
#include "lua_api/l_internal.h"
#include "common/c_converter.h"
#include "common/c_content.h"
#include "lua_api/l_http.h"
#include "httpfetch.h"
#include "settings.h"
#include "log.h"
#include <algorithm>
#include <iomanip>
#include <cctype>
#define HTTP_API(name) \
lua_pushstring(L, #name); \
lua_pushcfunction(L, l_http_##name); \
lua_settable(L, -3);
#if USE_CURL
void ModApiHttp::read_http_fetch_request(lua_State *L, HTTPFetchRequest &req)
{
luaL_checktype(L, 1, LUA_TTABLE);
req.caller = httpfetch_caller_alloc_secure();
getstringfield(L, 1, "url", req.url);
lua_getfield(L, 1, "user_agent");
if (lua_isstring(L, -1))
req.useragent = getstringfield_default(L, 1, "user_agent", "");
lua_pop(L, 1);
req.multipart = getboolfield_default(L, 1, "multipart", false);
req.timeout = getintfield_default(L, 1, "timeout", 3) * 1000;
// post_data: if table, post form data, otherwise raw data
lua_getfield(L, 1, "post_data");
if (lua_istable(L, 2)) {
lua_pushnil(L);
while (lua_next(L, 2) != 0)
{
req.post_fields[luaL_checkstring(L, -2)] = luaL_checkstring(L, -1);
lua_pop(L, 1);
}
} else if (lua_isstring(L, 2)) {
req.post_data = lua_tostring(L, 2);
}
lua_pop(L, 1);
lua_getfield(L, 1, "extra_headers");
if (lua_istable(L, 2)) {
lua_pushnil(L);
while (lua_next(L, 2) != 0)
{
const char *header = luaL_checkstring(L, -1);
req.extra_headers.push_back(header);
lua_pop(L, 1);
}
}
lua_pop(L, 1);
}
void ModApiHttp::push_http_fetch_result(lua_State *L, HTTPFetchResult &res, bool completed)
{
lua_newtable(L);
setboolfield(L, -1, "succeeded", res.succeeded);
setboolfield(L, -1, "timeout", res.timeout);
setboolfield(L, -1, "completed", completed);
setintfield(L, -1, "code", res.response_code);
setstringfield(L, -1, "data", res.data.c_str());
}
// http_api.fetch_async(HTTPRequest definition)
int ModApiHttp::l_http_fetch_async(lua_State *L)
{
NO_MAP_LOCK_REQUIRED;
HTTPFetchRequest req;
read_http_fetch_request(L, req);
actionstream << "Mod performs HTTP request with URL " << req.url << std::endl;
httpfetch_async(req);
// Convert handle to hex string since lua can't handle 64-bit integers
std::stringstream handle_conversion_stream;
handle_conversion_stream << std::hex << req.caller;
std::string caller_handle(handle_conversion_stream.str());
lua_pushstring(L, caller_handle.c_str());
return 1;
}
// http_api.fetch_async_get(handle)
int ModApiHttp::l_http_fetch_async_get(lua_State *L)
{
NO_MAP_LOCK_REQUIRED;
std::string handle_str = luaL_checkstring(L, 1);
// Convert hex string back to 64-bit handle
u64 handle;
std::stringstream handle_conversion_stream;
handle_conversion_stream << std::hex << handle_str;
handle_conversion_stream >> handle;
HTTPFetchResult res;
bool completed = httpfetch_async_get(handle, res);
push_http_fetch_result(L, res, completed);
return 1;
}
int ModApiHttp::l_request_http_api(lua_State *L)
{
NO_MAP_LOCK_REQUIRED;
// Mod must be listed in secure.http_mods or secure.trusted_mods
lua_rawgeti(L, LUA_REGISTRYINDEX, CUSTOM_RIDX_CURRENT_MOD_NAME);
if (!lua_isstring(L, -1)) {
lua_pushnil(L);
return 1;
}
const char *mod_name = lua_tostring(L, -1);
std::string http_mods = g_settings->get("secure.http_mods");
http_mods.erase(std::remove(http_mods.begin(), http_mods.end(), ' '), http_mods.end());
std::vector<std::string> mod_list_http = str_split(http_mods, ',');
std::string trusted_mods = g_settings->get("secure.trusted_mods");
trusted_mods.erase(std::remove(trusted_mods.begin(), trusted_mods.end(), ' '), trusted_mods.end());
std::vector<std::string> mod_list_trusted = str_split(trusted_mods, ',');
mod_list_http.insert(mod_list_http.end(), mod_list_trusted.begin(), mod_list_trusted.end());
if (std::find(mod_list_http.begin(), mod_list_http.end(), mod_name) == mod_list_http.end()) {
lua_pushnil(L);
return 1;
}
lua_getglobal(L, "core");
lua_getfield(L, -1, "http_add_fetch");
lua_newtable(L);
HTTP_API(fetch_async);
HTTP_API(fetch_async_get);
// Stack now looks like this:
// <core.http_add_fetch> <table with fetch_async, fetch_async_get>
// Now call core.http_add_fetch to append .fetch(request, callback) to table
lua_call(L, 1, 1);
return 1;
}
#endif
void ModApiHttp::Initialize(lua_State *L, int top)
{
#if USE_CURL
API_FCT(request_http_api);
#endif
}

@ -0,0 +1,50 @@
/*
Minetest
Copyright (C) 2013 celeron55, Perttu Ahola <celeron55@gmail.com>
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation; either version 2.1 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License along
with this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
#ifndef L_HTTP_H_
#define L_HTTP_H_
#include "lua_api/l_base.h"
#include "config.h"
struct HTTPFetchRequest;
struct HTTPFetchResult;
class ModApiHttp : public ModApiBase {
private:
#if USE_CURL
// Helpers for HTTP fetch functions
static void read_http_fetch_request(lua_State *L, HTTPFetchRequest &req);
static void push_http_fetch_result(lua_State *L, HTTPFetchResult &res, bool completed = true);
// http_fetch_async({url=, timeout=, post_data=})
static int l_http_fetch_async(lua_State *L);
// http_fetch_async_get(handle)
static int l_http_fetch_async_get(lua_State *L);
// request_http_api()
static int l_request_http_api(lua_State *L);
#endif
public:
static void Initialize(lua_State *L, int top);
};
#endif /* L_HTTP_H_ */

@ -39,6 +39,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
#include "lua_api/l_util.h"
#include "lua_api/l_vmanip.h"
#include "lua_api/l_settings.h"
#include "lua_api/l_http.h"
extern "C" {
#include "lualib.h"
@ -89,6 +90,7 @@ void GameScripting::InitializeModApi(lua_State *L, int top)
ModApiRollback::Initialize(L, top);
ModApiServer::Initialize(L, top);
ModApiUtil::Initialize(L, top);
ModApiHttp::Initialize(L, top);
// Register reference classes (userdata)
InvRef::Register(L);