Add server side translations capability (#9733)

* Add server side translations capability
This commit is contained in:
EvidenceB Kidscode 2020-04-25 07:20:00 +02:00 committed by GitHub
parent 914dbeaa0b
commit cee3c5e73d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 126 additions and 18 deletions

@ -3176,8 +3176,22 @@ Strings that need to be translated can contain several escapes, preceded by `@`.
`minetest.translate`, but is in translation files. `minetest.translate`, but is in translation files.
* `@n` acts as a literal newline as well. * `@n` acts as a literal newline as well.
Server side translations
------------------------
On some specific cases, server translation could be useful. For example, filter
a list on labels and send results to client. A method is supplied to achieve
that:
`minetest.get_translated_string(lang_code, string)`: Translates `string` using
translations for `lang_code` language. It gives the same result as if the string
was translated by the client.
The `lang_code` to use for a given player can be retrieved from
the table returned by `minetest.get_player_information(name)`.
IMPORTANT: This functionality should only be used for sorting, filtering or similar purposes.
You do not need to use this to get translated strings to show up on the client.
Perlin noise Perlin noise
============ ============
@ -4153,6 +4167,7 @@ Utilities
connection_uptime = 200, -- seconds since client connected connection_uptime = 200, -- seconds since client connected
protocol_version = 32, -- protocol version used by client protocol_version = 32, -- protocol version used by client
formspec_version = 2, -- supported formspec version formspec_version = 2, -- supported formspec version
lang_code = "fr" -- Language code used for translation
-- following information is available on debug build only!!! -- following information is available on debug build only!!!
-- DO NOT USE IN MODS -- DO NOT USE IN MODS
--ser_vers = 26, -- serialization version used by client --ser_vers = 26, -- serialization version used by client

@ -736,7 +736,7 @@ bool Client::loadMedia(const std::string &data, const std::string &filename)
if (!name.empty()) { if (!name.empty()) {
TRACESTREAM(<< "Client: Loading translation: " TRACESTREAM(<< "Client: Loading translation: "
<< "\"" << filename << "\"" << std::endl); << "\"" << filename << "\"" << std::endl);
g_translations->loadTranslation(data); g_client_translations->loadTranslation(data);
return true; return true;
} }

@ -1055,7 +1055,7 @@ bool Game::startup(bool *kill,
m_invert_mouse = g_settings->getBool("invert_mouse"); m_invert_mouse = g_settings->getBool("invert_mouse");
m_first_loop_after_window_activation = true; m_first_loop_after_window_activation = true;
g_translations->clear(); g_client_translations->clear();
if (!init(map_dir, address, port, gamespec)) if (!init(map_dir, address, port, gamespec))
return false; return false;

@ -339,6 +339,9 @@ public:
u8 getMinor() const { return m_version_minor; } u8 getMinor() const { return m_version_minor; }
u8 getPatch() const { return m_version_patch; } u8 getPatch() const { return m_version_patch; }
const std::string &getFull() const { return m_full_version; } const std::string &getFull() const { return m_full_version; }
void setLangCode(const std::string &code) { m_lang_code = code; }
const std::string &getLangCode() const { return m_lang_code; }
private: private:
// Version is stored in here after INIT before INIT2 // Version is stored in here after INIT before INIT2
u8 m_pending_serialization_version = SER_FMT_VER_INVALID; u8 m_pending_serialization_version = SER_FMT_VER_INVALID;
@ -346,6 +349,9 @@ private:
/* current state of client */ /* current state of client */
ClientState m_state = CS_Created; ClientState m_state = CS_Created;
// Client sent language code
std::string m_lang_code;
/* /*
Blocks that have been sent to client. Blocks that have been sent to client.
- These don't have to be sent again. - These don't have to be sent again.

@ -311,6 +311,9 @@ void Server::handleCommand_Init2(NetworkPacket* pkt)
RemoteClient *client = getClient(peer_id, CS_InitDone); RemoteClient *client = getClient(peer_id, CS_InitDone);
// Keep client language for server translations
client->setLangCode(lang);
// Send active objects // Send active objects
{ {
PlayerSAO *sao = getPlayerSAO(peer_id); PlayerSAO *sao = getPlayerSAO(peer_id);

@ -40,6 +40,8 @@ with this program; if not, write to the Free Software Foundation, Inc.,
#include "remoteplayer.h" #include "remoteplayer.h"
#include "server/luaentity_sao.h" #include "server/luaentity_sao.h"
#include "server/player_sao.h" #include "server/player_sao.h"
#include "util/string.h"
#include "translation.h"
#ifndef SERVER #ifndef SERVER
#include "client/client.h" #include "client/client.h"
#endif #endif
@ -1302,6 +1304,19 @@ int ModApiEnvMod::l_forceload_free_block(lua_State *L)
return 0; return 0;
} }
// get_translated_string(lang_code, string)
int ModApiEnvMod::l_get_translated_string(lua_State * L)
{
GET_ENV_PTR;
std::string lang_code = luaL_checkstring(L, 1);
std::string string = luaL_checkstring(L, 2);
getServer(L)->loadTranslationLanguage(lang_code);
string = wide_to_utf8(translate_string(utf8_to_wide(string),
&(*g_server_translations)[lang_code]));
lua_pushstring(L, string.c_str());
return 1;
}
void ModApiEnvMod::Initialize(lua_State *L, int top) void ModApiEnvMod::Initialize(lua_State *L, int top)
{ {
API_FCT(set_node); API_FCT(set_node);
@ -1349,6 +1364,7 @@ void ModApiEnvMod::Initialize(lua_State *L, int top)
API_FCT(transforming_liquid_add); API_FCT(transforming_liquid_add);
API_FCT(forceload_block); API_FCT(forceload_block);
API_FCT(forceload_free_block); API_FCT(forceload_free_block);
API_FCT(get_translated_string);
} }
void ModApiEnvMod::InitializeClient(lua_State *L, int top) void ModApiEnvMod::InitializeClient(lua_State *L, int top)

@ -187,6 +187,9 @@ private:
// stops forceloading a position // stops forceloading a position
static int l_forceload_free_block(lua_State *L); static int l_forceload_free_block(lua_State *L);
// Get a string translated server side
static int l_get_translated_string(lua_State * L);
public: public:
static void Initialize(lua_State *L, int top); static void Initialize(lua_State *L, int top);
static void InitializeClient(lua_State *L, int top); static void InitializeClient(lua_State *L, int top);

@ -163,6 +163,7 @@ int ModApiServer::l_get_player_information(lua_State *L)
u16 prot_vers; u16 prot_vers;
u8 ser_vers,major,minor,patch; u8 ser_vers,major,minor,patch;
std::string vers_string; std::string vers_string;
std::string lang_code;
#define ERET(code) \ #define ERET(code) \
if (!(code)) { \ if (!(code)) { \
@ -182,7 +183,7 @@ int ModApiServer::l_get_player_information(lua_State *L)
&avg_jitter)) &avg_jitter))
ERET(getServer(L)->getClientInfo(player->getPeerId(), &state, &uptime, &ser_vers, ERET(getServer(L)->getClientInfo(player->getPeerId(), &state, &uptime, &ser_vers,
&prot_vers, &major, &minor, &patch, &vers_string)) &prot_vers, &major, &minor, &patch, &vers_string, &lang_code))
lua_newtable(L); lua_newtable(L);
int table = lua_gettop(L); int table = lua_gettop(L);
@ -237,6 +238,10 @@ int ModApiServer::l_get_player_information(lua_State *L)
lua_pushnumber(L, player->formspec_version); lua_pushnumber(L, player->formspec_version);
lua_settable(L, table); lua_settable(L, table);
lua_pushstring(L, "lang_code");
lua_pushstring(L, lang_code.c_str());
lua_settable(L, table);
#ifndef NDEBUG #ifndef NDEBUG
lua_pushstring(L,"serialization_version"); lua_pushstring(L,"serialization_version");
lua_pushnumber(L, ser_vers); lua_pushnumber(L, ser_vers);

@ -64,6 +64,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
#include "chat_interface.h" #include "chat_interface.h"
#include "remoteplayer.h" #include "remoteplayer.h"
#include "server/player_sao.h" #include "server/player_sao.h"
#include "translation.h"
class ClientNotFoundException : public BaseException class ClientNotFoundException : public BaseException
{ {
@ -1266,7 +1267,8 @@ bool Server::getClientInfo(
u8* major, u8* major,
u8* minor, u8* minor,
u8* patch, u8* patch,
std::string* vers_string std::string* vers_string,
std::string* lang_code
) )
{ {
*state = m_clients.getClientState(peer_id); *state = m_clients.getClientState(peer_id);
@ -1286,6 +1288,7 @@ bool Server::getClientInfo(
*minor = client->getMinor(); *minor = client->getMinor();
*patch = client->getPatch(); *patch = client->getPatch();
*vers_string = client->getFull(); *vers_string = client->getFull();
*lang_code = client->getLangCode();
m_clients.unlock(); m_clients.unlock();
@ -3937,3 +3940,20 @@ void Server::broadcastModChannelMessage(const std::string &channel,
m_script->on_modchannel_message(channel, sender, message); m_script->on_modchannel_message(channel, sender, message);
} }
} }
void Server::loadTranslationLanguage(const std::string &lang_code)
{
if (g_server_translations->count(lang_code))
return; // Already loaded
std::string suffix = "." + lang_code + ".tr";
for (const auto &i : m_media) {
if (str_ends_with(i.first, suffix)) {
std::ifstream t(i.second.path);
std::string data((std::istreambuf_iterator<char>(t)),
std::istreambuf_iterator<char>());
(*g_server_translations)[lang_code].loadTranslation(data);
}
}
}

@ -334,7 +334,7 @@ public:
bool getClientConInfo(session_t peer_id, con::rtt_stat_type type, float *retval); bool getClientConInfo(session_t peer_id, con::rtt_stat_type type, float *retval);
bool getClientInfo(session_t peer_id, ClientState *state, u32 *uptime, bool getClientInfo(session_t peer_id, ClientState *state, u32 *uptime,
u8* ser_vers, u16* prot_vers, u8* major, u8* minor, u8* patch, u8* ser_vers, u16* prot_vers, u8* major, u8* minor, u8* patch,
std::string* vers_string); std::string* vers_string, std::string* lang_code);
void printToConsoleOnly(const std::string &text); void printToConsoleOnly(const std::string &text);
@ -358,6 +358,9 @@ public:
// Send block to specific player only // Send block to specific player only
bool SendBlock(session_t peer_id, const v3s16 &blockpos); bool SendBlock(session_t peer_id, const v3s16 &blockpos);
// Load translations for a language
void loadTranslationLanguage(const std::string &lang_code);
// Bind address // Bind address
Address m_bind_addr; Address m_bind_addr;

@ -20,9 +20,18 @@ with this program; if not, write to the Free Software Foundation, Inc.,
#include "translation.h" #include "translation.h"
#include "log.h" #include "log.h"
#include "util/string.h" #include "util/string.h"
#include <unordered_map>
static Translations main_translations;
Translations *g_translations = &main_translations; #ifndef SERVER
// Client translations
Translations client_translations;
Translations *g_client_translations = &client_translations;
#endif
// Per language server translations
std::unordered_map<std::string,Translations> server_translations;
std::unordered_map<std::string,Translations> *g_server_translations = &server_translations;
Translations::~Translations() Translations::~Translations()
{ {

@ -23,7 +23,10 @@ with this program; if not, write to the Free Software Foundation, Inc.,
#include <string> #include <string>
class Translations; class Translations;
extern Translations *g_translations; extern std::unordered_map<std::string, Translations> *g_server_translations;
#ifndef SERVER
extern Translations *g_client_translations;
#endif
class Translations class Translations
{ {

@ -693,10 +693,12 @@ void str_replace(std::string &str, char from, char to)
* before filling it again. * before filling it again.
*/ */
void translate_all(const std::wstring &s, size_t &i, std::wstring &res); void translate_all(const std::wstring &s, size_t &i,
Translations *translations, std::wstring &res);
void translate_string(const std::wstring &s, const std::wstring &textdomain, void translate_string(const std::wstring &s, Translations *translations,
size_t &i, std::wstring &res) { const std::wstring &textdomain, size_t &i, std::wstring &res)
{
std::wostringstream output; std::wostringstream output;
std::vector<std::wstring> args; std::vector<std::wstring> args;
int arg_number = 1; int arg_number = 1;
@ -750,7 +752,7 @@ void translate_string(const std::wstring &s, const std::wstring &textdomain,
if (arg_number >= 10) { if (arg_number >= 10) {
errorstream << "Ignoring too many arguments to translation" << std::endl; errorstream << "Ignoring too many arguments to translation" << std::endl;
std::wstring arg; std::wstring arg;
translate_all(s, i, arg); translate_all(s, i, translations, arg);
args.push_back(arg); args.push_back(arg);
continue; continue;
} }
@ -758,7 +760,7 @@ void translate_string(const std::wstring &s, const std::wstring &textdomain,
output << arg_number; output << arg_number;
++arg_number; ++arg_number;
std::wstring arg; std::wstring arg;
translate_all(s, i, arg); translate_all(s, i, translations, arg);
args.push_back(arg); args.push_back(arg);
} else { } else {
// This is an escape sequence *inside* the template string to translate itself. // This is an escape sequence *inside* the template string to translate itself.
@ -767,8 +769,13 @@ void translate_string(const std::wstring &s, const std::wstring &textdomain,
} }
} }
std::wstring toutput;
// Translate the template. // Translate the template.
std::wstring toutput = g_translations->getTranslation(textdomain, output.str()); if (translations != nullptr)
toutput = translations->getTranslation(
textdomain, output.str());
else
toutput = output.str();
// Put back the arguments in the translated template. // Put back the arguments in the translated template.
std::wostringstream result; std::wostringstream result;
@ -802,7 +809,9 @@ void translate_string(const std::wstring &s, const std::wstring &textdomain,
res = result.str(); res = result.str();
} }
void translate_all(const std::wstring &s, size_t &i, std::wstring &res) { void translate_all(const std::wstring &s, size_t &i,
Translations *translations, std::wstring &res)
{
std::wostringstream output; std::wostringstream output;
while (i < s.length()) { while (i < s.length()) {
// Not an escape sequence: just add the character. // Not an escape sequence: just add the character.
@ -851,7 +860,7 @@ void translate_all(const std::wstring &s, size_t &i, std::wstring &res) {
if (parts.size() > 1) if (parts.size() > 1)
textdomain = parts[1]; textdomain = parts[1];
std::wstring translated; std::wstring translated;
translate_string(s, textdomain, i, translated); translate_string(s, translations, textdomain, i, translated);
output << translated; output << translated;
} else { } else {
// Another escape sequence, such as colors. Preserve it. // Another escape sequence, such as colors. Preserve it.
@ -862,9 +871,21 @@ void translate_all(const std::wstring &s, size_t &i, std::wstring &res) {
res = output.str(); res = output.str();
} }
std::wstring translate_string(const std::wstring &s) { // Translate string server side
std::wstring translate_string(const std::wstring &s, Translations *translations)
{
size_t i = 0; size_t i = 0;
std::wstring res; std::wstring res;
translate_all(s, i, res); translate_all(s, i, translations, res);
return res; return res;
} }
// Translate string client side
std::wstring translate_string(const std::wstring &s)
{
#ifdef SERVER
return translate_string(s, nullptr);
#else
return translate_string(s, g_client_translations);
#endif
}

@ -31,6 +31,8 @@ with this program; if not, write to the Free Software Foundation, Inc.,
#include <cctype> #include <cctype>
#include <unordered_map> #include <unordered_map>
class Translations;
#define STRINGIFY(x) #x #define STRINGIFY(x) #x
#define TOSTRING(x) STRINGIFY(x) #define TOSTRING(x) STRINGIFY(x)
@ -650,6 +652,8 @@ std::vector<std::basic_string<T> > split(const std::basic_string<T> &s, T delim)
return tokens; return tokens;
} }
std::wstring translate_string(const std::wstring &s, Translations *translations);
std::wstring translate_string(const std::wstring &s); std::wstring translate_string(const std::wstring &s);
inline std::wstring unescape_translate(const std::wstring &s) { inline std::wstring unescape_translate(const std::wstring &s) {