Add utf-8 conversion utilities and re-add intlGUIEditBox

build/android/Makefile

@@ -131,6 +131,13 @@ FREETYPE_TIMESTAMP = $(FREETYPE_DIR)timestamp
 FREETYPE_TIMESTAMP_INT = $(ROOT)/deps/freetype_timestamp
 FREETYPE_URL_GIT = https://github.com/cdave1/freetype2-android
 
+ICONV_VERSION = 1.14
+ICONV_DIR = $(ROOT)/deps/libiconv/
+ICONV_LIB = $(ICONV_DIR)/iconv.so
+ICONV_TIMESTAMP = $(ICONV_DIR)timestamp
+ICONV_TIMESTAMP_INT = $(ROOT)/deps/iconv_timestamp
+ICONV_URL_HTTP = http://ftp.gnu.org/pub/gnu/libiconv/libiconv-$(ICONV_VERSION).tar.gz
+
 SQLITE3_FOLDER = sqlite-amalgamation-3080704
 SQLITE3_URL = http://www.sqlite.org/2014/$(SQLITE3_FOLDER).zip
 
@@ -431,6 +438,64 @@ $(FREETYPE_LIB) : $(FREETYPE_TIMESTAMP)
 clean_freetype :
 	$(RM) -rf ${FREETYPE_DIR}
 
+$(ICONV_TIMESTAMP) : iconv_download
+	@LAST_MODIF=$$(find ${ICONV_DIR} -type f -printf '%T@ %p\n' | sort -n | tail -1 | cut -f2- -d" "); \
+	if [ $$(basename $$LAST_MODIF) != "timestamp" ] ; then                     \
+		touch ${ICONV_TIMESTAMP};                                           \
+	fi
+
+iconv_download :
+	@if [ ! -d ${ICONV_DIR} ] ; then                                        \
+		echo "iconv sources missing, downloading...";                       \
+		mkdir -p ${ROOT}/deps;                                                 \
+		cd ${ROOT}/deps;                                                               \
+		wget ${ICONV_URL_HTTP} || exit 1;                                    \
+		tar -xzf libiconv-${ICONV_VERSION}.tar.gz || exit 1;                     \
+		rm libiconv-${ICONV_VERSION}.tar.gz;                                     \
+		ln -s libiconv-${ICONV_VERSION} libiconv;                                      \
+		cd ${ICONV_DIR};                                                                   \
+		patch -p1 < ${ROOT}/libiconv_android.patch;                                                     \
+		patch -p1 < ${ROOT}/libiconv_android_mk.patch;                                                     \
+		patch -p1 < ${ROOT}/libiconv_stdio.patch;                                                     \
+		cd jni;                                                                       \
+		ln -s .. src;                                                                         \
+	fi
+
+iconv : $(ICONV_LIB)
+
+$(ICONV_LIB) : $(ICONV_TIMESTAMP)
+	@REFRESH=0;                                                                \
+	if [ ! -e ${ICONV_TIMESTAMP_INT} ] ; then                               \
+		REFRESH=1;                                                             \
+	fi;                                                                        \
+	if [ ! -e ${ICONV_LIB} ] ; then                                         \
+		REFRESH=1;                                                             \
+	fi;                                                                        \
+	if [ ${ICONV_TIMESTAMP} -nt ${ICONV_TIMESTAMP_INT} ] ; then          \
+		REFRESH=1;                                                             \
+	fi;                                                                        \
+	if [ $$REFRESH -ne 0 ] ; then                                              \
+	mkdir -p ${ICONV_DIR};                                                  \
+	export PATH=$$PATH:${SDKFOLDER}/platform-tools:${ANDROID_NDK};             \
+	echo "changed timestamp for iconv detected building...";                \
+	cd ${ICONV_DIR};                                            \
+	./configure;                                            \
+	make;                                            \
+	export NDK_PROJECT_PATH=${ICONV_DIR};                                \
+	ndk-build NDEBUG=${NDEBUG} NDK_MODULE_PATH=${NDK_MODULE_PATH}              \
+	APP_PLATFORM=${APP_PLATFORM} APP_ABI=${TARGET_ABI}                         \
+	TARGET_CFLAGS+="${TARGET_CFLAGS_ADDON}"                                    \
+	TARGET_LDFLAGS+="${TARGET_LDFLAGS_ADDON}"                                  \
+	TARGET_CXXFLAGS+="${TARGET_CXXFLAGS_ADDON}" || exit 1;                     \
+	touch ${ICONV_TIMESTAMP};                                               \
+	touch ${ICONV_TIMESTAMP_INT};                                           \
+	else                                                                       \
+		echo "nothing to be done for iconv";                                \
+	fi
+
+clean_iconv :
+	$(RM) -rf ${ICONV_DIR}
+
 #Note: Texturehack patch is required for gpu's not supporting color format
 #      correctly. Known bad GPU:
 #      -geforce on emulator
@@ -736,7 +801,7 @@ assets : $(ASSETS_TIMESTAMP)
 clean_assets :
 	@$(RM) -r assets
 
-apk: $(PATHCFGFILE) assets $(IRRLICHT_LIB) $(CURL_LIB) $(GMP_LIB) $(LEVELDB_TARGET)       \
+apk: $(PATHCFGFILE) assets $(ICONV_LIB) $(IRRLICHT_LIB) $(CURL_LIB) $(GMP_LIB) $(LEVELDB_TARGET)       \
 	$(OPENAL_LIB) $(OGG_LIB) prep_srcdir $(ROOT)/jni/src/android_version.h     \
 	sqlite3_download
 	@export NDEBUG=$$NDEBUG; $(MAKE) manifest;                                 \
@@ -778,7 +843,7 @@ envpaths :
 
 clean_all :
 	@$(MAKE) clean_apk;                                                        \
-	$(MAKE) clean_assets clean_irrlicht clean_leveldb clean_curl clean_openssl \
+	$(MAKE) clean_assets clean_iconv clean_irrlicht clean_leveldb clean_curl clean_openssl \
 	clean_openal clean_ogg clean_gmp clean_manifest;                           \
 	sleep 1;                                                                   \
 	$(RM) -r gen libs obj deps bin Debug and_env

build/android/jni/Android.mk

@@ -24,6 +24,11 @@ LOCAL_MODULE := freetype
 LOCAL_SRC_FILES := deps/freetype2-android/Android/obj/local/$(TARGET_ARCH_ABI)/libfreetype2-static.a
 include $(PREBUILT_STATIC_LIBRARY)
 
+include $(CLEAR_VARS)
+LOCAL_MODULE := iconv
+LOCAL_SRC_FILES := deps/libiconv/obj/local/$(TARGET_ARCH_ABI)/libiconv.a
+include $(PREBUILT_STATIC_LIBRARY)
+
 include $(CLEAR_VARS)
 LOCAL_MODULE := openal
 LOCAL_SRC_FILES := deps/openal-soft/libs/$(TARGET_LIBDIR)/libopenal.so
@@ -97,6 +102,7 @@ LOCAL_C_INCLUDES :=                               \
 		jni/src/json                              \
 		jni/src/cguittfont                        \
 		deps/irrlicht/include                     \
+		deps/libiconv/include                     \
 		deps/freetype2-android/include            \
 		deps/curl/include                         \
 		deps/openal-soft/jni/OpenAL/include       \
@@ -153,6 +159,7 @@ LOCAL_SRC_FILES :=                                \
 		jni/src/httpfetch.cpp                     \
 		jni/src/hud.cpp                           \
 		jni/src/imagefilters.cpp                  \
+		jni/src/intlGUIEditBox.cpp                \
 		jni/src/inventory.cpp                     \
 		jni/src/inventorymanager.cpp              \
 		jni/src/itemdef.cpp                       \
@@ -346,7 +353,7 @@ LOCAL_SRC_FILES +=                                \
 LOCAL_SRC_FILES += jni/src/json/jsoncpp.cpp
 
 LOCAL_SHARED_LIBRARIES := openal ogg vorbis gmp
-LOCAL_STATIC_LIBRARIES := Irrlicht freetype curl ssl crypto android_native_app_glue $(PROFILER_LIBS)
+LOCAL_STATIC_LIBRARIES := Irrlicht iconv freetype curl ssl crypto android_native_app_glue $(PROFILER_LIBS)
 
 ifeq ($(HAVE_LEVELDB), 1)
 	LOCAL_STATIC_LIBRARIES += LevelDB

build/android/libiconv_android.patch

@@ -0,0 +1,39 @@
+--- a/libcharset/lib/localcharset.c	2015-06-10 11:55:25.933870724 +0200
++++ b/libcharset/lib/localcharset.c	2015-06-10 11:55:39.578063493 +0200
+@@ -47,7 +47,7 @@
+ 
+ #if !defined WIN32_NATIVE
+ # include <unistd.h>
+-# if HAVE_LANGINFO_CODESET
++# if HAVE_LANGINFO_CODESET && !(defined __ANDROID__)
+ #  include <langinfo.h>
+ # else
+ #  if 0 /* see comment below */
+@@ -124,7 +124,7 @@ get_charset_aliases (void)
+   cp = charset_aliases;
+   if (cp == NULL)
+     {
+-#if !(defined DARWIN7 || defined VMS || defined WIN32_NATIVE || defined __CYGWIN__)
++#if !(defined DARWIN7 || defined VMS || defined WIN32_NATIVE || defined __CYGWIN__ || defined __ANDROID__)
+       const char *dir;
+       const char *base = "charset.alias";
+       char *file_name;
+@@ -338,6 +338,9 @@ get_charset_aliases (void)
+            "CP54936" "\0" "GB18030" "\0"
+            "CP65001" "\0" "UTF-8" "\0";
+ # endif
++# if defined __ANDROID__
++      cp = "*" "\0" "UTF-8" "\0";
++# endif
+ #endif
+ 
+       charset_aliases = cp;
+@@ -361,7 +364,7 @@ locale_charset (void)
+   const char *codeset;
+   const char *aliases;
+ 
+-#if !(defined WIN32_NATIVE || defined OS2)
++#if !(defined WIN32_NATIVE || defined OS2 || defined __ANDROID__)
+ 
+ # if HAVE_LANGINFO_CODESET
+ 

build/android/libiconv_android_mk.patch

@@ -0,0 +1,51 @@
+From fe27aae178d65b06d5f4104158343b0d2d33e3f0 Mon Sep 17 00:00:00 2001
+From: Pierre Zurek <pierrezurek@gmail.com>
+Date: Sat, 2 Apr 2011 23:11:57 +0200
+Subject: [PATCH] Added Android.mk.
+
+This makefile first executes the configure script, that will
+generate the config.h files necessary to build iconv.
+---
+ Android.mk | 29 +++++++++++++++++++++++++++++
+ 1 file changed, 29 insertions(+)
+ create mode 100644 Android.mk
+
+diff --git a/jni/Android.mk b/jni/Android.mk
+new file mode 100644
+index 0000000..799b22d
+--- /dev/null
++++ b/jni/Android.mk
+@@ -0,0 +1,32 @@
++LOCAL_PATH := $(call my-dir)
++include $(CLEAR_VARS)
++
++LOCAL_ARM_MODE := arm
++
++LOCAL_SRC_FILES := src/lib/iconv.c \
++                   src/libcharset/lib/localcharset.c \
++                   src/lib/relocatable.c
++
++LOCAL_C_INCLUDES += $(LOCAL_PATH)/src/include \
++                    $(LOCAL_PATH)/src/libcharset \
++                    $(LOCAL_PATH)/src/libcharset/include
++
++LOCAL_CFLAGS := \
++    -DLIBDIR="\"c\"" \
++    -D_ANDROID \
++    -DBUILDING_LIBCHARSET \
++    -DBUILDING_LIBICONV \
++    -DBUILDING_LIBICONV \
++    -DIN_LIBRARY
++
++LOCAL_MODULE:= iconv
++
++$(info Configuring iconv...)
++COMMAND := $(shell \
++           export PATH=$(TOOLCHAIN_INSTALL_DIR)/bin:$$PATH; \
++           cd $(LOCAL_PATH); \
++           make distclean; \
++           ./configure --host="arm-linux-androideabi")
++$(info iconv configured.)
++
++include $(BUILD_STATIC_LIBRARY)
++

build/android/libiconv_stdio.patch

@@ -0,0 +1,10 @@
+--- a/srclib/stdio.in.h	2011-08-07 15:42:06.000000000 +0200
++++ b/srclib/stdio.in.h	2015-06-10 09:27:58.129056262 +0200
+@@ -695,7 +696,8 @@ _GL_CXXALIASWARN (gets);
+ /* It is very rare that the developer ever has full control of stdin,
+    so any use of gets warrants an unconditional warning.  Assume it is
+    always declared, since it is required by C89.  */
+-_GL_WARN_ON_USE (gets, "gets is a security hole - use fgets instead");
++/*_GL_WARN_ON_USE (gets, "gets is a security hole - use fgets instead");*/
++#define gets(a) fgets( a, sizeof(*(a)), stdin)
+ #endif

src/CMakeLists.txt

@@ -417,6 +417,7 @@ set(client_SRCS
 	guiVolumeChange.cpp
 	hud.cpp
 	imagefilters.cpp
+	intlGUIEditBox.cpp
 	keycode.cpp
 	localplayer.cpp
 	main.cpp

src/intlGUIEditBox.h

@@ -0,0 +1,178 @@
+// Copyright (C) 2002-2013 Nikolaus Gebhardt
+// This file is part of the "Irrlicht Engine".
+// For conditions of distribution and use, see copyright notice in irrlicht.h
+
+#ifndef __C_INTL_GUI_EDIT_BOX_H_INCLUDED__
+#define __C_INTL_GUI_EDIT_BOX_H_INCLUDED__
+
+#include "IrrCompileConfig.h"
+//#ifdef _IRR_COMPILE_WITH_GUI_
+
+#include "IGUIEditBox.h"
+#include "irrArray.h"
+#include "IOSOperator.h"
+
+namespace irr
+{
+namespace gui
+{
+	class intlGUIEditBox : public IGUIEditBox
+	{
+	public:
+
+		//! constructor
+		intlGUIEditBox(const wchar_t* text, bool border, IGUIEnvironment* environment,
+			IGUIElement* parent, s32 id, const core::rect<s32>& rectangle);
+
+		//! destructor
+		virtual ~intlGUIEditBox();
+
+		//! Sets another skin independent font.
+		virtual void setOverrideFont(IGUIFont* font=0);
+
+		//! Gets the override font (if any)
+		/** \return The override font (may be 0) */
+		virtual IGUIFont* getOverrideFont() const;
+
+		//! Get the font which is used right now for drawing
+		/** Currently this is the override font when one is set and the
+		font of the active skin otherwise */
+		virtual IGUIFont* getActiveFont() const;
+
+		//! Sets another color for the text.
+		virtual void setOverrideColor(video::SColor color);
+
+		//! Gets the override color
+		virtual video::SColor getOverrideColor() const;
+
+		//! Sets if the text should use the overide color or the
+		//! color in the gui skin.
+		virtual void enableOverrideColor(bool enable);
+
+		//! Checks if an override color is enabled
+		/** \return true if the override color is enabled, false otherwise */
+		virtual bool isOverrideColorEnabled(void) const;
+
+		//! Sets whether to draw the background
+		virtual void setDrawBackground(bool draw);
+
+		//! Turns the border on or off
+		virtual void setDrawBorder(bool border);
+
+		//! Enables or disables word wrap for using the edit box as multiline text editor.
+		virtual void setWordWrap(bool enable);
+
+		//! Checks if word wrap is enabled
+		//! \return true if word wrap is enabled, false otherwise
+		virtual bool isWordWrapEnabled() const;
+
+		//! Enables or disables newlines.
+		/** \param enable: If set to true, the EGET_EDITBOX_ENTER event will not be fired,
+		instead a newline character will be inserted. */
+		virtual void setMultiLine(bool enable);
+
+		//! Checks if multi line editing is enabled
+		//! \return true if mult-line is enabled, false otherwise
+		virtual bool isMultiLineEnabled() const;
+
+		//! Enables or disables automatic scrolling with cursor position
+		//! \param enable: If set to true, the text will move around with the cursor position
+		virtual void setAutoScroll(bool enable);
+
+		//! Checks to see if automatic scrolling is enabled
+		//! \return true if automatic scrolling is enabled, false if not
+		virtual bool isAutoScrollEnabled() const;
+
+		//! Gets the size area of the text in the edit box
+		//! \return Returns the size in pixels of the text
+		virtual core::dimension2du getTextDimension();
+
+		//! Sets text justification
+		virtual void setTextAlignment(EGUI_ALIGNMENT horizontal, EGUI_ALIGNMENT vertical);
+
+		//! called if an event happened.
+		virtual bool OnEvent(const SEvent& event);
+
+		//! draws the element and its children
+		virtual void draw();
+
+		//! Sets the new caption of this element.
+		virtual void setText(const wchar_t* text);
+
+		//! Sets the maximum amount of characters which may be entered in the box.
+		//! \param max: Maximum amount of characters. If 0, the character amount is
+		//! infinity.
+		virtual void setMax(u32 max);
+
+		//! Returns maximum amount of characters, previously set by setMax();
+		virtual u32 getMax() const;
+
+		//! Sets whether the edit box is a password box. Setting this to true will
+		/** disable MultiLine, WordWrap and the ability to copy with ctrl+c or ctrl+x
+		\param passwordBox: true to enable password, false to disable
+		\param passwordChar: the character that is displayed instead of letters */
+		virtual void setPasswordBox(bool passwordBox, wchar_t passwordChar = L'*');
+
+		//! Returns true if the edit box is currently a password box.
+		virtual bool isPasswordBox() const;
+
+		//! Updates the absolute position, splits text if required
+		virtual void updateAbsolutePosition();
+
+		//! Writes attributes of the element.
+		virtual void serializeAttributes(io::IAttributes* out, io::SAttributeReadWriteOptions* options) const;
+
+		//! Reads attributes of the element
+		virtual void deserializeAttributes(io::IAttributes* in, io::SAttributeReadWriteOptions* options);
+
+	protected:
+		//! Breaks the single text line.
+		void breakText();
+		//! sets the area of the given line
+		void setTextRect(s32 line);
+		//! returns the line number that the cursor is on
+		s32 getLineFromPos(s32 pos);
+		//! adds a letter to the edit box
+		void inputChar(wchar_t c);
+		//! calculates the current scroll position
+		void calculateScrollPos();
+		//! send some gui event to parent
+		void sendGuiEvent(EGUI_EVENT_TYPE type);
+		//! set text markers
+		void setTextMarkers(s32 begin, s32 end);
+
+		bool processKey(const SEvent& event);
+		bool processMouse(const SEvent& event);
+		s32 getCursorPos(s32 x, s32 y);
+
+		bool MouseMarking;
+		bool Border;
+		bool OverrideColorEnabled;
+		s32 MarkBegin;
+		s32 MarkEnd;
+
+		video::SColor OverrideColor;
+		gui::IGUIFont *OverrideFont, *LastBreakFont;
+		IOSOperator* Operator;
+
+		u32 BlinkStartTime;
+		s32 CursorPos;
+		s32 HScrollPos, VScrollPos; // scroll position in characters
+		u32 Max;
+
+		bool WordWrap, MultiLine, AutoScroll, PasswordBox;
+		wchar_t PasswordChar;
+		EGUI_ALIGNMENT HAlign, VAlign;
+
+		core::array< core::stringw > BrokenText;
+		core::array< s32 > BrokenTextPositions;
+
+		core::rect<s32> CurrentTextRect, FrameRect; // temporary values
+	};
+
+
+} // end namespace gui
+} // end namespace irr
+
+//#endif // _IRR_COMPILE_WITH_GUI_
+#endif // __C_GUI_EDIT_BOX_H_INCLUDED__

src/util/string.cpp

@@ -29,9 +29,103 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include <iomanip>
 #include <map>
 
+#ifndef _WIN32
+#include <iconv.h>
+#else
+#define _WIN32_WINNT 0x0501
+#include <windows.h>
+#endif
+
 static bool parseHexColorString(const std::string &value, video::SColor &color);
 static bool parseNamedColorString(const std::string &value, video::SColor &color);
 
+#ifndef _WIN32
+size_t convert(const char *to, const char *from, char *outbuf,
+		size_t outbuf_size, char *inbuf, size_t inbuf_size)
+{
+	iconv_t cd = iconv_open(to, from);
+
+#if defined(__FreeBSD__) || defined(__FreeBSD)
+	const char *inbuf_ptr = inbuf;
+#else
+	char *inbuf_ptr = inbuf;
+#endif
+
+	char *outbuf_ptr = outbuf;
+
+	size_t *inbuf_left_ptr = &inbuf_size;
+	size_t *outbuf_left_ptr = &outbuf_size;
+
+	while (inbuf_size > 0)
+		iconv(cd, &inbuf_ptr, inbuf_left_ptr, &outbuf_ptr, outbuf_left_ptr);
+
+	iconv_close(cd);
+	return 0;
+}
+
+std::wstring utf8_to_wide(const std::string &input)
+{
+	size_t inbuf_size = input.length() + 1;
+	// maximum possible size, every character is sizeof(wchar_t) bytes
+	size_t outbuf_size = (input.length() + 1) * sizeof(wchar_t);
+
+	char *inbuf = new char[inbuf_size];
+	memcpy(inbuf, input.c_str(), inbuf_size);
+	char *outbuf = new char[outbuf_size];
+	memset(outbuf, 0, outbuf_size);
+
+	convert("WCHAR_T", "UTF-8", outbuf, outbuf_size, inbuf, inbuf_size);
+	std::wstring out((wchar_t*)outbuf);
+
+	delete[] inbuf;
+	delete[] outbuf;
+
+	return out;
+}
+
+std::string wide_to_utf8(const std::wstring &input)
+{
+	size_t inbuf_size = (input.length() + 1) * sizeof(wchar_t);
+	// maximum possible size: utf-8 encodes codepoints using 1 up to 6 bytes
+	size_t outbuf_size = (input.length() + 1) * 6;
+
+	char *inbuf = new char[inbuf_size];
+	memcpy(inbuf, input.c_str(), inbuf_size);
+	char *outbuf = new char[outbuf_size];
+	memset(outbuf, 0, outbuf_size);
+
+	convert("UTF-8", "WCHAR_T", outbuf, outbuf_size, inbuf, inbuf_size);
+	std::string out(outbuf);
+
+	delete[] inbuf;
+	delete[] outbuf;
+
+	return out;
+}
+#else
+std::wstring utf8_to_wide(const std::string &input)
+{
+	size_t outbuf_size = input.size() + 1;
+	wchar_t *outbuf = new wchar_t[outbuf_size];
+	memset(outbuf, 0, outbuf_size * sizeof(wchar_t));
+	MultiByteToWideChar(CP_UTF8, 0, input.c_str(), input.size(), outbuf, outbuf_size);
+	std::wstring out(outbuf);
+	delete[] outbuf;
+	return out;
+}
+
+std::string wide_to_utf8(const std::wstring &input)
+{
+	size_t outbuf_size = (input.size() + 1) * 6;
+	char *outbuf = new char[outbuf_size];
+	memset(outbuf, 0, outbuf_size);
+	WideCharToMultiByte(CP_UTF8, 0, input.c_str(), input.size(), outbuf, outbuf_size, NULL, NULL);
+	std::string out(outbuf);
+	delete[] outbuf;
+	return out;
+}
+#endif
+
 
 // You must free the returned string!
 // The returned string is allocated using new

src/util/string.h

@@ -39,10 +39,18 @@ struct FlagDesc {
 	u32 flag;
 };
 
+// try not to convert between wide/utf8 encodings; this can result in data loss
+// try to only convert between them when you need to input/output stuff via Irrlicht
+std::wstring utf8_to_wide(const std::string &input);
+std::string wide_to_utf8(const std::wstring &input);
+
+// NEVER use those two functions unless you have a VERY GOOD reason to
+// they just convert between wide and multibyte encoding
+// multibyte encoding depends on current locale, this is no good, especially on Windows
+
 // You must free the returned string!
 // The returned string is allocated using new
 wchar_t *narrow_to_wide_c(const char *str);
-
 std::wstring narrow_to_wide(const std::string &mbs);
 std::string wide_to_narrow(const std::wstring &wcs);