From c3968006298bcce46e7659d11c6cbee56bbed4f0 Mon Sep 17 00:00:00 2001 From: Maksim Date: Mon, 13 Jan 2020 07:10:15 +0100 Subject: [PATCH] Android: fix cyrillic characters, update iconv lib (#9117) --- build/android/Makefile | 5 +- build/android/patches/libiconv_android.patch | 39 ------ build/android/patches/libiconv_stdio.patch | 13 -- src/util/string.cpp | 123 +++---------------- 4 files changed, 19 insertions(+), 161 deletions(-) delete mode 100644 build/android/patches/libiconv_android.patch delete mode 100644 build/android/patches/libiconv_stdio.patch diff --git a/build/android/Makefile b/build/android/Makefile index c5a21c290..9ec237a75 100644 --- a/build/android/Makefile +++ b/build/android/Makefile @@ -112,7 +112,7 @@ FREETYPE_TIMESTAMP = $(FREETYPE_DIR)timestamp FREETYPE_TIMESTAMP_INT = $(ANDR_ROOT)/deps/freetype_timestamp FREETYPE_URL_GIT = https://github.com/cdave1/freetype2-android -ICONV_VERSION = 1.14 +ICONV_VERSION = 1.16 ICONV_DIR = $(ANDR_ROOT)/deps/libiconv/ ICONV_LIB = $(ICONV_DIR)/lib/.libs/libiconv.so ICONV_TIMESTAMP = $(ICONV_DIR)timestamp @@ -445,9 +445,6 @@ iconv_download : tar -xzf libiconv-${ICONV_VERSION}.tar.gz || exit 1; \ rm libiconv-${ICONV_VERSION}.tar.gz; \ ln -s libiconv-${ICONV_VERSION} libiconv; \ - cd ${ICONV_DIR}; \ - patch -p1 < ${ANDR_ROOT}/patches/libiconv_android.patch; \ - patch -p1 < ${ANDR_ROOT}/patches/libiconv_stdio.patch; \ fi iconv : $(ICONV_LIB) diff --git a/build/android/patches/libiconv_android.patch b/build/android/patches/libiconv_android.patch deleted file mode 100644 index 4eca0a4ef..000000000 --- a/build/android/patches/libiconv_android.patch +++ /dev/null @@ -1,39 +0,0 @@ ---- a/libcharset/lib/localcharset.c 2015-06-10 11:55:25.933870724 +0200 -+++ b/libcharset/lib/localcharset.c 2015-06-10 11:55:39.578063493 +0200 -@@ -47,7 +47,7 @@ - - #if !defined WIN32_NATIVE - # include --# if HAVE_LANGINFO_CODESET -+# if HAVE_LANGINFO_CODESET && !(defined __ANDROID__) - # include - # else - # if 0 /* see comment below */ -@@ -124,7 +124,7 @@ get_charset_aliases (void) - cp = charset_aliases; - if (cp == NULL) - { --#if !(defined DARWIN7 || defined VMS || defined WIN32_NATIVE || defined __CYGWIN__) -+#if !(defined DARWIN7 || defined VMS || defined WIN32_NATIVE || defined __CYGWIN__ || defined __ANDROID__) - const char *dir; - const char *base = "charset.alias"; - char *file_name; -@@ -338,6 +338,9 @@ get_charset_aliases (void) - "CP54936" "\0" "GB18030" "\0" - "CP65001" "\0" "UTF-8" "\0"; - # endif -+# if defined __ANDROID__ -+ cp = "*" "\0" "UTF-8" "\0"; -+# endif - #endif - - charset_aliases = cp; -@@ -361,7 +364,7 @@ locale_charset (void) - const char *codeset; - const char *aliases; - --#if !(defined WIN32_NATIVE || defined OS2) -+#if !(defined WIN32_NATIVE || defined OS2 || defined __ANDROID__) - - # if HAVE_LANGINFO_CODESET - diff --git a/build/android/patches/libiconv_stdio.patch b/build/android/patches/libiconv_stdio.patch deleted file mode 100644 index 9fa50f79a..000000000 --- a/build/android/patches/libiconv_stdio.patch +++ /dev/null @@ -1,13 +0,0 @@ ---- a/srclib/stdio.in.h 2011-08-07 15:42:06.000000000 +0200 -+++ b/srclib/stdio.in.h 2015-06-10 09:27:58.129056262 +0200 -@@ -695,8 +696,9 @@ _GL_CXXALIASWARN (gets); - /* It is very rare that the developer ever has full control of stdin, - so any use of gets warrants an unconditional warning. Assume it is - always declared, since it is required by C89. */ --_GL_WARN_ON_USE (gets, "gets is a security hole - use fgets instead"); -+/*_GL_WARN_ON_USE (gets, "gets is a security hole - use fgets instead");*/ -+#define gets(a) fgets( a, sizeof(*(a)), stdin) - #endif - - -#if @GNULIB_OBSTACK_PRINTF@ || @GNULIB_OBSTACK_PRINTF_POSIX@ diff --git a/src/util/string.cpp b/src/util/string.cpp index caaef9b30..2134fbd15 100644 --- a/src/util/string.cpp +++ b/src/util/string.cpp @@ -79,6 +79,13 @@ bool convert(const char *to, const char *from, char *outbuf, return true; } +#ifdef __ANDROID__ +// Android need manual caring to support the full character set possible with wchar_t +const char *DEFAULT_ENCODING = "UTF-32LE"; +#else +const char *DEFAULT_ENCODING = "WCHAR_T"; +#endif + std::wstring utf8_to_wide(const std::string &input) { size_t inbuf_size = input.length() + 1; @@ -90,7 +97,12 @@ std::wstring utf8_to_wide(const std::string &input) char *outbuf = new char[outbuf_size]; memset(outbuf, 0, outbuf_size); - if (!convert("WCHAR_T", "UTF-8", outbuf, outbuf_size, inbuf, inbuf_size)) { +#ifdef __ANDROID__ + // Android need manual caring to support the full character set possible with wchar_t + SANITY_CHECK(sizeof(wchar_t) == 4); +#endif + + if (!convert(DEFAULT_ENCODING, "UTF-8", outbuf, outbuf_size, inbuf, inbuf_size)) { infostream << "Couldn't convert UTF-8 string 0x" << hex_encode(input) << " into wstring" << std::endl; delete[] inbuf; @@ -105,13 +117,6 @@ std::wstring utf8_to_wide(const std::string &input) return out; } -#ifdef __ANDROID__ -// TODO: this is an ugly fix for wide_to_utf8 somehow not working on android -std::string wide_to_utf8(const std::wstring &input) -{ - return wide_to_narrow(input); -} -#else std::string wide_to_utf8(const std::wstring &input) { size_t inbuf_size = (input.length() + 1) * sizeof(wchar_t); @@ -123,7 +128,7 @@ std::string wide_to_utf8(const std::wstring &input) char *outbuf = new char[outbuf_size]; memset(outbuf, 0, outbuf_size); - if (!convert("UTF-8", "WCHAR_T", outbuf, outbuf_size, inbuf, inbuf_size)) { + if (!convert("UTF-8", DEFAULT_ENCODING, outbuf, outbuf_size, inbuf, inbuf_size)) { infostream << "Couldn't convert wstring 0x" << hex_encode(inbuf, inbuf_size) << " into UTF-8 string" << std::endl; delete[] inbuf; @@ -138,7 +143,6 @@ std::string wide_to_utf8(const std::wstring &input) return out; } -#endif #else // _WIN32 std::wstring utf8_to_wide(const std::string &input) @@ -183,7 +187,7 @@ wchar_t *utf8_to_wide_c(const char *str) // The returned string is allocated using new wchar_t *narrow_to_wide_c(const char *str) { - wchar_t *nstr = NULL; + wchar_t *nstr = nullptr; #if defined(_WIN32) int nResult = MultiByteToWideChar(CP_UTF8, 0, (LPCSTR) str, -1, 0, 0); if (nResult == 0) { @@ -204,67 +208,8 @@ wchar_t *narrow_to_wide_c(const char *str) return nstr; } - -#ifdef __ANDROID__ - -const wchar_t* wide_chars = - L" !\"#$%&'()*+,-./0123456789:;<=>?@" - L"ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`" - L"abcdefghijklmnopqrstuvwxyz{|}~"; - -int wctomb(char *s, wchar_t wc) -{ - for (unsigned int j = 0; j < (sizeof(wide_chars)/sizeof(wchar_t));j++) { - if (wc == wide_chars[j]) { - *s = (char) (j+32); - return 1; - } - else if (wc == L'\n') { - *s = '\n'; - return 1; - } - } - return -1; -} - -int mbtowc(wchar_t *pwc, const char *s, size_t n) -{ - std::wstring intermediate = narrow_to_wide(s); - - if (intermediate.length() > 0) { - *pwc = intermediate[0]; - return 1; - } - else { - return -1; - } -} - std::wstring narrow_to_wide(const std::string &mbs) { size_t wcl = mbs.size(); - - std::wstring retval = L""; - - for (unsigned int i = 0; i < wcl; i++) { - if (((unsigned char) mbs[i] >31) && - ((unsigned char) mbs[i] < 127)) { - - retval += wide_chars[(unsigned char) mbs[i] -32]; - } - //handle newline - else if (mbs[i] == '\n') { - retval += L'\n'; - } - } - - return retval; -} - -#else // not Android - -std::wstring narrow_to_wide(const std::string &mbs) -{ - size_t wcl = mbs.size(); Buffer wcs(wcl + 1); size_t len = mbstowcs(*wcs, mbs.c_str(), wcl); if (len == (size_t)(-1)) @@ -273,37 +218,6 @@ std::wstring narrow_to_wide(const std::string &mbs) return *wcs; } -#endif - -#ifdef __ANDROID__ - -std::string wide_to_narrow(const std::wstring &wcs) { - size_t mbl = wcs.size()*4; - - std::string retval = ""; - for (unsigned int i = 0; i < wcs.size(); i++) { - wchar_t char1 = (wchar_t) wcs[i]; - - if (char1 == L'\n') { - retval += '\n'; - continue; - } - - for (unsigned int j = 0; j < wcslen(wide_chars);j++) { - wchar_t char2 = (wchar_t) wide_chars[j]; - - if (char1 == char2) { - char toadd = (j+32); - retval += toadd; - break; - } - } - } - - return retval; -} - -#else // not Android std::string wide_to_narrow(const std::wstring &wcs) { @@ -317,7 +231,6 @@ std::string wide_to_narrow(const std::wstring &wcs) return *mbs; } -#endif std::string urlencode(const std::string &str) { @@ -361,10 +274,10 @@ u32 readFlagString(std::string str, const FlagDesc *flagdesc, u32 *flagmask) u32 mask = 0; char *s = &str[0]; char *flagstr; - char *strpos = NULL; + char *strpos = nullptr; while ((flagstr = strtok_r(s, ",", &strpos))) { - s = NULL; + s = nullptr; while (*flagstr == ' ' || *flagstr == '\t') flagstr++; @@ -436,7 +349,7 @@ char *mystrtok_r(char *s, const char *sep, char **lasts) s++; if (!*s) - return NULL; + return nullptr; t = s; while (*t) {