Use UTF-8 for Linux

CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.15 FATAL_ERROR)
+cmake_minimum_required(VERSION 3.20 FATAL_ERROR)
 project(qsp VERSION 5.9.0)
 
 set(QSP_GAMEMIN_VER "5.9.0")
@@ -11,6 +11,10 @@ elseif(APPLE OR NOT UNIX)
 	message(FATAL_ERROR "Only Windows and Linux targets are supported")
 endif()
 
+if (LINUX)
+	find_package(Iconv REQUIRED)
+endif()
+
 list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake/Modules)
 
 set(CMAKE_INCLUDE_CURRENT_DIR True)
@@ -79,35 +83,47 @@ else()
 	add_subdirectory("${CMAKE_BINARY_DIR}/oniguruma-src"
 					 "${CMAKE_BINARY_DIR}/oniguruma-build"
 	)
+	add_library(oniguruma::onig ALIAS onig)
+endif()
+
+if (CMAKE_C_BYTE_ORDER STREQUAL "LITTLE_ENDIAN")
+	set(QSP_LITTLE_ENDIAN 1)
+elseif (CMAKE_C_BYTE_ORDER STREQUAL "BIG_ENDIAN")
+	set(QSP_LITTLE_ENDIAN 0)
+else()
+	message(FATAL_ERROR "Could not recognize endianess: ${CMAKE_C_BYTE_ORDER}")
 endif()
 
 configure_file(qsp_config.h.cmakein qsp_config.h @ONLY)
 
 set(QSP_SOURCES
+	qsp/bindings/bindings_config.h
 	qsp/bindings/default/default_callbacks.c
 	qsp/bindings/default/default_control.c
-	qsp/actions.c
-	qsp/callbacks.c
-	qsp/codetools.c
-	qsp/coding.c
-	qsp/common.c
-	qsp/errors.c
-	qsp/game.c
-	qsp/locations.c
-	qsp/mathops.c
-	qsp/memwatch.c
-	qsp/menu.c
-	qsp/objects.c
-	qsp/playlist.c
-	qsp/regexp.c
-	qsp/statements.c
-	qsp/text.c
-	qsp/time.c
+	qsp/bindings/default/qsp_default.h
+	qsp/bindings/qsp.h
+	qsp/actions.c qsp/actions.h
+	qsp/callbacks.c qsp/callbacks.h
+	qsp/codetools.c qsp/codetools.h
+	qsp/coding.c qsp/coding.h
+	qsp/common.c qsp/common.h
+	qsp/errors.c qsp/errors.h
+	qsp/game.c qsp/game.h
+	qsp/locations.c qsp/locations.h
+	qsp/mathops.c qsp/mathops.h
+	qsp/memwatch.c qsp/memwatch.h
+	qsp/menu.c qsp/menu.h
+	qsp/objects.c qsp/objects.h
+	qsp/playlist.c qsp/playlist.h
+	qsp/regexp.c qsp/regexp.h
+	qsp/statements.c qsp/statements.h
+	qsp/text.c qsp/text.h
+	qsp/time.c qsp/time.h
 	qsp/towlower.c
 	qsp/towupper.c
-	qsp/tuples.c
-	qsp/variables.c
-	qsp/variant.c
+	qsp/tuples.c qsp/tuples.h
+	qsp/variables.c qsp/variables.h
+	qsp/variant.c qsp/variant.h
 )
 add_library(qsp SHARED ${QSP_SOURCES})
 target_compile_definitions(qsp PUBLIC _UNICODE)
@@ -116,37 +132,40 @@ if(WIN32)
 endif()
 target_compile_options(qsp PRIVATE ${PROJECT_COMPILER_FLAGS})
 target_link_options(qsp PRIVATE ${PROJECT_LINKER_FLAGS})
-if(USE_INSTALLED_ONIGURUMA)
-	target_link_libraries(qsp PRIVATE oniguruma::onig)
-else()
-	target_link_libraries(qsp PRIVATE onig)
-	install(TARGETS onig)
+target_link_libraries(qsp PRIVATE oniguruma::onig)
+if(LINUX)
+	target_link_libraries(qsp PRIVATE Iconv::Iconv)
 endif()
 
 target_include_directories(qsp
 	INTERFACE
+		"$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/qsp/bindings/default>"
+		"$<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>"
 		"$<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}/qsp>"
 		"$<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}/qsp/default>"
 )
 
 install(TARGETS qsp EXPORT QspTargets DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT runtime)
+if(NOT USE_INSTALLED_ONIGURUMA)
+	install(TARGETS onig DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT runtime)
+endif()
 
 generate_export_header(qsp
   BASE_NAME Qsp
   EXPORT_MACRO_NAME QSP_EXTERN
 )
 
-write_basic_package_version_file(${CMAKE_CURRENT_BINARY_DIR}/QspConfigVersion.cmake COMPATIBILITY AnyNewerVersion)
-configure_package_config_file(QspConfig.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/QspConfig.cmake
+write_basic_package_version_file(${CMAKE_CURRENT_BINARY_DIR}/cmake/Qsp/QspConfigVersion.cmake COMPATIBILITY AnyNewerVersion)
+configure_package_config_file(QspConfig.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/cmake/Qsp/QspConfig.cmake
 	INSTALL_DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/Qsp"
 )
 
+export(TARGETS qsp NAMESPACE Qsp:: FILE ${CMAKE_CURRENT_BINARY_DIR}/QspConfig.cmake)
 install(EXPORT QspTargets NAMESPACE Qsp:: DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/Qsp")
-install(FILES
-		${CMAKE_CURRENT_BINARY_DIR}/QspConfigVersion.cmake
-		${CMAKE_CURRENT_BINARY_DIR}/QspConfig.cmake
-	DESTINATION
-		"${CMAKE_INSTALL_LIBDIR}/cmake/Qsp"
+
+install(
+	DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/cmake
+	DESTINATION "${CMAKE_INSTALL_LIBDIR}"
 )
 
 install(FILES
@@ -287,7 +306,10 @@ if(BUILD_QSPGUI)
 	target_compile_options(qspgui PRIVATE ${PROJECT_COMPILER_FLAGS})
 	target_link_options(qspgui PRIVATE ${PROJECT_LINKER_FLAGS})
 	target_link_libraries(qspgui PRIVATE qsp fmod wxbase wxcore wxadv wxaui wxhtml)
-	install(TARGETS qspgui qsp wxbase wxcore wxadv wxaui wxhtml)
+	install(TARGETS qspgui qsp)
+	if (NOT USE_INSTALLED_WX)
+		install(TARGETS wxbase wxcore wxadv wxaui wxhtml)
+	endif()
 
 	install(DIRECTORY "${CMAKE_SOURCE_DIR}/players/classic/misc/common/langs" DESTINATION "${RESOURCES_DIR}")
 	install(DIRECTORY "${CMAKE_SOURCE_DIR}/players/classic/misc/common/sound" DESTINATION "${RESOURCES_DIR}")

cmake/Modules/Findoniguruma.cmake

@@ -3,15 +3,16 @@
 # Redistribution and use is allowed according to the terms of the BSD license.
 
 if (NOT TARGET oniguruma::onig)
-	find_package(oniguruma CONFIG)
+	find_package(oniguruma CONFIG QUIET)
 	if (TARGET onig)
 		add_library(oniguruma::onig ALIAS onig)
 		set(oniguruma_FOUND True)
 	else(TARGET onig)
 		include(FindPackageHandleStandardArgs)
 		if (NOT WIN32)
 			# use pkg-config to get the directories and then use these values
-			include(FindPkgConfig)
+			include(CMakeFindDependencyMacro)
+			find_dependency(PkgConfig)
 			message(VERBOSE "FindOnigurama: trying pkg-config")
 			pkg_check_modules(oniguruma REQUIRED IMPORTED_TARGET GLOBAL oniguruma)
 			add_library(oniguruma::onig ALIAS PkgConfig::oniguruma)

qsp/bindings/default/qsp_default.h

@@ -15,25 +15,41 @@
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
 */
 
+#include "qsp_config.h"
+
+#if defined _UNICODE && !defined __linux__
+    #include <wchar.h>
+#endif
+
 #ifndef QSP_DEFAULTDEFINES
     #define QSP_DEFAULTDEFINES
 
-    static int qspEndiannessTestValue = 1;
-
     #ifdef _UNICODE
-        typedef wchar_t QSP_CHAR;
-        #define QSP_FMT2(x) L##x
-        #define QSP_FMT(x) QSP_FMT2(x)
-
-        #define QSP_ONIG_ENC ((*(char *)&(qspEndiannessTestValue) == 1) ? \
-                    (sizeof(QSP_CHAR) == 2 ? ONIG_ENCODING_UTF16_LE : ONIG_ENCODING_UTF32_LE) : \
-                    (sizeof(QSP_CHAR) == 2 ? ONIG_ENCODING_UTF16_BE : ONIG_ENCODING_UTF32_BE))
-        #define QSP_FROM_OS_CHAR(a) qspReverseConvertUC(a, qspCP1251ToUnicodeTable)
-        #define QSP_TO_OS_CHAR(a) qspDirectConvertUC(a, qspCP1251ToUnicodeTable)
-        #define QSP_CHRLWR qspToWLower
-        #define QSP_CHRUPR qspToWUpper
-        #define QSP_WCTOB
-        #define QSP_BTOWC
+        #ifdef __linux__
+            typedef char QSP_CHAR;
+            #define QSP_FMT(x) x
+
+            #define QSP_ONIG_ENC ONIG_ENCODING_UTF8
+
+            #define QSP_CHRLWR tolower
+            #define QSP_CHRUPR toupper
+        #else
+            typedef wchar_t QSP_CHAR;
+            #define QSP_FMT2(x) L##x
+            #define QSP_FMT(x) QSP_FMT2(x)
+
+            #if QSP_LITTLE_ENDIAN
+                #define QSP_ONIG_ENC (sizeof(QSP_CHAR) == 2 ? ONIG_ENCODING_UTF16_LE : ONIG_ENCODING_UTF32_LE)
+            #else
+                #define QSP_ONIG_ENC (sizeof(QSP_CHAR) == 2 ? ONIG_ENCODING_UTF16_BE : ONIG_ENCODING_UTF32_BE)
+            #endif
+            #define QSP_FROM_OS_CHAR(a) qspReverseConvertUC(a, qspCP1251ToUnicodeTable)
+            #define QSP_TO_OS_CHAR(a) qspDirectConvertUC(a, qspCP1251ToUnicodeTable)
+            #define QSP_CHRLWR qspToWLower
+            #define QSP_CHRUPR qspToWUpper
+            #define QSP_WCTOB
+            #define QSP_BTOWC
+        #endif
     #else
         typedef char QSP_CHAR;
         #define QSP_FMT(x) x
@@ -57,9 +73,11 @@
         #endif
     #endif
 
-    #define QSP_FIXBYTESORDER(a) ((*(char *)&(qspEndiannessTestValue) == 1) ? \
-                                 (a) : \
-                                 ((unsigned short)(((a) << 8) | ((a) >> 8))))
+    #if QSP_LITTLE_ENDIAN
+        #define QSP_FIXBYTESORDER(a) (a)
+    #else
+        #define QSP_FIXBYTESORDER(a) ((unsigned short)(((a) << 8) | ((a) >> 8))))
+    #endif
     #if defined(_MSC_VER)
         #define QSP_TIME _time64
     #else

qsp/coding.c

@@ -19,6 +19,13 @@
 #include "text.h"
 #include "variant.h"
 
+#ifdef __linux__
+    #include <assert.h>
+    #include <errno.h>
+    #include <iconv.h>
+    #include <stdint.h>
+#endif
+
 unsigned char qspCP1251ToKOI8RTable[] =
 {
     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
@@ -191,6 +198,7 @@ unsigned char qspKOI8ROrderTable[] =
     0x96, 0xA6, 0x97, 0x98, 0x99, 0x9A, 0x8D, 0x89, 0xA3, 0xA2, 0x8E, 0x9F, 0xA4, 0xA0, 0x9E, 0xA1
 };
 
+#ifndef __linux__
 INLINE char qspDirectConvertSB(char, unsigned char *);
 INLINE char qspReverseConvertSB(char, unsigned char *);
 INLINE int qspDirectConvertUC(char, int *);
@@ -226,10 +234,65 @@ INLINE char qspReverseConvertUC(int ch, int *table)
         if (table[i] == ch) return (char)(i + 0x80);
     return 0x20;
 }
+#else
+size_t usc2StrSizeInUtf8(uint16_t *str, size_t size)
+{
+    size_t res = size;
+    uint16_t *end = str + size;
+    for (uint16_t *p = str; p != end; ++p)
+    {
+        if (*p > 0x7ff)
+        {
+            res += 2;
+        }
+        else if (*p > 0x7f)
+        {
+            res += 1;
+        }
+    }
+    return res;
+}
+
+size_t utf8CodePointsCount(const char *str, const char *end)
+{
+    size_t res = 0;
+    for (; str != end; ++str)
+        res += ((*str & 0xc0) != 0x80);
+    return res;
+}
+
+size_t koi8RStrSizeInUtf8(const char *data, size_t dataSize)
+{
+    size_t res = dataSize;
+    const char *end = data + dataSize;
+    for (; data != end; ++data)
+    {
+        if (*data < 0)
+        {
+            res += 1;
+        }
+    }
+    return res;
+}
+
+#endif
 
 void *qspStringToFileData(QSPString s, QSP_BOOL isUCS2, int *dataSize)
 {
     char *buf;
+#ifdef __linux__
+    size_t len = qspStrLen(s);
+    size_t bufSize = utf8CodePointsCount(s.Str, s.End);
+    if (isUCS2) bufSize *= 2;
+    buf = (char *)malloc(bufSize);
+    iconv_t iconvConvDescriptor = iconv_open(isUCS2 ? "UCS−2" : "KOI8-R", "UTF−8");
+    assert(iconvConvDescriptor != (iconv_t)(-1));
+    char *inBuf = s.Str;
+    char *outBuf = buf;
+    size_t outBufSize = bufSize;
+    iconv(iconvConvDescriptor, &inBuf, &len, &outBuf, &outBufSize);
+    iconv_close(iconvConvDescriptor);
+#else
     unsigned short uCh, *uPtr;
     QSP_CHAR *origBuf = s.Str;
     int bufSize, len = qspStrLen(s);
@@ -249,12 +312,35 @@ void *qspStringToFileData(QSPString s, QSP_BOOL isUCS2, int *dataSize)
         while (--len >= 0)
             buf[len] = QSP_FROM_OS_CHAR(origBuf[len]);
     }
+#endif
     *dataSize = bufSize;
     return buf;
 }
 
 QSPString qspStringFromFileData(void *data, int dataSize, QSP_BOOL isUCS2)
 {
+#ifdef __linux__
+    size_t inbufSize;
+    iconv_t iconv_cd;
+    size_t len, bytesLeft;
+    QSP_CHAR *ret;
+    char *inData, *outData;
+
+    len = isUCS2 ? usc2StrSizeInUtf8(data, dataSize / 2) : koi8RStrSizeInUtf8(data, dataSize);
+    if (!len) return qspNullString;
+    ret = (QSP_CHAR *)malloc(len + 1);
+    errno = 0;
+    iconv_cd = iconv_open("UTF-8", isUCS2 ? "UCS-2" : "KOI8-R");
+    assert(iconv_cd != (iconv_t)(-1));
+    inbufSize = dataSize;
+    inData = data;
+    outData = ret;
+    bytesLeft = len;
+    iconv(iconv_cd, &inData, &inbufSize, &outData, &bytesLeft);
+    assert(bytesLeft == 0);
+    iconv_close(iconv_cd);
+    ret[len] = 0;
+#else
     char *ptr;
     unsigned short uCh, *uPtr;
     QSP_CHAR *ret;
@@ -277,6 +363,7 @@ QSPString qspStringFromFileData(void *data, int dataSize, QSP_BOOL isUCS2)
         while (--curLen >= 0)
             ret[curLen] = (QSP_CHAR)QSP_TO_OS_CHAR(ptr[curLen]);
     }
+#endif
     return qspStringFromLen(ret, len);
 }
 

qsp_config.h.cmakein

@@ -3,6 +3,6 @@
 
 #define QSP_VER_STR "@qsp_VERSION@"
 #define QSP_GAMEMINVER_VER_STR "@QSP_GAMEMIN_VER@"
+#define QSP_LITTLE_ENDIAN @QSP_LITTLE_ENDIAN@
 
 #endif
-