From 870cc09d01d004cdba34c4423660ec0cc1efa3ae Mon Sep 17 00:00:00 2001 From: DevGeniusCode <136935333+DevGeniusCode@users.noreply.github.com> Date: Sun, 15 Dec 2024 08:50:14 +0200 Subject: [PATCH 1/9] Added ReverseString function to Utf16String class --- src/game/common/system/unicodestring.cpp | 30 ++++++++++++++++++++++++ src/game/common/system/unicodestring.h | 1 + 2 files changed, 31 insertions(+) diff --git a/src/game/common/system/unicodestring.cpp b/src/game/common/system/unicodestring.cpp index 0c1bdc283..9321bbd38 100644 --- a/src/game/common/system/unicodestring.cpp +++ b/src/game/common/system/unicodestring.cpp @@ -334,6 +334,36 @@ void Utf16String::Trim() } } +/** + * Reverse string for RTL languages + */ +void Utf16String::ReverseString() +{ + if (m_data == nullptr) { + return; + } + + size_type len = Get_Length(); + if (len <= 1) { + return; + } + + unichar_t* buffer = Get_Buffer_For_Read(len); + + size_t start = 0; + size_t end = len - 1; + + while (start < end) { + unichar_t temp = buffer[start]; + buffer[start] = buffer[end]; + buffer[end] = temp; + + start++; + end--; + } +} + + void Utf16String::To_Lower() { unichar_t buf[MAX_FORMAT_BUF_LEN]; diff --git a/src/game/common/system/unicodestring.h b/src/game/common/system/unicodestring.h index d8ec8e30a..2346fe252 100644 --- a/src/game/common/system/unicodestring.h +++ b/src/game/common/system/unicodestring.h @@ -132,6 +132,7 @@ class Utf16String void Concat(unichar_t c); void Concat(const unichar_t *s); void Concat(Utf16String const &string) { Concat(string.Str()); } + void ReverseString(); void Trim(); void To_Lower(); From c6507791833510e29468c5faa8f11e0315890b8e Mon Sep 17 00:00:00 2001 From: DevGeniusCode <136935333+DevGeniusCode@users.noreply.github.com> Date: Sun, 15 Dec 2024 08:52:43 +0200 Subject: [PATCH 2/9] Added RTL_Reverse option --- src/game/client/gametextfile.cpp | 3 +++ src/game/client/gametextfile.h | 1 + src/tools/gametextcompiler/src/main.cpp | 2 +- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/game/client/gametextfile.cpp b/src/game/client/gametextfile.cpp index a88dfdba8..9525e2697 100644 --- a/src/game/client/gametextfile.cpp +++ b/src/game/client/gametextfile.cpp @@ -69,6 +69,7 @@ constexpr const char *const s_option_2 = "Check_Buffer_Length_On_Load"; constexpr const char *const s_option_3 = "Check_Buffer_Length_On_Save"; constexpr const char *const s_option_4 = "Keep_Obsolete_Spaces_On_Load"; constexpr const char *const s_option_5 = "Write_Extra_LF_On_STR_Save"; +constexpr const char *const s_option_6 = "RTL_Reverse"; constexpr const char *const s_options[] = { s_option_0, @@ -77,6 +78,7 @@ constexpr const char *const s_options[] = { s_option_3, s_option_4, s_option_5, + s_option_6, }; static_assert(s_option_0 == s_options[size_t(GameTextOption::NONE)]); @@ -85,6 +87,7 @@ static_assert(s_option_2 == s_options[1 + Bit_To_Index(GameTextOption::CHECK_BUF static_assert(s_option_3 == s_options[1 + Bit_To_Index(GameTextOption::CHECK_BUFFER_LENGTH_ON_SAVE)]); static_assert(s_option_4 == s_options[1 + Bit_To_Index(GameTextOption::KEEP_OBSOLETE_SPACES_ON_LOAD)]); static_assert(s_option_5 == s_options[1 + Bit_To_Index(GameTextOption::WRITE_EXTRA_LF_ON_STR_SAVE)]); +static_assert(s_option_6 == s_options[1 + Bit_To_Index(GameTextOption::RTL_REVERSE)]); } // namespace bool Name_To_Game_Text_Option(const char *name, GameTextOption &option) diff --git a/src/game/client/gametextfile.h b/src/game/client/gametextfile.h index ad3556f99..52cf04110 100644 --- a/src/game/client/gametextfile.h +++ b/src/game/client/gametextfile.h @@ -33,6 +33,7 @@ enum class GameTextOption : uint32_t CHECK_BUFFER_LENGTH_ON_SAVE = (1 << 2), KEEP_OBSOLETE_SPACES_ON_LOAD = (1 << 3), WRITE_EXTRA_LF_ON_STR_SAVE = (1 << 4), + RTL_REVERSE = (1 << 5), }; bool Name_To_Game_Text_Option(const char *name, GameTextOption &option); diff --git a/src/tools/gametextcompiler/src/main.cpp b/src/tools/gametextcompiler/src/main.cpp index 1c36409a3..0078804cf 100644 --- a/src/tools/gametextcompiler/src/main.cpp +++ b/src/tools/gametextcompiler/src/main.cpp @@ -106,7 +106,7 @@ All|English|German|French|Spanish|Italian|Japanese|Korean|Chinese|Brazilian|Poli OPTION:enum OPTION takes one [1] or multiple [n] options, separated by pipe: -None|Check_Buffer_Length_On_Load|Check_Buffer_Length_On_Save| +None|Check_Buffer_Length_On_Load|Check_Buffer_Length_On_Save|RTL_Reverse Keep_Obsolete_Spaces_On_Load|Write_Extra_LF_On_STR_Save|Optimize_Memory_Size )#"); // 1 2 3 4 5 6 7 8 9 10 11 12 From 2a1f9811ea3326e2002f4ae71ec9fc21b97ed8cf Mon Sep 17 00:00:00 2001 From: DevGeniusCode <136935333+DevGeniusCode@users.noreply.github.com> Date: Sun, 15 Dec 2024 08:54:25 +0200 Subject: [PATCH 3/9] Added RTL_Reverse support --- src/game/client/gametextfile.cpp | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/game/client/gametextfile.cpp b/src/game/client/gametextfile.cpp index 9525e2697..886b0ca9b 100644 --- a/src/game/client/gametextfile.cpp +++ b/src/game/client/gametextfile.cpp @@ -849,6 +849,10 @@ void GameTextFile::Read_STR_File_T(FileRef &file, StringInfosType &string_infos, case StrReadStep::TEXT: if (languages.has(read_language)) { + if (read_language == LanguageID::ARABIC) + { + options = Options::Value::RTL_REVERSE; + } Parse_STR_Text(buf, Get_Text(string_info, read_language), options); } Change_Step(step, StrReadStep::SEARCH, eol_chars); @@ -922,6 +926,10 @@ void GameTextFile::Parse_STR_Text(Utf8Array &buf, Utf16String &text, Options opt // Translate final UTF16 string. text.Translate(buf.data()); + if (options.has(Options::Value::RTL_REVERSE)) { + text.ReverseString(); + } + } void GameTextFile::Parse_STR_Speech(Utf8View &buf, Utf8String &speech) @@ -1151,6 +1159,10 @@ bool GameTextFile::Write_Multi_STR_Entry( For_Each_Language(languages, [&](LanguageID language) { const size_t index = static_cast(language); ok &= Write_STR_Language(file, language); + if (language == LanguageID::US) + { + options = Options::Value::RTL_REVERSE; + } ok &= Write_STR_Text(file, string_info.text[index], options, buf, str); }); @@ -1221,6 +1233,9 @@ bool GameTextFile::Write_STR_Label(FileRef &file, const Utf8String &label) bool GameTextFile::Write_STR_Text(FileRef &file, const Utf16String &text, Options options, Utf8Array &buf, Utf8String &str) { // Convert utf16 to utf8. + if (options.has(Options::Value::RTL_REVERSE)) { + text.ReverseString(); + } str.Translate(text.Str()); // STR does support escaped characters for special control characters. Write them out as escaped characters so they From 450d2fee3908baccb7a2738fbb59943eab19e92f Mon Sep 17 00:00:00 2001 From: DevGeniusCode <136935333+DevGeniusCode@users.noreply.github.com> Date: Sun, 15 Dec 2024 08:55:57 +0200 Subject: [PATCH 4/9] Help text update and added build subdirectory --- CMakeLists.txt | 3 +++ src/tools/gametextcompiler/src/main.cpp | 16 +++++++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 633d57455..9819500ca 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -314,6 +314,9 @@ endif() # Build Thyme add_subdirectory(src) +# Build gametextcompiler +add_subdirectory(src/tools/gametextcompiler) + if(STANDALONE) list(APPEND INSTALL_TARGETS thyme) endif() diff --git a/src/tools/gametextcompiler/src/main.cpp b/src/tools/gametextcompiler/src/main.cpp index 0078804cf..4951cab87 100644 --- a/src/tools/gametextcompiler/src/main.cpp +++ b/src/tools/gametextcompiler/src/main.cpp @@ -119,6 +119,8 @@ All capital words are NOT interpreted keywords and are substituted by the comman [1] and [n] words show that argument takes one or multiple values. Commands and command arguments are not case sensitive. +WARNING: Commands that save files will overwrite existing files without confirmation. + -options OPTION[n] > Sets option(s) for loaded and saved file. @@ -142,6 +144,18 @@ Commands and command arguments are not case sensitive. -save_str_languages LANGUAGE[n] > Sets language(s) to save Multi STR file with. + +Example 1: Convert a CSF file to STR format + > gametextcompiler.exe -load_csf D:\generals.csf -save_str D:\generals.str + +Example 2: Convert an STR file to CSF format with given language + > gametextcompiler.exe -load_str D:\generals.str -swap_and_set_language English -save_csf D:\generals.csf + +Example 3: Load a specific language from a multi-language file and save to a CSF file + > gametextcompiler.exe -load_str_languages English -load_str D:\generals.str -save_csf D:\generals.csf + +Example 4: Convert a CSF file to STR format with RTL reversing + > -load_csf D:\generals.csf -save_str D:\generals.str -options RTL_REVERSE )#"); } // clang-format on @@ -219,7 +233,7 @@ struct EngineSystemsCreator int main(int argc, const char *argv[]) { - Print_Line("Game Text Compiler 1.1 by The Assembly Armada"); + Print_Line("Game Text Compiler 1.2 by The Assembly Armada"); if (argc < 2) { Print_Help(); From c8ce01a155ca8da92b83f68f0c9eb465fdec5bed Mon Sep 17 00:00:00 2001 From: DevGeniusCode <136935333+DevGeniusCode@users.noreply.github.com> Date: Sun, 15 Dec 2024 08:57:20 +0200 Subject: [PATCH 5/9] fix const --- src/game/client/gametextfile.cpp | 14 +++++++------- src/game/client/gametextfile.h | 8 ++++---- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/game/client/gametextfile.cpp b/src/game/client/gametextfile.cpp index 886b0ca9b..973396a8f 100644 --- a/src/game/client/gametextfile.cpp +++ b/src/game/client/gametextfile.cpp @@ -362,7 +362,7 @@ bool GameTextFile::Save(const char *filename, FileType filetype, Languages langu break; } case FileType::STR: { - success = Write_STR_File(file, Get_String_Infos(), m_options); + success = Write_STR_File(file, const_cast(Get_String_Infos()), m_options); break; } case FileType::MULTI_STR: { @@ -1139,7 +1139,7 @@ bool GameTextFile::Write_Multi_STR_File( Utf8String str; str.Get_Buffer_For_Read(TEXT_8_SIZE); - for (const MultiStringInfo &string_info : multi_string_infos) { + for (MultiStringInfo &string_info : multi_string_infos) { if (!string_info.label.Is_Empty()) { if (!Write_Multi_STR_Entry(file, string_info, languages, options, buf, str)) { return false; @@ -1150,7 +1150,7 @@ bool GameTextFile::Write_Multi_STR_File( } bool GameTextFile::Write_Multi_STR_Entry( - FileRef &file, const MultiStringInfo &string_info, Languages languages, Options options, Utf8Array &buf, Utf8String &str) + FileRef &file, MultiStringInfo &string_info, Languages languages, Options options, Utf8Array &buf, Utf8String &str) { bool ok = true; @@ -1190,7 +1190,7 @@ bool GameTextFile::Write_STR_Language(FileRef &file, LanguageID language) return ok; } -bool GameTextFile::Write_STR_File(FileRef &file, const StringInfos &string_infos, Options options) +bool GameTextFile::Write_STR_File(FileRef &file, StringInfos &string_infos, Options options) { GAMETEXTLOG_INFO("Writing text file '%s' in STR format", file.Get_File_Name().Str()); @@ -1198,7 +1198,7 @@ bool GameTextFile::Write_STR_File(FileRef &file, const StringInfos &string_infos Utf8String str; str.Get_Buffer_For_Read(TEXT_8_SIZE); - for (const StringInfo &string_info : string_infos) { + for (StringInfo &string_info : string_infos) { if (!string_info.label.Is_Empty()) { if (!Write_STR_Entry(file, string_info, options, buf, str)) { return false; @@ -1209,7 +1209,7 @@ bool GameTextFile::Write_STR_File(FileRef &file, const StringInfos &string_infos } bool GameTextFile::Write_STR_Entry( - FileRef &file, const StringInfo &string_info, Options options, Utf8Array &buf, Utf8String &str) + FileRef &file, StringInfo &string_info, Options options, Utf8Array &buf, Utf8String &str) { bool ok = true; ok &= Write_STR_Label(file, string_info.label); @@ -1230,7 +1230,7 @@ bool GameTextFile::Write_STR_Label(FileRef &file, const Utf8String &label) return ok; } -bool GameTextFile::Write_STR_Text(FileRef &file, const Utf16String &text, Options options, Utf8Array &buf, Utf8String &str) +bool GameTextFile::Write_STR_Text(FileRef &file, Utf16String &text, Options options, Utf8Array &buf, Utf8String &str) { // Convert utf16 to utf8. if (options.has(Options::Value::RTL_REVERSE)) { diff --git a/src/game/client/gametextfile.h b/src/game/client/gametextfile.h index 52cf04110..07b61667a 100644 --- a/src/game/client/gametextfile.h +++ b/src/game/client/gametextfile.h @@ -219,18 +219,18 @@ class GameTextFile static bool Write_Multi_STR_File( FileRef &file, const ConstStringInfosPtrArray &string_infos_ptrs, Languages languages, Options options); static bool Write_Multi_STR_Entry(FileRef &file, - const MultiStringInfo &string_info, + MultiStringInfo &string_info, Languages languages, Options options, Utf8Array &buf, Utf8String &str); static bool Write_STR_Language(FileRef &file, LanguageID language); - static bool Write_STR_File(FileRef &file, const StringInfos &string_infos, Options options); + static bool Write_STR_File(FileRef &file, StringInfos &string_infos, Options options); static bool Write_STR_Entry( - FileRef &file, const StringInfo &string_info, Options options, Utf8Array &buf, Utf8String &str); + FileRef &file, StringInfo &string_info, Options options, Utf8Array &buf, Utf8String &str); static bool Write_STR_Label(FileRef &file, const Utf8String &label); - static bool Write_STR_Text(FileRef &file, const Utf16String &text, Options options, Utf8Array &buf, Utf8String &str); + static bool Write_STR_Text(FileRef &file, Utf16String &text, Options options, Utf8Array &buf, Utf8String &str); static bool Write_STR_Speech(FileRef &file, const Utf8String &speech); static bool Write_STR_End(FileRef &file); From fdab2da3158e9fa091150df607618a35ea0d2003 Mon Sep 17 00:00:00 2001 From: DevGeniusCode <136935333+DevGeniusCode@users.noreply.github.com> Date: Sun, 15 Dec 2024 09:26:42 +0200 Subject: [PATCH 6/9] fix mistake --- src/game/client/gametextfile.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/game/client/gametextfile.cpp b/src/game/client/gametextfile.cpp index 973396a8f..f46da9149 100644 --- a/src/game/client/gametextfile.cpp +++ b/src/game/client/gametextfile.cpp @@ -849,10 +849,6 @@ void GameTextFile::Read_STR_File_T(FileRef &file, StringInfosType &string_infos, case StrReadStep::TEXT: if (languages.has(read_language)) { - if (read_language == LanguageID::ARABIC) - { - options = Options::Value::RTL_REVERSE; - } Parse_STR_Text(buf, Get_Text(string_info, read_language), options); } Change_Step(step, StrReadStep::SEARCH, eol_chars); From c0103d609dcc3111fde685a8a37b24646189a2b3 Mon Sep 17 00:00:00 2001 From: DevGeniusCode <136935333+DevGeniusCode@users.noreply.github.com> Date: Thu, 26 Dec 2024 07:35:35 +0200 Subject: [PATCH 7/9] fix mistake --- src/game/client/gametextfile.cpp | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/game/client/gametextfile.cpp b/src/game/client/gametextfile.cpp index 973396a8f..a3e5d6c28 100644 --- a/src/game/client/gametextfile.cpp +++ b/src/game/client/gametextfile.cpp @@ -849,10 +849,6 @@ void GameTextFile::Read_STR_File_T(FileRef &file, StringInfosType &string_infos, case StrReadStep::TEXT: if (languages.has(read_language)) { - if (read_language == LanguageID::ARABIC) - { - options = Options::Value::RTL_REVERSE; - } Parse_STR_Text(buf, Get_Text(string_info, read_language), options); } Change_Step(step, StrReadStep::SEARCH, eol_chars); @@ -1159,10 +1155,6 @@ bool GameTextFile::Write_Multi_STR_Entry( For_Each_Language(languages, [&](LanguageID language) { const size_t index = static_cast(language); ok &= Write_STR_Language(file, language); - if (language == LanguageID::US) - { - options = Options::Value::RTL_REVERSE; - } ok &= Write_STR_Text(file, string_info.text[index], options, buf, str); }); From 5187362fe03e6d07f40eb66bd025d4a11e3b3a48 Mon Sep 17 00:00:00 2001 From: DevGeniusCode <136935333+DevGeniusCode@users.noreply.github.com> Date: Mon, 27 Jan 2025 07:01:38 +0200 Subject: [PATCH 8/9] Refactor Utf16String Reverse functions --- src/game/client/gametextfile.cpp | 25 +++--- src/game/client/gametextfile.h | 8 +- src/game/common/system/unicodestring.cpp | 102 ++++++++++++++++++++--- src/game/common/system/unicodestring.h | 4 +- src/tools/gametextcompiler/src/main.cpp | 2 +- 5 files changed, 114 insertions(+), 27 deletions(-) diff --git a/src/game/client/gametextfile.cpp b/src/game/client/gametextfile.cpp index a3e5d6c28..c0d749314 100644 --- a/src/game/client/gametextfile.cpp +++ b/src/game/client/gametextfile.cpp @@ -362,7 +362,7 @@ bool GameTextFile::Save(const char *filename, FileType filetype, Languages langu break; } case FileType::STR: { - success = Write_STR_File(file, const_cast(Get_String_Infos()), m_options); + success = Write_STR_File(file, Get_String_Infos(), m_options); break; } case FileType::MULTI_STR: { @@ -923,7 +923,7 @@ void GameTextFile::Parse_STR_Text(Utf8Array &buf, Utf16String &text, Options opt // Translate final UTF16 string. text.Translate(buf.data()); if (options.has(Options::Value::RTL_REVERSE)) { - text.ReverseString(); + text.Reverse(); } } @@ -1135,7 +1135,7 @@ bool GameTextFile::Write_Multi_STR_File( Utf8String str; str.Get_Buffer_For_Read(TEXT_8_SIZE); - for (MultiStringInfo &string_info : multi_string_infos) { + for (const MultiStringInfo &string_info : multi_string_infos) { if (!string_info.label.Is_Empty()) { if (!Write_Multi_STR_Entry(file, string_info, languages, options, buf, str)) { return false; @@ -1146,7 +1146,7 @@ bool GameTextFile::Write_Multi_STR_File( } bool GameTextFile::Write_Multi_STR_Entry( - FileRef &file, MultiStringInfo &string_info, Languages languages, Options options, Utf8Array &buf, Utf8String &str) + FileRef &file, const MultiStringInfo &string_info, Languages languages, Options options, Utf8Array &buf, Utf8String &str) { bool ok = true; @@ -1182,7 +1182,7 @@ bool GameTextFile::Write_STR_Language(FileRef &file, LanguageID language) return ok; } -bool GameTextFile::Write_STR_File(FileRef &file, StringInfos &string_infos, Options options) +bool GameTextFile::Write_STR_File(FileRef &file, const StringInfos &string_infos, Options options) { GAMETEXTLOG_INFO("Writing text file '%s' in STR format", file.Get_File_Name().Str()); @@ -1190,7 +1190,7 @@ bool GameTextFile::Write_STR_File(FileRef &file, StringInfos &string_infos, Opti Utf8String str; str.Get_Buffer_For_Read(TEXT_8_SIZE); - for (StringInfo &string_info : string_infos) { + for (const StringInfo &string_info : string_infos) { if (!string_info.label.Is_Empty()) { if (!Write_STR_Entry(file, string_info, options, buf, str)) { return false; @@ -1201,7 +1201,7 @@ bool GameTextFile::Write_STR_File(FileRef &file, StringInfos &string_infos, Opti } bool GameTextFile::Write_STR_Entry( - FileRef &file, StringInfo &string_info, Options options, Utf8Array &buf, Utf8String &str) + FileRef &file, const StringInfo &string_info, Options options, Utf8Array &buf, Utf8String &str) { bool ok = true; ok &= Write_STR_Label(file, string_info.label); @@ -1222,13 +1222,18 @@ bool GameTextFile::Write_STR_Label(FileRef &file, const Utf8String &label) return ok; } -bool GameTextFile::Write_STR_Text(FileRef &file, Utf16String &text, Options options, Utf8Array &buf, Utf8String &str) +bool GameTextFile::Write_STR_Text(FileRef &file, const Utf16String &text, Options options, Utf8Array &buf, Utf8String &str) { // Convert utf16 to utf8. if (options.has(Options::Value::RTL_REVERSE)) { - text.ReverseString(); + Utf16String reversed; + reversed.Set(text.Str()); // Copy the string to avoid modifying the original. + reversed.Reverse(); + str.Translate(reversed.Str()); + } + else { + str.Translate(text.Str()); } - str.Translate(text.Str()); // STR does support escaped characters for special control characters. Write them out as escaped characters so they // are easily modifiable in text editor. diff --git a/src/game/client/gametextfile.h b/src/game/client/gametextfile.h index 07b61667a..52cf04110 100644 --- a/src/game/client/gametextfile.h +++ b/src/game/client/gametextfile.h @@ -219,18 +219,18 @@ class GameTextFile static bool Write_Multi_STR_File( FileRef &file, const ConstStringInfosPtrArray &string_infos_ptrs, Languages languages, Options options); static bool Write_Multi_STR_Entry(FileRef &file, - MultiStringInfo &string_info, + const MultiStringInfo &string_info, Languages languages, Options options, Utf8Array &buf, Utf8String &str); static bool Write_STR_Language(FileRef &file, LanguageID language); - static bool Write_STR_File(FileRef &file, StringInfos &string_infos, Options options); + static bool Write_STR_File(FileRef &file, const StringInfos &string_infos, Options options); static bool Write_STR_Entry( - FileRef &file, StringInfo &string_info, Options options, Utf8Array &buf, Utf8String &str); + FileRef &file, const StringInfo &string_info, Options options, Utf8Array &buf, Utf8String &str); static bool Write_STR_Label(FileRef &file, const Utf8String &label); - static bool Write_STR_Text(FileRef &file, Utf16String &text, Options options, Utf8Array &buf, Utf8String &str); + static bool Write_STR_Text(FileRef &file, const Utf16String &text, Options options, Utf8Array &buf, Utf8String &str); static bool Write_STR_Speech(FileRef &file, const Utf8String &speech); static bool Write_STR_End(FileRef &file); diff --git a/src/game/common/system/unicodestring.cpp b/src/game/common/system/unicodestring.cpp index 9321bbd38..bb20f9cd8 100644 --- a/src/game/common/system/unicodestring.cpp +++ b/src/game/common/system/unicodestring.cpp @@ -334,11 +334,59 @@ void Utf16String::Trim() } } -/** - * Reverse string for RTL languages - */ -void Utf16String::ReverseString() +// Helper function to check if a character is considered "ignored" +bool is_ignored(unichar_t ch) +{ + bool result = ((ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || // English letters + (ch >= '0' && ch <= '9') || // Digits + (ch == '%') || // Percentage + (ch == '&')); // Special number format + return result; +} + +bool is_special_char(unichar_t ch) +{ + // Check if character is a special character (comma, period, colon) + bool result = (ch == ',' || ch == '.' || ch == ':' || ch == '\'' || ch == '\"' || ch == '(' || ch == ')' || ch == '[' + || ch == ']' || ch == '{' || ch == '}' || ch == '<' || ch == '>' || ch == '?' || ch == '!' || ch == ';' || ch == '/' + || ch == '+' || ch == '|' || ch == '@' || ch == '#' || ch == '$' || ch == '=' || ch == '*' || ch == '~' || ch == '`' + || ch == '_' + || ch == 0x00A9 /*©*/ || ch == 0x201D /*”*/ || ch == 0x2019 /*’*/ || ch == 0x201C /*“*/ || ch == 0x2018 /*‘*/); + return result; +} + +// Helper function to check if a character is "ignored" or a special character +bool is_ignored_or_special(unichar_t ch) +{ + return is_ignored(ch) || is_special_char(ch); +} + +void Utf16String::Reverse() { + // This function reverses the string for RTL languages while ensuring certain characters + // are not reversed. The skipping logic is as follows: + // + // 1. Characters considered "ignored" (e.g., English letters, digits, and %) + // will not be reversed. These characters will remain in their original order within the string. + // + // 2. If a special char like comma, period, space or colon appears between two "ignored" characters (e.g., in numbers + // like 1,000 | 25.14 | %.0f%% | 'New York' | %d.%02d.%d), the punctuation will be treated as part of the "ignored" + // sequence and will not be reversed. + // + // 2.1. If a special char appears at the start or end of the sequence, + // it will be treated as part of the "non-ignored" sequence (e.g., "Level : 1" -> "1 : leveL" when 'Level' is + // non-English word). + // + // 2.2. If a space appears after a special/ignored char, it will check the next word/char for "ignored" status. + // + // 3. If a special char appears next to an "ignored" character but is followed by a non-ignored character + // (e.g., a space or a letter), the special will be reversed along with the surrounding text. + // + // 4. Empty lines and `\n` character are skipped and not affected by the reversal. + // + // The function loops through the string, reverses it, and ensures the proper skipping and + // handling of "ignored" characters and special based on the above rules. + if (m_data == nullptr) { return; } @@ -348,18 +396,50 @@ void Utf16String::ReverseString() return; } - unichar_t* buffer = Get_Buffer_For_Read(len); + unichar_t *buffer = Get_Buffer_For_Read(len); size_t start = 0; size_t end = len - 1; - while (start < end) { - unichar_t temp = buffer[start]; - buffer[start] = buffer[end]; - buffer[end] = temp; + while (start <= end) { + // Skip "ignored" sequences and their associated special characters + if (is_ignored(buffer[start])) { + while (start <= end && is_ignored_or_special(buffer[start])) { + start++; + } + // Move back to the last valid "ignored" character + start--; + // Skip any trailing special characters that are part of the "ignored" sequence + while (start + 1 <= end && is_special_char(buffer[start + 1]) && is_ignored(buffer[start + 2])) { + start++; + } + // Skip any spaces that are part of the "ignored" sequence + while (start + 1 <= end && buffer[start + 1] == ' ' && is_ignored(buffer[start + 2])) { + start++; + } + // Skip any trailing special characters after spaces + while (start + 1 <= end && is_special_char(buffer[start + 1]) && is_ignored(buffer[start + 2])) { + start++; + } + // Move to the next character after the "ignored" sequence + start++; + continue; + } + + // Reverse non-"ignored" sequences + size_t reverse_start = start; + while (start <= end && buffer[start] != '\n' && !is_ignored_or_special(buffer[start])) { + start++; + } + size_t reverse_end = start - 1; + std::reverse(buffer + reverse_start, buffer + reverse_end + 1); - start++; - end--; + if (start <= end && !is_ignored(buffer[start])) { + start++; + } + if (start <= end && buffer[start] == ' ') { + start++; + } } } diff --git a/src/game/common/system/unicodestring.h b/src/game/common/system/unicodestring.h index 2346fe252..d29eba9f7 100644 --- a/src/game/common/system/unicodestring.h +++ b/src/game/common/system/unicodestring.h @@ -18,6 +18,8 @@ #include "critsection.h" #include #include +#include +#include #ifdef BUILD_WITH_ICU #include @@ -132,7 +134,7 @@ class Utf16String void Concat(unichar_t c); void Concat(const unichar_t *s); void Concat(Utf16String const &string) { Concat(string.Str()); } - void ReverseString(); + void Reverse(); // Thyme specific void Trim(); void To_Lower(); diff --git a/src/tools/gametextcompiler/src/main.cpp b/src/tools/gametextcompiler/src/main.cpp index 4951cab87..48009a25e 100644 --- a/src/tools/gametextcompiler/src/main.cpp +++ b/src/tools/gametextcompiler/src/main.cpp @@ -155,7 +155,7 @@ Example 3: Load a specific language from a multi-language file and save to a CSF > gametextcompiler.exe -load_str_languages English -load_str D:\generals.str -save_csf D:\generals.csf Example 4: Convert a CSF file to STR format with RTL reversing - > -load_csf D:\generals.csf -save_str D:\generals.str -options RTL_REVERSE + > gametextcompiler.exe -load_csf D:\generals.csf -save_str D:\generals.str -options RTL_REVERSE )#"); } // clang-format on From dd388c8650eb2ef2e615803ed09ad0e0a3319664 Mon Sep 17 00:00:00 2001 From: DevGeniusCode <136935333+DevGeniusCode@users.noreply.github.com> Date: Tue, 28 Jan 2025 02:09:03 +0200 Subject: [PATCH 9/9] Update Reverse function --- src/game/common/system/unicodestring.cpp | 220 ++++++++++++++++++----- 1 file changed, 175 insertions(+), 45 deletions(-) diff --git a/src/game/common/system/unicodestring.cpp b/src/game/common/system/unicodestring.cpp index bb20f9cd8..262613f68 100644 --- a/src/game/common/system/unicodestring.cpp +++ b/src/game/common/system/unicodestring.cpp @@ -335,30 +335,170 @@ void Utf16String::Trim() } // Helper function to check if a character is considered "ignored" -bool is_ignored(unichar_t ch) +bool is_ignore(unichar_t ch) { bool result = ((ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || // English letters + (ch >= 0x00C0 && ch <= 0x00FF) || // Latin characters (ch >= '0' && ch <= '9') || // Digits - (ch == '%') || // Percentage + (ch == '%') || // Percentage or format specifier (ch == '&')); // Special number format return result; } -bool is_special_char(unichar_t ch) +bool is_special(unichar_t ch) { - // Check if character is a special character (comma, period, colon) + // Check if character is a special character (comma, period, colon etc.) bool result = (ch == ',' || ch == '.' || ch == ':' || ch == '\'' || ch == '\"' || ch == '(' || ch == ')' || ch == '[' || ch == ']' || ch == '{' || ch == '}' || ch == '<' || ch == '>' || ch == '?' || ch == '!' || ch == ';' || ch == '/' || ch == '+' || ch == '|' || ch == '@' || ch == '#' || ch == '$' || ch == '=' || ch == '*' || ch == '~' || ch == '`' - || ch == '_' + || ch == '_' || ch == '-' || ch == 0x00A9 /*©*/ || ch == 0x201D /*”*/ || ch == 0x2019 /*’*/ || ch == 0x201C /*“*/ || ch == 0x2018 /*‘*/); + return result; } // Helper function to check if a character is "ignored" or a special character bool is_ignored_or_special(unichar_t ch) { - return is_ignored(ch) || is_special_char(ch); + return is_ignore(ch) || is_special(ch); +} + +Utf16String ignore_sequence(unichar_t *buffer, size_t *start, size_t end) +{ + Utf16String ignore_part; + size_t i = (*start); + while (i <= end) { + // Skip ignored sequences and their associated special characters + if (is_ignore(buffer[i]) || i + 1 <= end && is_special(buffer[i]) && is_ignore(buffer[i + 1])) { + // Skip any leading special characters that are part of the ignored sequence (e.g., "[Game]", "[+12]") + while (i <= end && is_ignore(buffer[i]) + || i + 1 <= end && is_special(buffer[i]) && is_ignore(buffer[i + 1])) { + ignore_part += buffer[i]; + i++; + } + // Move back to the last valid "ignored" character + i--; + // Skip any trailing special characters that are *between* of the ignored sequence (e.g., "24[:]00") + if (i + 2 <= end && is_special(buffer[i + 1]) && is_ignore(buffer[i + 2])) { + i++; + ignore_part += buffer[i]; + } + // Skip any spaces that are *between of the "ignored" sequence (e.g., "Game[ ]Over") + else if (i + 2 <= end && buffer[i + 1] == ' ' && is_ignore(buffer[i + 2])) { + i++; + ignore_part += buffer[i]; + } + // Skip any spaces and punctuation that are part of the ignored sequence (e.g., "Game["][ ]Over", "Game[ ]["]Over") + else if (i + 3 <= end && ((is_special(buffer[i + 1]) && buffer[i + 2] == ' ' && is_ignore(buffer[i + 3])) || + (buffer[i + 1] == ' ' && is_special(buffer[i + 2]) && is_ignore(buffer[i + 3])))) { + i++; + ignore_part += buffer[i]; + i++; + ignore_part += buffer[i]; + } + // Skip any spaces that are part of the ignored sequence with special characters (e.g., "12[ ][+][ ]34") + else if (i + 4 <= end && buffer[i + 1] == ' ' && is_special(buffer[i + 2]) + && buffer[i + 3] == ' ' && is_ignore(buffer[i + 4])) { + i++; + ignore_part += buffer[i]; + i++; + ignore_part += buffer[i]; + i++; + ignore_part += buffer[i]; + } + // TODO: Ensure if the below condition is needed and correct + // Skip punctuation that is in end of line (e.g., "Game[:]") + else if (i + 1 == end && is_special(buffer[i + 1])) { + i++; + ignore_part += buffer[i]; + } + // Skip sequence of special characters header and numbers (e.g., "Level[ ][1][:][ ]12"). (The condition "Level 1[:][ ]12" is not enough to skip + else if (i + 5 <= end && buffer[i + 1] == ' ' && is_ignore(buffer[i + 2]) && is_special(buffer[i + 3]) + && buffer[i + 4] == ' ' && is_ignore(buffer[i + 5])){ + i++; + ignore_part += buffer[i]; + i++; + ignore_part += buffer[i]; + i++; + ignore_part += buffer[i]; + i++; + ignore_part += buffer[i]; + } + i++; + } + else { + break; + } + } + (*start) = i; + return ignore_part; +} + +Utf16String reverse_non_ignored(unichar_t *buffer, size_t *start, size_t end) +{ + // Reverse non-ignored sequences + size_t reverse_start = (*start); + while (reverse_start <= end) { + if (buffer[reverse_start] == ' ' && reverse_start + 1 <= end && is_ignore(buffer[reverse_start + 1]) + || is_special(buffer[reverse_start]) && reverse_start + 1 <= end && is_ignore(buffer[reverse_start + 1])) { + break; // space or special followed by ignored char (e.g., "[+]34", "[ ]Over") + } + reverse_start++; + } + size_t reverse_end = reverse_start - 1; + reverse_start = (*start); + Utf16String to_reverse; + if (reverse_start < reverse_end) { // 2 or more characters to reverse + for (size_t i = reverse_start; i <= reverse_end; ++i) { + to_reverse += buffer[i]; + } + + unichar_t *word_start = const_cast(to_reverse.Str()); + unichar_t *word_end = word_start + to_reverse.Get_Length() - 1; + + while (word_start < word_end) { + std::swap(*word_start, *word_end); + ++word_start; + --word_end; + } + } else + { + if (reverse_start == reverse_end) { // 1 character to reverse + to_reverse += buffer[reverse_start]; + } + } + (*start) = reverse_end + 1; + return to_reverse; +} + +Utf16String reverse_line(unichar_t *buffer, size_t *start, size_t end) +{ + Utf16String reversed; + while ((*start) <= end) { + + Utf16String was_space; + while ((*start) <= end && buffer[(*start)] == ' ') { + was_space += U_CHAR(' '); + (*start)++; + } + + Utf16String ignore_part = ignore_sequence(buffer, start, end); + ignore_part += reversed; // Add in the front + reversed = ignore_part; + + Utf16String reverse_part = reverse_non_ignored(buffer, start, end); + was_space += reversed; + reverse_part += was_space; + reversed = reverse_part; + + while ((*start) <= end && buffer[(*start)] == ' ') { + Utf16String temp = U_CHAR(" "); + temp += reversed; + reversed = temp; + (*start)++; + } + } + return reversed; } void Utf16String::Reverse() @@ -370,8 +510,8 @@ void Utf16String::Reverse() // will not be reversed. These characters will remain in their original order within the string. // // 2. If a special char like comma, period, space or colon appears between two "ignored" characters (e.g., in numbers - // like 1,000 | 25.14 | %.0f%% | 'New York' | %d.%02d.%d), the punctuation will be treated as part of the "ignored" - // sequence and will not be reversed. + // like 1,000 | 25.14 | %.0f%% | 'New York' | %d.%02d.%d), the punctuation will be treated as part of the + // "ignored" sequence and will not be reversed. // // 2.1. If a special char appears at the start or end of the sequence, // it will be treated as part of the "non-ignored" sequence (e.g., "Level : 1" -> "1 : leveL" when 'Level' is @@ -398,52 +538,42 @@ void Utf16String::Reverse() unichar_t *buffer = Get_Buffer_For_Read(len); - size_t start = 0; + size_t *start = new size_t; + *start = 0; size_t end = len - 1; - while (start <= end) { - // Skip "ignored" sequences and their associated special characters - if (is_ignored(buffer[start])) { - while (start <= end && is_ignored_or_special(buffer[start])) { - start++; - } - // Move back to the last valid "ignored" character - start--; - // Skip any trailing special characters that are part of the "ignored" sequence - while (start + 1 <= end && is_special_char(buffer[start + 1]) && is_ignored(buffer[start + 2])) { - start++; - } - // Skip any spaces that are part of the "ignored" sequence - while (start + 1 <= end && buffer[start + 1] == ' ' && is_ignored(buffer[start + 2])) { - start++; - } - // Skip any trailing special characters after spaces - while (start + 1 <= end && is_special_char(buffer[start + 1]) && is_ignored(buffer[start + 2])) { - start++; - } - // Move to the next character after the "ignored" sequence - start++; - continue; + // Buffer to hold the reversed parts of the string + Utf16String to_reversed; + Utf16String ignore_part; + while ((*start) <= end) { + // Skip breaks and new lines + Utf16String break_char; + while (buffer[(*start)] == '\n') { + break_char += U_CHAR("\n"); + (*start)++; } - // Reverse non-"ignored" sequences - size_t reverse_start = start; - while (start <= end && buffer[start] != '\n' && !is_ignored_or_special(buffer[start])) { - start++; + size_t end_line = (*start); + while (end_line <= end && buffer[end_line] != '\n') { + end_line++; } - size_t reverse_end = start - 1; - std::reverse(buffer + reverse_start, buffer + reverse_end + 1); + // sent the line to function to reverse it + Utf16String reversed = reverse_line(buffer, start, end_line - 1); + to_reversed += break_char; + to_reversed += reversed; + } - if (start <= end && !is_ignored(buffer[start])) { - start++; - } - if (start <= end && buffer[start] == ' ') { - start++; - } + // Update the buffer with the final result + if (ignore_part.Get_Length() > 0 && to_reversed.Is_Empty()) { + Set(ignore_part); + } else { + ignore_part += to_reversed; + to_reversed = ignore_part; + Set(to_reversed); } + delete start; } - void Utf16String::To_Lower() { unichar_t buf[MAX_FORMAT_BUF_LEN];