From 040a6bc65e60aa0cdb5d8bce0372dccd79c081f4 Mon Sep 17 00:00:00 2001 From: Roy Shilkrot Date: Wed, 20 Nov 2024 15:43:20 -0500 Subject: [PATCH 01/10] Add cloud translation support with multiple providers and configuration options --- CMakeLists.txt | 2 + data/locale/en-US.ini | 23 +- src/transcription-filter-callbacks.cpp | 36 +++ src/transcription-filter-data.h | 8 + src/transcription-filter-properties.cpp | 107 +++++++- src/transcription-filter.cpp | 8 + .../cloud-translation/CMakeLists.txt | 11 + .../cloud-translation/ITranslator.h | 24 ++ src/translation/cloud-translation/aws.cpp | 206 ++++++++++++++++ src/translation/cloud-translation/aws.h | 38 +++ src/translation/cloud-translation/azure.cpp | 113 +++++++++ src/translation/cloud-translation/azure.h | 24 ++ src/translation/cloud-translation/claude.cpp | 171 +++++++++++++ src/translation/cloud-translation/claude.h | 25 ++ .../cloud-translation/curl-helper.cpp | 88 +++++++ .../cloud-translation/curl-helper.h | 29 +++ src/translation/cloud-translation/deepl.cpp | 109 +++++++++ src/translation/cloud-translation/deepl.h | 20 ++ .../cloud-translation/google-cloud.cpp | 79 ++++++ .../cloud-translation/google-cloud.h | 20 ++ src/translation/cloud-translation/openai.cpp | 181 ++++++++++++++ src/translation/cloud-translation/openai.h | 25 ++ src/translation/cloud-translation/papago.cpp | 230 ++++++++++++++++++ src/translation/cloud-translation/papago.h | 23 ++ .../cloud-translation/translation-cloud.cpp | 54 ++++ .../cloud-translation/translation-cloud.h | 15 ++ 26 files changed, 1662 insertions(+), 7 deletions(-) create mode 100644 src/translation/cloud-translation/CMakeLists.txt create mode 100644 src/translation/cloud-translation/ITranslator.h create mode 100644 src/translation/cloud-translation/aws.cpp create mode 100644 src/translation/cloud-translation/aws.h create mode 100644 src/translation/cloud-translation/azure.cpp create mode 100644 src/translation/cloud-translation/azure.h create mode 100644 src/translation/cloud-translation/claude.cpp create mode 100644 src/translation/cloud-translation/claude.h create mode 100644 src/translation/cloud-translation/curl-helper.cpp create mode 100644 src/translation/cloud-translation/curl-helper.h create mode 100644 src/translation/cloud-translation/deepl.cpp create mode 100644 src/translation/cloud-translation/deepl.h create mode 100644 src/translation/cloud-translation/google-cloud.cpp create mode 100644 src/translation/cloud-translation/google-cloud.h create mode 100644 src/translation/cloud-translation/openai.cpp create mode 100644 src/translation/cloud-translation/openai.h create mode 100644 src/translation/cloud-translation/papago.cpp create mode 100644 src/translation/cloud-translation/papago.h create mode 100644 src/translation/cloud-translation/translation-cloud.cpp create mode 100644 src/translation/cloud-translation/translation-cloud.h diff --git a/CMakeLists.txt b/CMakeLists.txt index cea698e..c18ca66 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -127,6 +127,8 @@ target_sources( src/translation/translation-language-utils.cpp src/ui/filter-replace-dialog.cpp) +add_subdirectory(src/translation/cloud-translation) + set_target_properties_plugin(${CMAKE_PROJECT_NAME} PROPERTIES OUTPUT_NAME ${_name}) if(ENABLE_TESTS) diff --git a/data/locale/en-US.ini b/data/locale/en-US.ini index 58f15cf..215f2df 100644 --- a/data/locale/en-US.ini +++ b/data/locale/en-US.ini @@ -16,6 +16,8 @@ whisper_sampling_method="Whisper Sampling Method" n_threads="Number of threads" n_max_text_ctx="Max text context" translate="Translate" +translate_local="Local Translation" +translate_cloud="Cloud Translation" no_context="No context" single_segment="Single segment" print_special="Print special" @@ -75,6 +77,10 @@ general_group="General" transcription_group="Transcription" file_output_group="File Output Configuration" translate_explaination="Enabling translation will increase the processing load on your machine, This feature uses additional resources to translate content in real-time, which may impact performance. Learn More" +translate_cloud_explaination="Cloud translation requires an active internet connection and API keys to the translation provider." +translate_cloud_provider="Translation Provider" +translate_cloud_api_key="Access Key" +translate_cloud_secret_key="Secret Key" log_group="Logging" advanced_group="Advanced Configuration" buffered_output_parameters="Buffered Output Configuration" @@ -89,4 +95,19 @@ translate_only_full_sentences="Translate only full sentences" duration_filter_threshold="Duration filter" segment_duration="Segment duration" n_context_sentences="# Context sentences" -max_sub_duration="Max. sub duration (ms)" \ No newline at end of file +max_sub_duration="Max. sub duration (ms)" +Google-Cloud-Translation="Google Cloud Translation" +Microsoft-Translator="Microsoft Azure Translator" +Amazon-Translate="AWS Translate" +IBM-Watson-Translate="IBM Watson Translate" +Yandex-Translate="Yandex Translate" +Baidu-Translate="Baidu Translate" +Tencent-Translate="Tencent Translate" +Alibaba-Translate="Alibaba Translate" +Naver-Translate="Naver Translate" +Kakao-Translate="Kakao Translate" +Papago-Translate="Papago Translate" +Deepl-Translate="Deepl Translate" +Bing-Translate="Bing Translate" +OpenAI-Translate="OpenAI Translate" +Claude-Translate="Claude Translate" diff --git a/src/transcription-filter-callbacks.cpp b/src/transcription-filter-callbacks.cpp index 0095c54..2e7ec81 100644 --- a/src/transcription-filter-callbacks.cpp +++ b/src/transcription-filter-callbacks.cpp @@ -21,6 +21,7 @@ #include "whisper-utils/whisper-utils.h" #include "whisper-utils/whisper-model-utils.h" #include "translation/language_codes.h" +#include "translation/cloud-translation/translation-cloud.h" void send_caption_to_source(const std::string &target_source_name, const std::string &caption, struct transcription_filter_data *gf) @@ -80,6 +81,41 @@ std::string send_sentence_to_translation(const std::string &sentence, return ""; } +std::string send_text_to_cloud_translation(const std::string &sentence, + struct transcription_filter_data *gf, + const std::string &source_language) +{ + const std::string last_text = gf->last_text_for_translation; + gf->last_text_for_translation = sentence; + if (gf->translate_cloud && !sentence.empty()) { + obs_log(gf->log_level, "Translating text with cloud provider. %s -> %s", + source_language.c_str(), gf->target_lang.c_str()); + std::string translated_text; + if (sentence == last_text) { + // do not translate the same sentence twice + return gf->last_text_translation; + } + CloudTranslatorConfig config; + config.provider = gf->translate_cloud_provider; + config.access_key = gf->translate_cloud_api_key; + config.secret_key = gf->translate_cloud_secret_key; + + translated_text = translate_cloud( + config, sentence, gf->translate_cloud_target_language, source_language); + if (!translated_text.empty()) { + if (gf->log_words) { + obs_log(LOG_INFO, "Translation: '%s' -> '%s'", sentence.c_str(), + translated_text.c_str()); + } + gf->last_text_translation = translated_text; + return translated_text; + } else { + obs_log(gf->log_level, "Failed to translate text"); + } + } + return ""; +} + void send_sentence_to_file(struct transcription_filter_data *gf, const DetectionResultWithText &result, const std::string &str_copy, const std::string &translated_sentence) diff --git a/src/transcription-filter-data.h b/src/transcription-filter-data.h index e8990be..1c71aaa 100644 --- a/src/transcription-filter-data.h +++ b/src/transcription-filter-data.h @@ -89,6 +89,14 @@ struct transcription_filter_data { float duration_filter_threshold = 2.25f; int segment_duration = 7000; + // Cloud translation options + bool translate_cloud = false; + std::string translate_cloud_provider; + std::string translate_cloud_target_language; + std::string translate_cloud_output; + std::string translate_cloud_api_key; + std::string translate_cloud_secret_key; + // Last transcription result std::string last_text_for_translation; std::string last_text_translation; diff --git a/src/transcription-filter-properties.cpp b/src/transcription-filter-properties.cpp index 0adda33..f2b325f 100644 --- a/src/transcription-filter-properties.cpp +++ b/src/transcription-filter-properties.cpp @@ -43,6 +43,19 @@ bool translation_options_callback(obs_properties_t *props, obs_property_t *prope return true; } +bool translation_cloud_options_callback(obs_properties_t *props, obs_property_t *property, + obs_data_t *settings) +{ + UNUSED_PARAMETER(property); + // Show/Hide the cloud translation group options + const bool translate_enabled = obs_data_get_bool(settings, "translate_cloud"); + for (const auto &prop : {"translate_cloud_provider", "translate_cloud_target_language", + "translate_cloud_output", "translate_cloud_api_key"}) { + obs_property_set_visible(obs_properties_get(props, prop), translate_enabled); + } + return true; +} + bool advanced_settings_callback(obs_properties_t *props, obs_property_t *property, obs_data_t *settings) { @@ -55,6 +68,7 @@ bool advanced_settings_callback(obs_properties_t *props, obs_property_t *propert obs_property_set_visible(obs_properties_get(props, prop_name.c_str()), show_hide); } translation_options_callback(props, NULL, settings); + translation_cloud_options_callback(props, NULL, settings); return true; } @@ -174,12 +188,85 @@ void add_transcription_group_properties(obs_properties_t *ppts, obs_property_set_modified_callback2(whisper_models_list, external_model_file_selection, gf); } +void add_translation_cloud_group_properties(obs_properties_t *ppts) +{ + // add translation cloud group + obs_properties_t *translation_cloud_group = obs_properties_create(); + obs_property_t *translation_cloud_group_prop = + obs_properties_add_group(ppts, "translate_cloud", MT_("translate_cloud"), + OBS_GROUP_CHECKABLE, translation_cloud_group); + + obs_property_set_modified_callback(translation_cloud_group_prop, + translation_cloud_options_callback); + + // add explaination text + obs_properties_add_text(translation_cloud_group, "translate_cloud_explaination", + MT_("translate_cloud_explaination"), OBS_TEXT_INFO); + + // add cloud translation service provider selection + obs_property_t *prop_translate_cloud_provider = obs_properties_add_list( + translation_cloud_group, "translate_cloud_provider", + MT_("translate_cloud_provider"), OBS_COMBO_TYPE_LIST, OBS_COMBO_FORMAT_STRING); + // Populate the dropdown with the cloud translation service providers + obs_property_list_add_string(prop_translate_cloud_provider, MT_("Google-Cloud-Translation"), + "google-cloud-translation"); + obs_property_list_add_string(prop_translate_cloud_provider, MT_("Microsoft-Translator"), + "microsoft-translator"); + // obs_property_list_add_string(prop_translate_cloud_provider, MT_("Amazon-Translate"), + // "amazon-translate"); + // obs_property_list_add_string(prop_translate_cloud_provider, MT_("IBM-Watson-Translate"), + // "ibm-watson-translate"); + // obs_property_list_add_string(prop_translate_cloud_provider, MT_("Yandex-Translate"), + // "yandex-translate"); + // obs_property_list_add_string(prop_translate_cloud_provider, MT_("Baidu-Translate"), + // "baidu-translate"); + // obs_property_list_add_string(prop_translate_cloud_provider, MT_("Tencent-Translate"), + // "tencent-translate"); + // obs_property_list_add_string(prop_translate_cloud_provider, MT_("Alibaba-Translate"), + // "alibaba-translate"); + // obs_property_list_add_string(prop_translate_cloud_provider, MT_("Naver-Translate"), + // "naver-translate"); + // obs_property_list_add_string(prop_translate_cloud_provider, MT_("Kakao-Translate"), + // "kakao-translate"); + obs_property_list_add_string(prop_translate_cloud_provider, MT_("Papago-Translate"), + "papago-translate"); + obs_property_list_add_string(prop_translate_cloud_provider, MT_("Deepl-Translate"), + "deepl-translate"); + obs_property_list_add_string(prop_translate_cloud_provider, MT_("OpenAI-Translate"), + "openai-translate"); + obs_property_list_add_string(prop_translate_cloud_provider, MT_("Claude-Translate"), + "claude-translate"); + + // add target language selection + obs_property_t *prop_tgt = obs_properties_add_list( + translation_cloud_group, "translate_cloud_target_language", MT_("target_language"), + OBS_COMBO_TYPE_LIST, OBS_COMBO_FORMAT_STRING); + // Populate the dropdown with the language codes + for (const auto &language : language_codes) { + obs_property_list_add_string(prop_tgt, language.second.c_str(), + language.first.c_str()); + } + // add option for routing the translation to an output source + obs_property_t *prop_output = obs_properties_add_list( + translation_cloud_group, "translate_cloud_output", MT_("translate_output"), + OBS_COMBO_TYPE_LIST, OBS_COMBO_FORMAT_STRING); + obs_property_list_add_string(prop_output, "Write to captions output", "none"); + obs_enum_sources(add_sources_to_list, prop_output); + + // add input for API Key + obs_properties_add_text(translation_cloud_group, "translate_cloud_api_key", + MT_("translate_cloud_api_key"), OBS_TEXT_DEFAULT); + // add input for secret key + obs_properties_add_text(translation_cloud_group, "translate_cloud_secret_key", + MT_("translate_cloud_secret_key"), OBS_TEXT_PASSWORD); +} + void add_translation_group_properties(obs_properties_t *ppts) { // add translation option group obs_properties_t *translation_group = obs_properties_create(); obs_property_t *translation_group_prop = obs_properties_add_group( - ppts, "translate", MT_("translate"), OBS_GROUP_CHECKABLE, translation_group); + ppts, "translate", MT_("translate_local"), OBS_GROUP_CHECKABLE, translation_group); // add explaination text obs_properties_add_text(translation_group, "translate_explaination", @@ -541,6 +628,7 @@ obs_properties_t *transcription_filter_properties(void *data) add_general_group_properties(ppts); add_transcription_group_properties(ppts, gf); add_translation_group_properties(ppts); + add_translation_cloud_group_properties(ppts); add_file_output_group_properties(ppts); add_buffered_output_group_properties(ppts); add_advanced_group_properties(ppts, gf); @@ -586,6 +674,11 @@ void transcription_filter_defaults(obs_data_t *s) obs_data_set_default_int(s, "min_sub_duration", 1000); obs_data_set_default_int(s, "max_sub_duration", 3000); obs_data_set_default_bool(s, "advanced_settings", false); + obs_data_set_default_double(s, "sentence_psum_accept_thresh", 0.4); + obs_data_set_default_bool(s, "partial_group", true); + obs_data_set_default_int(s, "partial_latency", 1100); + + // translation options obs_data_set_default_bool(s, "translate", false); obs_data_set_default_string(s, "translate_target_language", "__es__"); obs_data_set_default_int(s, "translate_add_context", 1); @@ -593,11 +686,6 @@ void transcription_filter_defaults(obs_data_t *s) obs_data_set_default_string(s, "translate_model", "whisper-based-translation"); obs_data_set_default_string(s, "translation_model_path_external", ""); obs_data_set_default_int(s, "translate_input_tokenization_style", INPUT_TOKENIZAION_M2M100); - obs_data_set_default_double(s, "sentence_psum_accept_thresh", 0.4); - obs_data_set_default_bool(s, "partial_group", true); - obs_data_set_default_int(s, "partial_latency", 1100); - - // translation options obs_data_set_default_double(s, "translation_sampling_temperature", 0.1); obs_data_set_default_double(s, "translation_repetition_penalty", 2.0); obs_data_set_default_int(s, "translation_beam_size", 1); @@ -605,6 +693,13 @@ void transcription_filter_defaults(obs_data_t *s) obs_data_set_default_int(s, "translation_no_repeat_ngram_size", 1); obs_data_set_default_int(s, "translation_max_input_length", 65); + // cloud translation options + obs_data_set_default_bool(s, "translate_cloud", false); + obs_data_set_default_string(s, "translate_cloud_provider", "google-cloud-translation"); + obs_data_set_default_string(s, "translate_cloud_target_language", "en"); + obs_data_set_default_string(s, "translate_cloud_output", "none"); + obs_data_set_default_string(s, "translate_cloud_api_key", ""); + // Whisper parameters obs_data_set_default_int(s, "whisper_sampling_method", WHISPER_SAMPLING_BEAM_SEARCH); obs_data_set_default_int(s, "n_context_sentences", 0); diff --git a/src/transcription-filter.cpp b/src/transcription-filter.cpp index 9d4ebf5..8c46637 100644 --- a/src/transcription-filter.cpp +++ b/src/transcription-filter.cpp @@ -331,6 +331,14 @@ void transcription_filter_update(void *data, obs_data_t *s) } } + gf->translate_cloud = obs_data_get_bool(s, "translate_cloud"); + gf->translate_cloud_provider = obs_data_get_string(s, "translate_cloud_provider"); + gf->translate_cloud_target_language = + obs_data_get_string(s, "translate_cloud_target_language"); + gf->translate_cloud_output = obs_data_get_string(s, "translate_cloud_output"); + gf->translate_cloud_api_key = obs_data_get_string(s, "translate_cloud_api_key"); + gf->translate_cloud_secret_key = obs_data_get_string(s, "translate_cloud_secret_key"); + obs_log(gf->log_level, "update text source"); // update the text source const char *new_text_source_name = obs_data_get_string(s, "subtitle_sources"); diff --git a/src/translation/cloud-translation/CMakeLists.txt b/src/translation/cloud-translation/CMakeLists.txt new file mode 100644 index 0000000..dee765a --- /dev/null +++ b/src/translation/cloud-translation/CMakeLists.txt @@ -0,0 +1,11 @@ +# add source files +target_sources(${CMAKE_PROJECT_NAME} PRIVATE + # ${CMAKE_SOURCE_DIR}/src/translation/cloud-translation/aws.cpp + ${CMAKE_SOURCE_DIR}/src/translation/cloud-translation/azure.cpp + ${CMAKE_SOURCE_DIR}/src/translation/cloud-translation/claude.cpp + ${CMAKE_SOURCE_DIR}/src/translation/cloud-translation/curl-helper.cpp + ${CMAKE_SOURCE_DIR}/src/translation/cloud-translation/deepl.cpp + ${CMAKE_SOURCE_DIR}/src/translation/cloud-translation/google-cloud.cpp + ${CMAKE_SOURCE_DIR}/src/translation/cloud-translation/openai.cpp + ${CMAKE_SOURCE_DIR}/src/translation/cloud-translation/papago.cpp + ${CMAKE_SOURCE_DIR}/src/translation/cloud-translation/translation-cloud.cpp) diff --git a/src/translation/cloud-translation/ITranslator.h b/src/translation/cloud-translation/ITranslator.h new file mode 100644 index 0000000..6ac3388 --- /dev/null +++ b/src/translation/cloud-translation/ITranslator.h @@ -0,0 +1,24 @@ +#pragma once +#include +#include +#include + +// Custom exception +class TranslationError : public std::runtime_error { +public: + explicit TranslationError(const std::string &message) : std::runtime_error(message) {} +}; + +// Abstract translator interface +class ITranslator { +public: + virtual ~ITranslator() = default; + + virtual std::string translate(const std::string &text, const std::string &target_lang, + const std::string &source_lang = "auto") = 0; +}; + +// Factory function declaration +std::unique_ptr createTranslator(const std::string &provider, + const std::string &api_key, + const std::string &location = ""); diff --git a/src/translation/cloud-translation/aws.cpp b/src/translation/cloud-translation/aws.cpp new file mode 100644 index 0000000..8d1a63a --- /dev/null +++ b/src/translation/cloud-translation/aws.cpp @@ -0,0 +1,206 @@ +#include "aws.h" +#include "curl-helper.h" +#include +#include +#include +#include +#include +#include +#include + +using json = nlohmann::json; + +AWSTranslator::AWSTranslator(const std::string &access_key, const std::string &secret_key, + const std::string ®ion) + : access_key_(access_key), + secret_key_(secret_key), + region_(region), + curl_helper_(std::make_unique()) +{ +} + +AWSTranslator::~AWSTranslator() = default; + +// Helper function for SHA256 hashing +std::string AWSTranslator::sha256(const std::string &str) const +{ + unsigned char hash[SHA256_DIGEST_LENGTH]; + SHA256_CTX sha256; + SHA256_Init(&sha256); + SHA256_Update(&sha256, str.c_str(), str.size()); + SHA256_Final(hash, &sha256); + + std::stringstream ss; + for (int i = 0; i < SHA256_DIGEST_LENGTH; i++) { + ss << std::hex << std::setw(2) << std::setfill('0') << (int)hash[i]; + } + return ss.str(); +} + +// Helper function for HMAC-SHA256 +std::string AWSTranslator::hmacSha256(const std::string &key, const std::string &data) const +{ + unsigned char *digest = HMAC(EVP_sha256(), key.c_str(), key.length(), + (unsigned char *)data.c_str(), data.length(), nullptr, + nullptr); + + char hex[SHA256_DIGEST_LENGTH * 2 + 1]; + for (int i = 0; i < SHA256_DIGEST_LENGTH; i++) { + sprintf(&hex[i * 2], "%02x", digest[i]); + } + return std::string(hex, SHA256_DIGEST_LENGTH * 2); +} + +// Create AWS Signature Version 4 signing key +std::string AWSTranslator::createSigningKey(const std::string &date_stamp) const +{ + std::string k_date = hmacSha256("AWS4" + secret_key_, date_stamp); + std::string k_region = hmacSha256(k_date, region_); + std::string k_service = hmacSha256(k_region, SERVICE_NAME); + return hmacSha256(k_service, "aws4_request"); +} + +std::string AWSTranslator::calculateSignature(const std::string &string_to_sign, + const std::string &signing_key) const +{ + return hmacSha256(signing_key, string_to_sign); +} + +std::string AWSTranslator::getSignedHeaders(const std::map &headers) const +{ + std::stringstream ss; + for (auto it = headers.begin(); it != headers.end(); ++it) { + if (it != headers.begin()) + ss << ";"; + ss << it->first; + } + return ss.str(); +} + +std::string AWSTranslator::translate(const std::string &text, const std::string &target_lang, + const std::string &source_lang) +{ + std::unique_ptr curl(curl_easy_init(), + curl_easy_cleanup); + + if (!curl) { + throw TranslationError("Failed to initialize CURL session"); + } + + std::string response; + + try { + // Create request body + json request_body = {{"Text", text}, + {"TargetLanguageCode", target_lang}, + {"SourceLanguageCode", + source_lang == "auto" ? "auto" : source_lang}}; + std::string payload = request_body.dump(); + + // Get current timestamp + auto now = std::chrono::system_clock::now(); + auto time_t_now = std::chrono::system_clock::to_time_t(now); + + char amz_date[17]; + char date_stamp[9]; + strftime(amz_date, sizeof(amz_date), "%Y%m%dT%H%M%SZ", gmtime(&time_t_now)); + strftime(date_stamp, sizeof(date_stamp), "%Y%m%d", gmtime(&time_t_now)); + + // Create canonical request headers + std::map headers; + headers["content-type"] = "application/json"; + headers["host"] = "translate." + region_ + ".amazonaws.com"; + headers["x-amz-content-sha256"] = sha256(payload); + headers["x-amz-date"] = amz_date; + + // Create canonical request + std::stringstream canonical_request; + canonical_request << "POST\n" + << "/\n" // canonical URI + << "\n" // canonical query string (empty) + << "content-type:" << headers["content-type"] << "\n" + << "host:" << headers["host"] << "\n" + << "x-amz-content-sha256:" << headers["x-amz-content-sha256"] + << "\n" + << "x-amz-date:" << headers["x-amz-date"] << "\n" + << "\n" // end of headers + << getSignedHeaders(headers) << "\n" + << sha256(payload); + + // Create string to sign + std::stringstream string_to_sign; + string_to_sign << ALGORITHM << "\n" + << amz_date << "\n" + << date_stamp << "/" << region_ << "/" << SERVICE_NAME + << "/aws4_request\n" + << sha256(canonical_request.str()); + + // Calculate signature + std::string signing_key = createSigningKey(date_stamp); + std::string signature = calculateSignature(string_to_sign.str(), signing_key); + + // Create Authorization header + std::stringstream auth_header; + auth_header << ALGORITHM << " Credential=" << access_key_ << "/" << date_stamp + << "/" << region_ << "/" << SERVICE_NAME << "/aws4_request," + << "SignedHeaders=" << getSignedHeaders(headers) << "," + << "Signature=" << signature; + + // Set up CURL request + std::string url = "https://" + headers["host"] + "/"; + + curl_easy_setopt(curl.get(), CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl.get(), CURLOPT_POST, 1L); + curl_easy_setopt(curl.get(), CURLOPT_POSTFIELDS, payload.c_str()); + curl_easy_setopt(curl.get(), CURLOPT_WRITEFUNCTION, CurlHelper::WriteCallback); + curl_easy_setopt(curl.get(), CURLOPT_WRITEDATA, &response); + + // Set headers + struct curl_slist *header_list = nullptr; + header_list = curl_slist_append( + header_list, ("Content-Type: " + headers["content-type"]).c_str()); + header_list = curl_slist_append(header_list, + ("X-Amz-Date: " + headers["x-amz-date"]).c_str()); + header_list = curl_slist_append( + header_list, + ("X-Amz-Content-Sha256: " + headers["x-amz-content-sha256"]).c_str()); + header_list = curl_slist_append(header_list, + ("Authorization: " + auth_header.str()).c_str()); + + curl_easy_setopt(curl.get(), CURLOPT_HTTPHEADER, header_list); + + // Perform request + CURLcode res = curl_easy_perform(curl.get()); + + // Clean up + curl_slist_free_all(header_list); + + if (res != CURLE_OK) { + throw TranslationError(std::string("CURL request failed: ") + + curl_easy_strerror(res)); + } + + return parseResponse(response); + + } catch (const json::exception &e) { + throw TranslationError(std::string("JSON parsing error: ") + e.what()); + } +} + +std::string AWSTranslator::parseResponse(const std::string &response_str) +{ + try { + json response = json::parse(response_str); + + // Check for error response + if (response.contains("__type")) { + throw TranslationError("AWS API Error: " + + response.value("message", "Unknown error")); + } + + return response["TranslatedText"].get(); + + } catch (const json::exception &e) { + throw TranslationError(std::string("Failed to parse AWS response: ") + e.what()); + } +} diff --git a/src/translation/cloud-translation/aws.h b/src/translation/cloud-translation/aws.h new file mode 100644 index 0000000..ac9b478 --- /dev/null +++ b/src/translation/cloud-translation/aws.h @@ -0,0 +1,38 @@ +#pragma once +#include "ITranslator.h" +#include +#include +#include + +class CurlHelper; // Forward declaration + +class AWSTranslator : public ITranslator { +public: + AWSTranslator(const std::string &access_key, const std::string &secret_key, + const std::string ®ion = "us-east-1"); + ~AWSTranslator() override; + + std::string translate(const std::string &text, const std::string &target_lang, + const std::string &source_lang = "auto") override; + +private: + // AWS Signature V4 helper functions + std::string createSigningKey(const std::string &date_stamp) const; + std::string calculateSignature(const std::string &string_to_sign, + const std::string &signing_key) const; + std::string getSignedHeaders(const std::map &headers) const; + std::string sha256(const std::string &str) const; + std::string hmacSha256(const std::string &key, const std::string &data) const; + + // Response handling + std::string parseResponse(const std::string &response_str); + + std::string access_key_; + std::string secret_key_; + std::string region_; + std::unique_ptr curl_helper_; + + // AWS specific constants + const std::string SERVICE_NAME = "translate"; + const std::string ALGORITHM = "AWS4-HMAC-SHA256"; +}; diff --git a/src/translation/cloud-translation/azure.cpp b/src/translation/cloud-translation/azure.cpp new file mode 100644 index 0000000..e6671bf --- /dev/null +++ b/src/translation/cloud-translation/azure.cpp @@ -0,0 +1,113 @@ +#include "azure.h" +#include "curl-helper.h" +#include +#include + +using json = nlohmann::json; + +AzureTranslator::AzureTranslator(const std::string &api_key, const std::string &location, + const std::string &endpoint) + : api_key_(api_key), + location_(location), + endpoint_(endpoint), + curl_helper_(std::make_unique()) +{ +} + +AzureTranslator::~AzureTranslator() = default; + +std::string AzureTranslator::translate(const std::string &text, const std::string &target_lang, + const std::string &source_lang) +{ + std::unique_ptr curl(curl_easy_init(), + curl_easy_cleanup); + + if (!curl) { + throw TranslationError("Failed to initialize CURL session"); + } + + std::string response; + + try { + // Construct the route + std::stringstream route; + route << "/translate?api-version=3.0" + << "&to=" << target_lang; + + if (source_lang != "auto") { + route << "&from=" << source_lang; + } + + // Create the request body + json body = json::array({{{"Text", text}}}); + std::string requestBody = body.dump(); + + // Construct full URL + std::string url = endpoint_ + route.str(); + + // Set up curl options + curl_easy_setopt(curl.get(), CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl.get(), CURLOPT_WRITEFUNCTION, CurlHelper::WriteCallback); + curl_easy_setopt(curl.get(), CURLOPT_WRITEDATA, &response); + curl_easy_setopt(curl.get(), CURLOPT_SSL_VERIFYPEER, 1L); + curl_easy_setopt(curl.get(), CURLOPT_SSL_VERIFYHOST, 2L); + curl_easy_setopt(curl.get(), CURLOPT_TIMEOUT, 30L); + + // Set up POST request + curl_easy_setopt(curl.get(), CURLOPT_POST, 1L); + curl_easy_setopt(curl.get(), CURLOPT_POSTFIELDS, requestBody.c_str()); + + // Set up headers + struct curl_slist *headers = nullptr; + headers = curl_slist_append(headers, "Content-Type: application/json"); + + std::string auth_header = "Ocp-Apim-Subscription-Key: " + api_key_; + headers = curl_slist_append(headers, auth_header.c_str()); + + // Add location header if provided + if (!location_.empty()) { + std::string location_header = "Ocp-Apim-Subscription-Region: " + location_; + headers = curl_slist_append(headers, location_header.c_str()); + } + + curl_easy_setopt(curl.get(), CURLOPT_HTTPHEADER, headers); + + // Perform request + CURLcode res = curl_easy_perform(curl.get()); + + // Clean up headers + curl_slist_free_all(headers); + + if (res != CURLE_OK) { + throw TranslationError(std::string("CURL request failed: ") + + curl_easy_strerror(res)); + } + + return parseResponse(response); + + } catch (const json::exception &e) { + throw TranslationError(std::string("JSON parsing error: ") + e.what()); + } +} + +std::string AzureTranslator::parseResponse(const std::string &response_str) +{ + try { + json response = json::parse(response_str); + + // Check for error response + if (response.contains("error")) { + const auto &error = response["error"]; + throw TranslationError("Azure API Error: " + + error.value("message", "Unknown error")); + } + + // Azure returns an array of translations + // Each translation can have multiple target languages + // We'll take the first translation's first target + return response[0]["translations"][0]["text"].get(); + + } catch (const json::exception &e) { + throw TranslationError(std::string("Failed to parse Azure response: ") + e.what()); + } +} diff --git a/src/translation/cloud-translation/azure.h b/src/translation/cloud-translation/azure.h new file mode 100644 index 0000000..a350044 --- /dev/null +++ b/src/translation/cloud-translation/azure.h @@ -0,0 +1,24 @@ +#pragma once +#include "ITranslator.h" +#include + +class CurlHelper; // Forward declaration + +class AzureTranslator : public ITranslator { +public: + AzureTranslator( + const std::string &api_key, const std::string &location = "", + const std::string &endpoint = "https://api.cognitive.microsofttranslator.com"); + ~AzureTranslator() override; + + std::string translate(const std::string &text, const std::string &target_lang, + const std::string &source_lang = "auto") override; + +private: + std::string parseResponse(const std::string &response_str); + + std::string api_key_; + std::string location_; + std::string endpoint_; + std::unique_ptr curl_helper_; +}; diff --git a/src/translation/cloud-translation/claude.cpp b/src/translation/cloud-translation/claude.cpp new file mode 100644 index 0000000..e019be9 --- /dev/null +++ b/src/translation/cloud-translation/claude.cpp @@ -0,0 +1,171 @@ +#include "claude.h" +#include "curl-helper.h" +#include +#include +#include +#include + +using json = nlohmann::json; + +ClaudeTranslator::ClaudeTranslator(const std::string &api_key, const std::string &model) + : api_key_(api_key), + model_(model), + curl_helper_(std::make_unique()) +{ +} + +ClaudeTranslator::~ClaudeTranslator() = default; + +std::string ClaudeTranslator::getLanguageName(const std::string &lang_code) const +{ + static const std::unordered_map language_names = { + {"auto", "auto-detected language"}, + {"en", "English"}, + {"es", "Spanish"}, + {"fr", "French"}, + {"de", "German"}, + {"it", "Italian"}, + {"pt", "Portuguese"}, + {"nl", "Dutch"}, + {"pl", "Polish"}, + {"ru", "Russian"}, + {"ja", "Japanese"}, + {"ko", "Korean"}, + {"zh", "Chinese"}, + {"ar", "Arabic"}, + {"hi", "Hindi"}, + {"bn", "Bengali"}, + {"uk", "Ukrainian"}, + {"vi", "Vietnamese"}, + {"th", "Thai"}, + {"tr", "Turkish"}, + // Add more languages as needed + }; + + auto it = language_names.find(lang_code); + if (it != language_names.end()) { + return it->second; + } + return lang_code; // Return the code itself if no mapping exists +} + +bool ClaudeTranslator::isLanguageSupported(const std::string &lang_code) const +{ + static const std::unordered_set supported_languages = { + "auto", "en", "es", "fr", "de", "it", "pt", "nl", "pl", "ru", "ja", + "ko", "zh", "ar", "hi", "bn", "uk", "vi", "th", "tr" + // Add more supported languages as needed + }; + + return supported_languages.find(lang_code) != supported_languages.end(); +} + +std::string ClaudeTranslator::createSystemPrompt(const std::string &target_lang) const +{ + std::string target_language = getLanguageName(target_lang); + + return "You are a professional translator. Translate the user's text into " + + target_language + " while preserving the meaning, tone, and style. " + + "Provide only the translated text without explanations, notes, or any other content. " + + "Maintain any formatting, line breaks, or special characters from the original text."; +} + +std::string ClaudeTranslator::translate(const std::string &text, const std::string &target_lang, + const std::string &source_lang) +{ + if (!isLanguageSupported(target_lang)) { + throw TranslationError("Unsupported target language: " + target_lang); + } + + if (source_lang != "auto" && !isLanguageSupported(source_lang)) { + throw TranslationError("Unsupported source language: " + source_lang); + } + + std::unique_ptr curl(curl_easy_init(), + curl_easy_cleanup); + + if (!curl) { + throw TranslationError("Failed to initialize CURL session"); + } + + std::string response; + + try { + // Prepare the request + std::string url = "https://api.anthropic.com/v1/messages"; + + // Create request body + json request_body = {{"model", model_}, + {"max_tokens", 4096}, + {"system", createSystemPrompt(target_lang)}, + {"messages", + json::array({{{"role", "user"}, {"content", text}}})}}; + + if (source_lang != "auto") { + request_body["system"] = createSystemPrompt(target_lang) + + " The source text is in " + + getLanguageName(source_lang) + "."; + } + + std::string payload = request_body.dump(); + + // Set up headers + struct curl_slist *headers = nullptr; + headers = curl_slist_append(headers, "Content-Type: application/json"); + headers = curl_slist_append(headers, ("x-api-key: " + api_key_).c_str()); + headers = curl_slist_append(headers, "anthropic-version: 2023-06-01"); + + // Set up CURL request + curl_easy_setopt(curl.get(), CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl.get(), CURLOPT_POST, 1L); + curl_easy_setopt(curl.get(), CURLOPT_POSTFIELDS, payload.c_str()); + curl_easy_setopt(curl.get(), CURLOPT_HTTPHEADER, headers); + curl_easy_setopt(curl.get(), CURLOPT_WRITEFUNCTION, CurlHelper::WriteCallback); + curl_easy_setopt(curl.get(), CURLOPT_WRITEDATA, &response); + curl_easy_setopt(curl.get(), CURLOPT_SSL_VERIFYPEER, 1L); + curl_easy_setopt(curl.get(), CURLOPT_SSL_VERIFYHOST, 2L); + curl_easy_setopt(curl.get(), CURLOPT_TIMEOUT, 30L); + + // Perform request + CURLcode res = curl_easy_perform(curl.get()); + + // Clean up + curl_slist_free_all(headers); + + if (res != CURLE_OK) { + throw TranslationError(std::string("CURL request failed: ") + + curl_easy_strerror(res)); + } + + // Check HTTP response code + long response_code; + curl_easy_getinfo(curl.get(), CURLINFO_RESPONSE_CODE, &response_code); + + if (response_code != 200) { + throw TranslationError("HTTP error: " + std::to_string(response_code) + + "\nResponse: " + response); + } + + return parseResponse(response); + + } catch (const json::exception &e) { + throw TranslationError(std::string("JSON parsing error: ") + e.what()); + } +} + +std::string ClaudeTranslator::parseResponse(const std::string &response_str) +{ + try { + json response = json::parse(response_str); + + if (!response.contains("content") || !response["content"].is_array() || + response["content"].empty() || !response["content"][0].contains("text")) { + throw TranslationError("Invalid response format from Claude API"); + } + + return response["content"][0]["text"].get(); + + } catch (const json::exception &e) { + throw TranslationError(std::string("Failed to parse Claude response: ") + e.what()); + } +} diff --git a/src/translation/cloud-translation/claude.h b/src/translation/cloud-translation/claude.h new file mode 100644 index 0000000..3761058 --- /dev/null +++ b/src/translation/cloud-translation/claude.h @@ -0,0 +1,25 @@ +#pragma once +#include "ITranslator.h" +#include + +class CurlHelper; // Forward declaration + +class ClaudeTranslator : public ITranslator { +public: + explicit ClaudeTranslator(const std::string &api_key, + const std::string &model = "claude-3-sonnet-20240229"); + ~ClaudeTranslator() override; + + std::string translate(const std::string &text, const std::string &target_lang, + const std::string &source_lang = "auto") override; + +private: + std::string parseResponse(const std::string &response_str); + std::string createSystemPrompt(const std::string &target_lang) const; + std::string getLanguageName(const std::string &lang_code) const; + bool isLanguageSupported(const std::string &lang_code) const; + + std::string api_key_; + std::string model_; + std::unique_ptr curl_helper_; +}; diff --git a/src/translation/cloud-translation/curl-helper.cpp b/src/translation/cloud-translation/curl-helper.cpp new file mode 100644 index 0000000..9073e58 --- /dev/null +++ b/src/translation/cloud-translation/curl-helper.cpp @@ -0,0 +1,88 @@ +#include "curl-helper.h" +#include +#include + +bool CurlHelper::is_initialized_ = false; +std::mutex CurlHelper::curl_mutex_; + +CurlHelper::CurlHelper() +{ + std::lock_guard lock(curl_mutex_); + if (!is_initialized_) { + if (curl_global_init(CURL_GLOBAL_DEFAULT) != CURLE_OK) { + throw TranslationError("Failed to initialize CURL"); + } + is_initialized_ = true; + } +} + +CurlHelper::~CurlHelper() +{ + // Don't call curl_global_cleanup() in destructor + // Let it clean up when the program exits +} + +size_t CurlHelper::WriteCallback(void *contents, size_t size, size_t nmemb, void *userp) +{ + if (!userp) { + return 0; + } + + size_t realsize = size * nmemb; + auto *str = static_cast(userp); + try { + str->append(static_cast(contents), realsize); + return realsize; + } catch (const std::exception &) { + return 0; // Return 0 to indicate error to libcurl + } +} + +std::string CurlHelper::urlEncode(CURL *curl, const std::string &value) +{ + if (!curl) { + throw TranslationError("Invalid CURL handle for URL encoding"); + } + + std::unique_ptr escaped( + curl_easy_escape(curl, value.c_str(), value.length()), curl_free); + + if (!escaped) { + throw TranslationError("Failed to URL encode string"); + } + + return std::string(escaped.get()); +} + +struct curl_slist *CurlHelper::createBasicHeaders(CURL *curl, const std::string &content_type) +{ + struct curl_slist *headers = nullptr; + + try { + headers = curl_slist_append(headers, ("Content-Type: " + content_type).c_str()); + + if (!headers) { + throw TranslationError("Failed to create HTTP headers"); + } + + return headers; + } catch (...) { + if (headers) { + curl_slist_free_all(headers); + } + throw; + } +} + +void CurlHelper::setSSLVerification(CURL *curl, bool verify) +{ + if (!curl) { + throw TranslationError("Invalid CURL handle for SSL configuration"); + } + + long verify_peer = verify ? 1L : 0L; + long verify_host = verify ? 2L : 0L; + + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, verify_peer); + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, verify_host); +} diff --git a/src/translation/cloud-translation/curl-helper.h b/src/translation/cloud-translation/curl-helper.h new file mode 100644 index 0000000..a227ef6 --- /dev/null +++ b/src/translation/cloud-translation/curl-helper.h @@ -0,0 +1,29 @@ +#pragma once +#include +#include + +#include +#include "ITranslator.h" + +class CurlHelper { +public: + CurlHelper(); + ~CurlHelper(); + + // Callback for writing response data + static size_t WriteCallback(void *contents, size_t size, size_t nmemb, void *userp); + + // URL encode a string + static std::string urlEncode(CURL *curl, const std::string &value); + + // Common request builders + static struct curl_slist * + createBasicHeaders(CURL *curl, const std::string &content_type = "application/json"); + + // Verify HTTPS certificate + static void setSSLVerification(CURL *curl, bool verify = true); + +private: + static bool is_initialized_; + static std::mutex curl_mutex_; // For thread-safe global initialization +}; diff --git a/src/translation/cloud-translation/deepl.cpp b/src/translation/cloud-translation/deepl.cpp new file mode 100644 index 0000000..f9c775e --- /dev/null +++ b/src/translation/cloud-translation/deepl.cpp @@ -0,0 +1,109 @@ +#include "deepl.h" +#include "curl-helper.h" +#include +#include + +using json = nlohmann::json; + +DeepLTranslator::DeepLTranslator(const std::string &api_key) + : api_key_(api_key), + curl_helper_(std::make_unique()) +{ +} + +DeepLTranslator::~DeepLTranslator() = default; + +std::string DeepLTranslator::translate(const std::string &text, const std::string &target_lang, + const std::string &source_lang) +{ + std::unique_ptr curl(curl_easy_init(), + curl_easy_cleanup); + + if (!curl) { + throw TranslationError("Failed to initialize CURL session"); + } + + std::string response; + + try { + // Construct URL with parameters + // Note: DeepL uses uppercase language codes + std::string upperTarget = target_lang; + std::string upperSource = source_lang; + for (char &c : upperTarget) + c = std::toupper(c); + for (char &c : upperSource) + c = std::toupper(c); + + std::stringstream url; + url << "https://api-free.deepl.com/v2/translate" + << "?auth_key=" << api_key_ + << "&text=" << CurlHelper::urlEncode(curl.get(), text) + << "&target_lang=" << upperTarget; + + if (upperSource != "AUTO") { + url << "&source_lang=" << upperSource; + } + + // Set up curl options + curl_easy_setopt(curl.get(), CURLOPT_URL, url.str().c_str()); + curl_easy_setopt(curl.get(), CURLOPT_WRITEFUNCTION, CurlHelper::WriteCallback); + curl_easy_setopt(curl.get(), CURLOPT_WRITEDATA, &response); + curl_easy_setopt(curl.get(), CURLOPT_SSL_VERIFYPEER, 1L); + curl_easy_setopt(curl.get(), CURLOPT_SSL_VERIFYHOST, 2L); + curl_easy_setopt(curl.get(), CURLOPT_TIMEOUT, 30L); + + // DeepL requires specific headers + struct curl_slist *headers = nullptr; + headers = curl_slist_append(headers, + "Content-Type: application/x-www-form-urlencoded"); + curl_easy_setopt(curl.get(), CURLOPT_HTTPHEADER, headers); + + CURLcode res = curl_easy_perform(curl.get()); + + // Clean up headers + curl_slist_free_all(headers); + + if (res != CURLE_OK) { + throw TranslationError(std::string("CURL request failed: ") + + curl_easy_strerror(res)); + } + + return parseResponse(response); + + } catch (const json::exception &e) { + throw TranslationError(std::string("JSON parsing error: ") + e.what()); + } +} + +std::string DeepLTranslator::parseResponse(const std::string &response_str) +{ + json response = json::parse(response_str); + + // Check for API errors + if (response.contains("message")) { + throw TranslationError("DeepL API Error: " + + response["message"].get()); + } + + // Handle rate limiting errors + long response_code; + curl_easy_getinfo(curl_easy_init(), CURLINFO_RESPONSE_CODE, &response_code); + if (response_code == 429) { + throw TranslationError("DeepL API Error: Rate limit exceeded"); + } + + try { + // DeepL returns translations array with detected language + const auto &translation = response["translations"][0]; + + // Optionally, you can access the detected source language + // if (translation.contains("detected_source_language")) { + // std::string detected = translation["detected_source_language"]; + // } + + return translation["text"].get(); + } catch (const json::exception &) { + throw TranslationError("Unexpected response format from DeepL API"); + } +} diff --git a/src/translation/cloud-translation/deepl.h b/src/translation/cloud-translation/deepl.h new file mode 100644 index 0000000..9f661af --- /dev/null +++ b/src/translation/cloud-translation/deepl.h @@ -0,0 +1,20 @@ +#pragma once +#include "ITranslator.h" +#include + +class CurlHelper; // Forward declaration + +class DeepLTranslator : public ITranslator { +public: + explicit DeepLTranslator(const std::string &api_key); + ~DeepLTranslator() override; + + std::string translate(const std::string &text, const std::string &target_lang, + const std::string &source_lang = "auto") override; + +private: + std::string parseResponse(const std::string &response_str); + + std::string api_key_; + std::unique_ptr curl_helper_; +}; diff --git a/src/translation/cloud-translation/google-cloud.cpp b/src/translation/cloud-translation/google-cloud.cpp new file mode 100644 index 0000000..65ff7c5 --- /dev/null +++ b/src/translation/cloud-translation/google-cloud.cpp @@ -0,0 +1,79 @@ +#include "google-cloud.h" +#include "curl-helper.h" +#include +#include + +using json = nlohmann::json; + +GoogleTranslator::GoogleTranslator(const std::string &api_key) + : api_key_(api_key), + curl_helper_(std::make_unique()) +{ +} + +GoogleTranslator::~GoogleTranslator() = default; + +std::string GoogleTranslator::translate(const std::string &text, const std::string &target_lang, + const std::string &source_lang) +{ + std::unique_ptr curl(curl_easy_init(), + curl_easy_cleanup); + + if (!curl) { + throw TranslationError("Failed to initialize CURL session"); + } + + std::string response; + + try { + // Construct URL with parameters + std::stringstream url; + url << "https://translation.googleapis.com/language/translate/v2" + << "?key=" << api_key_ << "&q=" << CurlHelper::urlEncode(curl.get(), text) + << "&target=" << target_lang; + + if (source_lang != "auto") { + url << "&source=" << source_lang; + } + + // Set up curl options + curl_easy_setopt(curl.get(), CURLOPT_URL, url.str().c_str()); + curl_easy_setopt(curl.get(), CURLOPT_WRITEFUNCTION, CurlHelper::WriteCallback); + curl_easy_setopt(curl.get(), CURLOPT_WRITEDATA, &response); + curl_easy_setopt(curl.get(), CURLOPT_SSL_VERIFYPEER, 1L); + curl_easy_setopt(curl.get(), CURLOPT_SSL_VERIFYHOST, 2L); + curl_easy_setopt(curl.get(), CURLOPT_TIMEOUT, 30L); + + CURLcode res = curl_easy_perform(curl.get()); + + if (res != CURLE_OK) { + throw TranslationError(std::string("CURL request failed: ") + + curl_easy_strerror(res)); + } + + return parseResponse(response); + + } catch (const json::exception &e) { + throw TranslationError(std::string("JSON parsing error: ") + e.what()); + } +} + +std::string GoogleTranslator::parseResponse(const std::string &response_str) +{ + json response = json::parse(response_str); + + if (response.contains("error")) { + const auto &error = response["error"]; + std::stringstream error_msg; + error_msg << "Google API Error: "; + if (error.contains("message")) { + error_msg << error["message"].get(); + } + if (error.contains("code")) { + error_msg << " (Code: " << error["code"].get() << ")"; + } + throw TranslationError(error_msg.str()); + } + + return response["data"]["translations"][0]["translatedText"].get(); +} diff --git a/src/translation/cloud-translation/google-cloud.h b/src/translation/cloud-translation/google-cloud.h new file mode 100644 index 0000000..30417d1 --- /dev/null +++ b/src/translation/cloud-translation/google-cloud.h @@ -0,0 +1,20 @@ +#pragma once +#include "ITranslator.h" +#include + +class CurlHelper; // Forward declaration + +class GoogleTranslator : public ITranslator { +public: + explicit GoogleTranslator(const std::string &api_key); + ~GoogleTranslator() override; + + std::string translate(const std::string &text, const std::string &target_lang, + const std::string &source_lang = "auto") override; + +private: + std::string parseResponse(const std::string &response_str); + + std::string api_key_; + std::unique_ptr curl_helper_; +}; diff --git a/src/translation/cloud-translation/openai.cpp b/src/translation/cloud-translation/openai.cpp new file mode 100644 index 0000000..7d7cda8 --- /dev/null +++ b/src/translation/cloud-translation/openai.cpp @@ -0,0 +1,181 @@ +#include "openai.h" +#include "curl-helper.h" +#include +#include +#include +#include + +using json = nlohmann::json; + +OpenAITranslator::OpenAITranslator(const std::string &api_key, const std::string &model) + : api_key_(api_key), + model_(model), + curl_helper_(std::make_unique()) +{ +} + +OpenAITranslator::~OpenAITranslator() = default; + +std::string OpenAITranslator::getLanguageName(const std::string &lang_code) const +{ + static const std::unordered_map language_names = { + {"auto", "auto-detected language"}, + {"en", "English"}, + {"es", "Spanish"}, + {"fr", "French"}, + {"de", "German"}, + {"it", "Italian"}, + {"pt", "Portuguese"}, + {"nl", "Dutch"}, + {"pl", "Polish"}, + {"ru", "Russian"}, + {"ja", "Japanese"}, + {"ko", "Korean"}, + {"zh", "Chinese"}, + {"ar", "Arabic"}, + {"hi", "Hindi"}, + {"bn", "Bengali"}, + {"uk", "Ukrainian"}, + {"vi", "Vietnamese"}, + {"th", "Thai"}, + {"tr", "Turkish"}, + // Add more languages as needed + }; + + auto it = language_names.find(lang_code); + if (it != language_names.end()) { + return it->second; + } + return lang_code; // Return the code itself if no mapping exists +} + +bool OpenAITranslator::isLanguageSupported(const std::string &lang_code) const +{ + static const std::unordered_set supported_languages = { + "auto", "en", "es", "fr", "de", "it", "pt", "nl", "pl", "ru", "ja", + "ko", "zh", "ar", "hi", "bn", "uk", "vi", "th", "tr" + // Add more supported languages as needed + }; + + return supported_languages.find(lang_code) != supported_languages.end(); +} + +std::string OpenAITranslator::createSystemPrompt(const std::string &target_lang) const +{ + std::string target_language = getLanguageName(target_lang); + + return "You are a professional translator. Translate the user's text into " + + target_language + ". Maintain the exact meaning, tone, and style. " + + "Respond with only the translated text, without any explanations or additional content. " + + "Preserve all formatting, line breaks, and special characters from the original text."; +} + +std::string OpenAITranslator::translate(const std::string &text, const std::string &target_lang, + const std::string &source_lang) +{ + if (!isLanguageSupported(target_lang)) { + throw TranslationError("Unsupported target language: " + target_lang); + } + + if (source_lang != "auto" && !isLanguageSupported(source_lang)) { + throw TranslationError("Unsupported source language: " + source_lang); + } + + std::unique_ptr curl(curl_easy_init(), + curl_easy_cleanup); + + if (!curl) { + throw TranslationError("Failed to initialize CURL session"); + } + + std::string response; + + try { + // Prepare the request + std::string url = "https://api.openai.com/v1/chat/completions"; + + // Create messages array + json messages = json::array(); + + // Add system message + messages.push_back( + {{"role", "system"}, {"content", createSystemPrompt(target_lang)}}); + + // Add user message with source language if specified + std::string user_prompt = text; + if (source_lang != "auto") { + user_prompt = "Translate the following " + getLanguageName(source_lang) + + " text:\n\n" + text; + } + + messages.push_back({{"role", "user"}, {"content", user_prompt}}); + + // Create request body + json request_body = {{"model", model_}, + {"messages", messages}, + {"temperature", + 0.3}, // Lower temperature for more consistent translations + {"max_tokens", 4000}}; + + std::string payload = request_body.dump(); + + // Set up headers + struct curl_slist *headers = nullptr; + headers = curl_slist_append(headers, "Content-Type: application/json"); + headers = curl_slist_append(headers, ("Authorization: Bearer " + api_key_).c_str()); + + // Set up CURL request + curl_easy_setopt(curl.get(), CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl.get(), CURLOPT_POST, 1L); + curl_easy_setopt(curl.get(), CURLOPT_POSTFIELDS, payload.c_str()); + curl_easy_setopt(curl.get(), CURLOPT_HTTPHEADER, headers); + curl_easy_setopt(curl.get(), CURLOPT_WRITEFUNCTION, CurlHelper::WriteCallback); + curl_easy_setopt(curl.get(), CURLOPT_WRITEDATA, &response); + curl_easy_setopt(curl.get(), CURLOPT_SSL_VERIFYPEER, 1L); + curl_easy_setopt(curl.get(), CURLOPT_SSL_VERIFYHOST, 2L); + curl_easy_setopt(curl.get(), CURLOPT_TIMEOUT, 30L); + + // Perform request + CURLcode res = curl_easy_perform(curl.get()); + + // Clean up + curl_slist_free_all(headers); + + if (res != CURLE_OK) { + throw TranslationError(std::string("CURL request failed: ") + + curl_easy_strerror(res)); + } + + // Check HTTP response code + long response_code; + curl_easy_getinfo(curl.get(), CURLINFO_RESPONSE_CODE, &response_code); + + if (response_code != 200) { + throw TranslationError("HTTP error: " + std::to_string(response_code) + + "\nResponse: " + response); + } + + return parseResponse(response); + + } catch (const json::exception &e) { + throw TranslationError(std::string("JSON parsing error: ") + e.what()); + } +} + +std::string OpenAITranslator::parseResponse(const std::string &response_str) +{ + try { + json response = json::parse(response_str); + + if (!response.contains("choices") || response["choices"].empty() || + !response["choices"][0].contains("message") || + !response["choices"][0]["message"].contains("content")) { + throw TranslationError("Invalid response format from OpenAI API"); + } + + return response["choices"][0]["message"]["content"].get(); + + } catch (const json::exception &e) { + throw TranslationError(std::string("Failed to parse OpenAI response: ") + e.what()); + } +} diff --git a/src/translation/cloud-translation/openai.h b/src/translation/cloud-translation/openai.h new file mode 100644 index 0000000..a82f811 --- /dev/null +++ b/src/translation/cloud-translation/openai.h @@ -0,0 +1,25 @@ +#pragma once +#include "ITranslator.h" +#include + +class CurlHelper; // Forward declaration + +class OpenAITranslator : public ITranslator { +public: + explicit OpenAITranslator(const std::string &api_key, + const std::string &model = "gpt-4-turbo-preview"); + ~OpenAITranslator() override; + + std::string translate(const std::string &text, const std::string &target_lang, + const std::string &source_lang = "auto") override; + +private: + std::string parseResponse(const std::string &response_str); + std::string createSystemPrompt(const std::string &target_lang) const; + std::string getLanguageName(const std::string &lang_code) const; + bool isLanguageSupported(const std::string &lang_code) const; + + std::string api_key_; + std::string model_; + std::unique_ptr curl_helper_; +}; diff --git a/src/translation/cloud-translation/papago.cpp b/src/translation/cloud-translation/papago.cpp new file mode 100644 index 0000000..041fded --- /dev/null +++ b/src/translation/cloud-translation/papago.cpp @@ -0,0 +1,230 @@ +#include "papago.h" +#include "curl-helper.h" +#include +#include +#include +#include + +using json = nlohmann::json; + +// Language pair support mapping +struct LanguagePairHash { + size_t operator()(const std::pair &p) const + { + return std::hash()(p.first + p.second); + } +}; + +PapagoTranslator::PapagoTranslator(const std::string &client_id, const std::string &client_secret) + : client_id_(client_id), + client_secret_(client_secret), + curl_helper_(std::make_unique()) +{ +} + +PapagoTranslator::~PapagoTranslator() = default; + +std::string PapagoTranslator::mapLanguageCode(const std::string &lang_code) const +{ + // Map common ISO language codes to Papago codes + static const std::unordered_map code_map = { + {"auto", "auto"}, {"ko", "ko"}, // Korean + {"en", "en"}, // English + {"ja", "ja"}, // Japanese + {"zh", "zh-CN"}, // Chinese (Simplified) + {"zh-CN", "zh-CN"}, // Chinese (Simplified) + {"zh-TW", "zh-TW"}, // Chinese (Traditional) + {"vi", "vi"}, // Vietnamese + {"th", "th"}, // Thai + {"id", "id"}, // Indonesian + {"fr", "fr"}, // French + {"es", "es"}, // Spanish + {"ru", "ru"}, // Russian + {"de", "de"}, // German + {"it", "it"} // Italian + }; + + auto it = code_map.find(lang_code); + if (it != code_map.end()) { + return it->second; + } + throw TranslationError("Unsupported language code: " + lang_code); +} + +bool PapagoTranslator::isLanguagePairSupported(const std::string &source, + const std::string &target) const +{ + static const std::unordered_set, LanguagePairHash> + supported_pairs = {// Korean pairs + {"ko", "en"}, + {"en", "ko"}, + {"ko", "ja"}, + {"ja", "ko"}, + {"ko", "zh-CN"}, + {"zh-CN", "ko"}, + {"ko", "zh-TW"}, + {"zh-TW", "ko"}, + {"ko", "vi"}, + {"vi", "ko"}, + {"ko", "th"}, + {"th", "ko"}, + {"ko", "id"}, + {"id", "ko"}, + {"ko", "fr"}, + {"fr", "ko"}, + {"ko", "es"}, + {"es", "ko"}, + {"ko", "ru"}, + {"ru", "ko"}, + {"ko", "de"}, + {"de", "ko"}, + {"ko", "it"}, + {"it", "ko"}, + + // English pairs + {"en", "ja"}, + {"ja", "en"}, + {"en", "zh-CN"}, + {"zh-CN", "en"}, + {"en", "zh-TW"}, + {"zh-TW", "en"}, + {"en", "vi"}, + {"vi", "en"}, + {"en", "th"}, + {"th", "en"}, + {"en", "id"}, + {"id", "en"}, + {"en", "fr"}, + {"fr", "en"}, + {"en", "es"}, + {"es", "en"}, + {"en", "ru"}, + {"ru", "en"}, + {"en", "de"}, + {"de", "en"}, + + // Japanese pairs + {"ja", "zh-CN"}, + {"zh-CN", "ja"}, + {"ja", "zh-TW"}, + {"zh-TW", "ja"}, + {"ja", "vi"}, + {"vi", "ja"}, + {"ja", "th"}, + {"th", "ja"}, + {"ja", "id"}, + {"id", "ja"}, + {"ja", "fr"}, + {"fr", "ja"}, + + // Chinese pairs + {"zh-CN", "zh-TW"}, + {"zh-TW", "zh-CN"}}; + + // Special case for auto detection + if (source == "auto") { + return true; + } + + return supported_pairs.find({source, target}) != supported_pairs.end(); +} + +std::string PapagoTranslator::translate(const std::string &text, const std::string &target_lang, + const std::string &source_lang) +{ + if (text.length() > 5000) { + throw TranslationError("Text exceeds maximum length of 5000 characters"); + } + + std::string papago_source = mapLanguageCode(source_lang); + std::string papago_target = mapLanguageCode(target_lang); + + if (!isLanguagePairSupported(papago_source, papago_target)) { + throw TranslationError("Unsupported language pair: " + source_lang + " to " + + target_lang); + } + + std::unique_ptr curl(curl_easy_init(), + curl_easy_cleanup); + + if (!curl) { + throw TranslationError("Failed to initialize CURL session"); + } + + std::string response; + + try { + // Prepare request data + std::string url = "https://naveropenapi.apigw.ntruss.com/nmt/v1/translation"; + + // Create request body + json request_body = {{"source", papago_source}, + {"target", papago_target}, + {"text", text}}; + std::string payload = request_body.dump(); + + // Set up headers + struct curl_slist *headers = nullptr; + headers = curl_slist_append(headers, "Content-Type: application/json"); + headers = curl_slist_append(headers, + ("X-NCP-APIGW-API-KEY-ID: " + client_id_).c_str()); + headers = curl_slist_append(headers, + ("X-NCP-APIGW-API-KEY: " + client_secret_).c_str()); + + // Set up CURL request + curl_easy_setopt(curl.get(), CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl.get(), CURLOPT_POST, 1L); + curl_easy_setopt(curl.get(), CURLOPT_POSTFIELDS, payload.c_str()); + curl_easy_setopt(curl.get(), CURLOPT_HTTPHEADER, headers); + curl_easy_setopt(curl.get(), CURLOPT_WRITEFUNCTION, CurlHelper::WriteCallback); + curl_easy_setopt(curl.get(), CURLOPT_WRITEDATA, &response); + curl_easy_setopt(curl.get(), CURLOPT_SSL_VERIFYPEER, 1L); + curl_easy_setopt(curl.get(), CURLOPT_SSL_VERIFYHOST, 2L); + curl_easy_setopt(curl.get(), CURLOPT_TIMEOUT, 30L); + + // Perform request + CURLcode res = curl_easy_perform(curl.get()); + + // Clean up + curl_slist_free_all(headers); + + if (res != CURLE_OK) { + throw TranslationError(std::string("CURL request failed: ") + + curl_easy_strerror(res)); + } + + // Check HTTP response code + long response_code; + curl_easy_getinfo(curl.get(), CURLINFO_RESPONSE_CODE, &response_code); + + if (response_code != 200) { + throw TranslationError("HTTP error: " + std::to_string(response_code)); + } + + return parseResponse(response); + + } catch (const json::exception &e) { + throw TranslationError(std::string("JSON parsing error: ") + e.what()); + } +} + +std::string PapagoTranslator::parseResponse(const std::string &response_str) +{ + try { + json response = json::parse(response_str); + + if (!response.contains("message")) { + throw TranslationError("Invalid response format from Papago API"); + } + + const auto &message = response["message"]; + if (!message.contains("result") || !message["result"].contains("translatedText")) { + throw TranslationError("Translation result not found in response"); + } + + return message["result"]["translatedText"].get(); + + } catch (const json::exception &e) { + throw TranslationError(std::string("Failed to parse Papago response: ") + e.what()); + } +} diff --git a/src/translation/cloud-translation/papago.h b/src/translation/cloud-translation/papago.h new file mode 100644 index 0000000..03245e1 --- /dev/null +++ b/src/translation/cloud-translation/papago.h @@ -0,0 +1,23 @@ +#pragma once +#include "iTranslator.h" +#include + +class CurlHelper; // Forward declaration + +class PapagoTranslator : public ITranslator { +public: + PapagoTranslator(const std::string &client_id, const std::string &client_secret); + ~PapagoTranslator() override; + + std::string translate(const std::string &text, const std::string &target_lang, + const std::string &source_lang = "auto") override; + +private: + std::string parseResponse(const std::string &response_str); + std::string mapLanguageCode(const std::string &lang_code) const; + bool isLanguagePairSupported(const std::string &source, const std::string &target) const; + + std::string client_id_; + std::string client_secret_; + std::unique_ptr curl_helper_; +}; diff --git a/src/translation/cloud-translation/translation-cloud.cpp b/src/translation/cloud-translation/translation-cloud.cpp new file mode 100644 index 0000000..d55b3b9 --- /dev/null +++ b/src/translation/cloud-translation/translation-cloud.cpp @@ -0,0 +1,54 @@ +#include +#include +#include +#include + +#include "ITranslator.h" +#include "google-cloud.h" +#include "deepl.h" +#include "azure.h" +#include "papago.h" +#include "claude.h" +#include "openai.h" + +#include "plugin-support.h" +#include + +#include "translation-cloud.h" + +std::unique_ptr createTranslator(const CloudTranslatorConfig &config) +{ + if (config.provider == "google") { + return std::make_unique(config.access_key); + } else if (config.provider == "deepl") { + return std::make_unique(config.access_key); + } else if (config.provider == "azure") { + return std::make_unique(config.access_key, config.location); + // } else if (config.provider == "aws") { + // return std::make_unique(config.access_key, config.secret_key, config.region); + } else if (config.provider == "papago") { + return std::make_unique(config.access_key, config.secret_key); + } else if (config.provider == "claude") { + return std::make_unique( + config.access_key, + config.model.empty() ? "claude-3-sonnet-20240229" : config.model); + } else if (config.provider == "openai") { + return std::make_unique( + config.access_key, + config.model.empty() ? "gpt-4-turbo-preview" : config.model); + } + throw std::invalid_argument("Unknown translation provider: " + config.provider); +} + +std::string translate_cloud(const CloudTranslatorConfig &config, const std::string &text, + const std::string &target_lang, const std::string &source_lang) +{ + try { + auto translator = createTranslator(config); + std::string result = translator->translate(text, target_lang, source_lang); + return result; + } catch (const TranslationError &e) { + obs_log(LOG_ERROR, "Translation error: %s\n", e.what()); + } + return ""; +} diff --git a/src/translation/cloud-translation/translation-cloud.h b/src/translation/cloud-translation/translation-cloud.h new file mode 100644 index 0000000..4659042 --- /dev/null +++ b/src/translation/cloud-translation/translation-cloud.h @@ -0,0 +1,15 @@ +#pragma once + +#include + +struct CloudTranslatorConfig { + std::string provider; + std::string access_key; // Main API key/Client ID + std::string secret_key; // Secret key/Client secret + std::string region; // For AWS + std::string location; // For Azure + std::string model; // For Claude +}; + +std::string translate_cloud(const CloudTranslatorConfig &config, const std::string &text, + const std::string &target_lang, const std::string &source_lang); From 271affc124234d80cb33825d6af9b5b949bbfc36 Mon Sep 17 00:00:00 2001 From: Roy Shilkrot Date: Wed, 20 Nov 2024 15:45:59 -0500 Subject: [PATCH 02/10] Refactor CMakeLists.txt for cloud translation sources formatting --- .../cloud-translation/CMakeLists.txt | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/translation/cloud-translation/CMakeLists.txt b/src/translation/cloud-translation/CMakeLists.txt index dee765a..d6bb1af 100644 --- a/src/translation/cloud-translation/CMakeLists.txt +++ b/src/translation/cloud-translation/CMakeLists.txt @@ -1,11 +1,12 @@ # add source files -target_sources(${CMAKE_PROJECT_NAME} PRIVATE - # ${CMAKE_SOURCE_DIR}/src/translation/cloud-translation/aws.cpp - ${CMAKE_SOURCE_DIR}/src/translation/cloud-translation/azure.cpp - ${CMAKE_SOURCE_DIR}/src/translation/cloud-translation/claude.cpp - ${CMAKE_SOURCE_DIR}/src/translation/cloud-translation/curl-helper.cpp - ${CMAKE_SOURCE_DIR}/src/translation/cloud-translation/deepl.cpp - ${CMAKE_SOURCE_DIR}/src/translation/cloud-translation/google-cloud.cpp - ${CMAKE_SOURCE_DIR}/src/translation/cloud-translation/openai.cpp - ${CMAKE_SOURCE_DIR}/src/translation/cloud-translation/papago.cpp - ${CMAKE_SOURCE_DIR}/src/translation/cloud-translation/translation-cloud.cpp) +target_sources( + ${CMAKE_PROJECT_NAME} + PRIVATE # ${CMAKE_SOURCE_DIR}/src/translation/cloud-translation/aws.cpp + ${CMAKE_SOURCE_DIR}/src/translation/cloud-translation/azure.cpp + ${CMAKE_SOURCE_DIR}/src/translation/cloud-translation/claude.cpp + ${CMAKE_SOURCE_DIR}/src/translation/cloud-translation/curl-helper.cpp + ${CMAKE_SOURCE_DIR}/src/translation/cloud-translation/deepl.cpp + ${CMAKE_SOURCE_DIR}/src/translation/cloud-translation/google-cloud.cpp + ${CMAKE_SOURCE_DIR}/src/translation/cloud-translation/openai.cpp + ${CMAKE_SOURCE_DIR}/src/translation/cloud-translation/papago.cpp + ${CMAKE_SOURCE_DIR}/src/translation/cloud-translation/translation-cloud.cpp) From ec353e19e4a908686cf7eb072facc83eed2fd8d6 Mon Sep 17 00:00:00 2001 From: Roy Shilkrot Date: Thu, 21 Nov 2024 11:57:45 -0500 Subject: [PATCH 03/10] Add support for translating only full sentences in cloud translation --- data/locale/en-US.ini | 9 +- src/transcription-filter-callbacks.cpp | 100 +++++++++++------- src/transcription-filter-data.h | 10 +- src/transcription-filter-properties.cpp | 26 +++-- src/transcription-filter.cpp | 2 + src/translation/cloud-translation/papago.cpp | 7 +- .../cloud-translation/translation-cloud.cpp | 4 +- 7 files changed, 101 insertions(+), 57 deletions(-) diff --git a/data/locale/en-US.ini b/data/locale/en-US.ini index 215f2df..6afc2de 100644 --- a/data/locale/en-US.ini +++ b/data/locale/en-US.ini @@ -79,6 +79,7 @@ file_output_group="File Output Configuration" translate_explaination="Enabling translation will increase the processing load on your machine, This feature uses additional resources to translate content in real-time, which may impact performance. Learn More" translate_cloud_explaination="Cloud translation requires an active internet connection and API keys to the translation provider." translate_cloud_provider="Translation Provider" +translate_cloud_only_full_sentences="Translate only full sentences" translate_cloud_api_key="Access Key" translate_cloud_secret_key="Secret Key" log_group="Logging" @@ -106,8 +107,8 @@ Tencent-Translate="Tencent Translate" Alibaba-Translate="Alibaba Translate" Naver-Translate="Naver Translate" Kakao-Translate="Kakao Translate" -Papago-Translate="Papago Translate" -Deepl-Translate="Deepl Translate" +Papago-Translate="Papago" +Deepl-Translate="Deepl" Bing-Translate="Bing Translate" -OpenAI-Translate="OpenAI Translate" -Claude-Translate="Claude Translate" +OpenAI-Translate="OpenAI" +Claude-Translate="Claude" diff --git a/src/transcription-filter-callbacks.cpp b/src/transcription-filter-callbacks.cpp index 2e7ec81..30b2267 100644 --- a/src/transcription-filter-callbacks.cpp +++ b/src/transcription-filter-callbacks.cpp @@ -81,39 +81,46 @@ std::string send_sentence_to_translation(const std::string &sentence, return ""; } -std::string send_text_to_cloud_translation(const std::string &sentence, - struct transcription_filter_data *gf, - const std::string &source_language) +void send_sentence_to_cloud_translation_async(const std::string &sentence, + struct transcription_filter_data *gf, + const std::string &source_language, + std::function callback) { - const std::string last_text = gf->last_text_for_translation; - gf->last_text_for_translation = sentence; - if (gf->translate_cloud && !sentence.empty()) { - obs_log(gf->log_level, "Translating text with cloud provider. %s -> %s", - source_language.c_str(), gf->target_lang.c_str()); - std::string translated_text; - if (sentence == last_text) { - // do not translate the same sentence twice - return gf->last_text_translation; - } - CloudTranslatorConfig config; - config.provider = gf->translate_cloud_provider; - config.access_key = gf->translate_cloud_api_key; - config.secret_key = gf->translate_cloud_secret_key; - - translated_text = translate_cloud( - config, sentence, gf->translate_cloud_target_language, source_language); - if (!translated_text.empty()) { - if (gf->log_words) { - obs_log(LOG_INFO, "Translation: '%s' -> '%s'", sentence.c_str(), - translated_text.c_str()); + std::thread([sentence, gf, source_language, callback]() { + const std::string last_text = gf->last_text_for_cloud_translation; + gf->last_text_for_cloud_translation = sentence; + if (gf->translate_cloud && !sentence.empty()) { + obs_log(gf->log_level, "Translating text with cloud provider %s. %s -> %s", + gf->translate_cloud_provider.c_str(), source_language.c_str(), + gf->translate_cloud_target_language.c_str()); + std::string translated_text; + if (sentence == last_text) { + // do not translate the same sentence twice + callback(gf->last_text_cloud_translation); + return; + } + CloudTranslatorConfig config; + config.provider = gf->translate_cloud_provider; + config.access_key = gf->translate_cloud_api_key; + config.secret_key = gf->translate_cloud_secret_key; + + translated_text = translate_cloud(config, sentence, + gf->translate_cloud_target_language, + source_language); + if (!translated_text.empty()) { + if (gf->log_words) { + obs_log(LOG_INFO, "Cloud Translation: '%s' -> '%s'", + sentence.c_str(), translated_text.c_str()); + } + gf->last_text_translation = translated_text; + callback(translated_text); + return; + } else { + obs_log(gf->log_level, "Failed to translate text"); } - gf->last_text_translation = translated_text; - return translated_text; - } else { - obs_log(gf->log_level, "Failed to translate text"); } - } - return ""; + callback(""); + }).detach(); } void send_sentence_to_file(struct transcription_filter_data *gf, @@ -271,33 +278,50 @@ void set_text_callback(struct transcription_filter_data *gf, } } - bool should_translate = + bool should_translate_local = gf->translate_only_full_sentences ? result.result == DETECTION_RESULT_SPEECH : true; // send the sentence to translation (if enabled) - std::string translated_sentence = - should_translate ? send_sentence_to_translation(str_copy, gf, result.language) : ""; + std::string translated_sentence_local = + should_translate_local ? send_sentence_to_translation(str_copy, gf, result.language) + : ""; if (gf->translate) { if (gf->translation_output == "none") { // overwrite the original text with the translated text - str_copy = translated_sentence; + str_copy = translated_sentence_local; } else { if (gf->buffered_output) { // buffered output - add the sentence to the monitor gf->translation_monitor.addSentenceFromStdString( - translated_sentence, + translated_sentence_local, get_time_point_from_ms(result.start_timestamp_ms), get_time_point_from_ms(result.end_timestamp_ms), result.result == DETECTION_RESULT_PARTIAL); } else { // non-buffered output - send the sentence to the selected source - send_caption_to_source(gf->translation_output, translated_sentence, - gf); + send_caption_to_source(gf->translation_output, + translated_sentence_local, gf); } } } + bool should_translate_cloud = (gf->translate_cloud_only_full_sentences + ? result.result == DETECTION_RESULT_SPEECH + : true) && + gf->translate_cloud; + + if (should_translate_cloud) { + send_sentence_to_cloud_translation_async( + str_copy, gf, result.language, + [gf](const std::string &translated_sentence_cloud) { + if (gf->translate_cloud_output != "none") { + send_caption_to_source(gf->translate_cloud_output, + translated_sentence_cloud, gf); + } + }); + } + if (gf->buffered_output) { gf->captions_monitor.addSentenceFromStdString( str_copy, get_time_point_from_ms(result.start_timestamp_ms), @@ -315,7 +339,7 @@ void set_text_callback(struct transcription_filter_data *gf, if (gf->save_to_file && gf->output_file_path != "" && result.result == DETECTION_RESULT_SPEECH) { - send_sentence_to_file(gf, result, str_copy, translated_sentence); + send_sentence_to_file(gf, result, str_copy, translated_sentence_local); } if (!result.text.empty() && (result.result == DETECTION_RESULT_SPEECH || diff --git a/src/transcription-filter-data.h b/src/transcription-filter-data.h index 1c71aaa..1d248b9 100644 --- a/src/transcription-filter-data.h +++ b/src/transcription-filter-data.h @@ -96,10 +96,9 @@ struct transcription_filter_data { std::string translate_cloud_output; std::string translate_cloud_api_key; std::string translate_cloud_secret_key; - - // Last transcription result - std::string last_text_for_translation; - std::string last_text_translation; + bool translate_cloud_only_full_sentences = true; + std::string last_text_for_cloud_translation; + std::string last_text_cloud_translation; // Transcription context sentences int n_context_sentences; @@ -127,6 +126,9 @@ struct transcription_filter_data { std::string translation_model_index; std::string translation_model_path_external; bool translate_only_full_sentences; + // Last transcription result + std::string last_text_for_translation; + std::string last_text_translation; bool buffered_output = false; TokenBufferThread captions_monitor; diff --git a/src/transcription-filter-properties.cpp b/src/transcription-filter-properties.cpp index f2b325f..77ad89e 100644 --- a/src/transcription-filter-properties.cpp +++ b/src/transcription-filter-properties.cpp @@ -49,8 +49,10 @@ bool translation_cloud_options_callback(obs_properties_t *props, obs_property_t UNUSED_PARAMETER(property); // Show/Hide the cloud translation group options const bool translate_enabled = obs_data_get_bool(settings, "translate_cloud"); - for (const auto &prop : {"translate_cloud_provider", "translate_cloud_target_language", - "translate_cloud_output", "translate_cloud_api_key"}) { + for (const auto &prop : + {"translate_cloud_provider", "translate_cloud_target_language", + "translate_cloud_output", "translate_cloud_api_key", + "translate_cloud_only_full_sentences", "translate_cloud_secret_key"}) { obs_property_set_visible(obs_properties_get(props, prop), translate_enabled); } return true; @@ -209,9 +211,9 @@ void add_translation_cloud_group_properties(obs_properties_t *ppts) MT_("translate_cloud_provider"), OBS_COMBO_TYPE_LIST, OBS_COMBO_FORMAT_STRING); // Populate the dropdown with the cloud translation service providers obs_property_list_add_string(prop_translate_cloud_provider, MT_("Google-Cloud-Translation"), - "google-cloud-translation"); + "google"); obs_property_list_add_string(prop_translate_cloud_provider, MT_("Microsoft-Translator"), - "microsoft-translator"); + "azure"); // obs_property_list_add_string(prop_translate_cloud_provider, MT_("Amazon-Translate"), // "amazon-translate"); // obs_property_list_add_string(prop_translate_cloud_provider, MT_("IBM-Watson-Translate"), @@ -229,13 +231,13 @@ void add_translation_cloud_group_properties(obs_properties_t *ppts) // obs_property_list_add_string(prop_translate_cloud_provider, MT_("Kakao-Translate"), // "kakao-translate"); obs_property_list_add_string(prop_translate_cloud_provider, MT_("Papago-Translate"), - "papago-translate"); + "papago"); obs_property_list_add_string(prop_translate_cloud_provider, MT_("Deepl-Translate"), - "deepl-translate"); + "deepl"); obs_property_list_add_string(prop_translate_cloud_provider, MT_("OpenAI-Translate"), - "openai-translate"); + "openai"); obs_property_list_add_string(prop_translate_cloud_provider, MT_("Claude-Translate"), - "claude-translate"); + "claude"); // add target language selection obs_property_t *prop_tgt = obs_properties_add_list( @@ -253,6 +255,10 @@ void add_translation_cloud_group_properties(obs_properties_t *ppts) obs_property_list_add_string(prop_output, "Write to captions output", "none"); obs_enum_sources(add_sources_to_list, prop_output); + // add boolean option for only full sentences + obs_properties_add_bool(translation_cloud_group, "translate_cloud_only_full_sentences", + MT_("translate_cloud_only_full_sentences")); + // add input for API Key obs_properties_add_text(translation_cloud_group, "translate_cloud_api_key", MT_("translate_cloud_api_key"), OBS_TEXT_DEFAULT); @@ -695,10 +701,12 @@ void transcription_filter_defaults(obs_data_t *s) // cloud translation options obs_data_set_default_bool(s, "translate_cloud", false); - obs_data_set_default_string(s, "translate_cloud_provider", "google-cloud-translation"); + obs_data_set_default_string(s, "translate_cloud_provider", "google"); obs_data_set_default_string(s, "translate_cloud_target_language", "en"); obs_data_set_default_string(s, "translate_cloud_output", "none"); + obs_data_set_default_bool(s, "translate_cloud_only_full_sentences", true); obs_data_set_default_string(s, "translate_cloud_api_key", ""); + obs_data_set_default_string(s, "translate_cloud_secret_key", ""); // Whisper parameters obs_data_set_default_int(s, "whisper_sampling_method", WHISPER_SAMPLING_BEAM_SEARCH); diff --git a/src/transcription-filter.cpp b/src/transcription-filter.cpp index 8c46637..bb36635 100644 --- a/src/transcription-filter.cpp +++ b/src/transcription-filter.cpp @@ -336,6 +336,8 @@ void transcription_filter_update(void *data, obs_data_t *s) gf->translate_cloud_target_language = obs_data_get_string(s, "translate_cloud_target_language"); gf->translate_cloud_output = obs_data_get_string(s, "translate_cloud_output"); + gf->translate_cloud_only_full_sentences = + obs_data_get_bool(s, "translate_cloud_only_full_sentences"); gf->translate_cloud_api_key = obs_data_get_string(s, "translate_cloud_api_key"); gf->translate_cloud_secret_key = obs_data_get_string(s, "translate_cloud_secret_key"); diff --git a/src/translation/cloud-translation/papago.cpp b/src/translation/cloud-translation/papago.cpp index 041fded..3a7ef14 100644 --- a/src/translation/cloud-translation/papago.cpp +++ b/src/translation/cloud-translation/papago.cpp @@ -136,8 +136,13 @@ std::string PapagoTranslator::translate(const std::string &text, const std::stri throw TranslationError("Text exceeds maximum length of 5000 characters"); } + std::string target_lang_valid = target_lang; + target_lang_valid.erase(std::remove(target_lang_valid.begin(), target_lang_valid.end(), + '_'), + target_lang_valid.end()); + std::string papago_source = mapLanguageCode(source_lang); - std::string papago_target = mapLanguageCode(target_lang); + std::string papago_target = mapLanguageCode(target_lang_valid); if (!isLanguagePairSupported(papago_source, papago_target)) { throw TranslationError("Unsupported language pair: " + source_lang + " to " + diff --git a/src/translation/cloud-translation/translation-cloud.cpp b/src/translation/cloud-translation/translation-cloud.cpp index d55b3b9..a1f4537 100644 --- a/src/translation/cloud-translation/translation-cloud.cpp +++ b/src/translation/cloud-translation/translation-cloud.cpp @@ -37,7 +37,7 @@ std::unique_ptr createTranslator(const CloudTranslatorConfig &confi config.access_key, config.model.empty() ? "gpt-4-turbo-preview" : config.model); } - throw std::invalid_argument("Unknown translation provider: " + config.provider); + throw TranslationError("Unknown translation provider: " + config.provider); } std::string translate_cloud(const CloudTranslatorConfig &config, const std::string &text, @@ -45,6 +45,8 @@ std::string translate_cloud(const CloudTranslatorConfig &config, const std::stri { try { auto translator = createTranslator(config); + obs_log(LOG_INFO, "translate with cloud provider %s. %s -> %s", + config.provider.c_str(), source_lang.c_str(), target_lang.c_str()); std::string result = translator->translate(text, target_lang, source_lang); return result; } catch (const TranslationError &e) { From 2440502d6dece2a7f0136b786b1bd5241717614a Mon Sep 17 00:00:00 2001 From: Roy Shilkrot Date: Thu, 21 Nov 2024 12:48:33 -0500 Subject: [PATCH 04/10] Update ICU build configuration and fix header include case sensitivity --- cmake/BuildICU.cmake | 6 ++++-- src/translation/cloud-translation/papago.h | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/cmake/BuildICU.cmake b/cmake/BuildICU.cmake index a3c575d..2c48f87 100644 --- a/cmake/BuildICU.cmake +++ b/cmake/BuildICU.cmake @@ -66,8 +66,10 @@ else() DOWNLOAD_EXTRACT_TIMESTAMP true GIT_REPOSITORY "https://github.com/unicode-org/icu.git" GIT_TAG "release-${ICU_VERSION_DASH}" - CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env ${ICU_BUILD_ENV_VARS} /icu4c/source/runConfigureICU - ${ICU_PLATFORM} --prefix= --enable-static --disable-shared + CONFIGURE_COMMAND + ${CMAKE_COMMAND} -E env ${ICU_BUILD_ENV_VARS} /icu4c/source/runConfigureICU ${ICU_PLATFORM} + --prefix= --enable-static --disable-shared --disable-tools --disable-samples --disable-layout + --disable-layoutex --disable-tests --disable-draft --disable-extras --disable-icuio BUILD_COMMAND make -j4 BUILD_BYPRODUCTS /lib/${CMAKE_STATIC_LIBRARY_PREFIX}icudata${CMAKE_STATIC_LIBRARY_SUFFIX} diff --git a/src/translation/cloud-translation/papago.h b/src/translation/cloud-translation/papago.h index 03245e1..68f5ede 100644 --- a/src/translation/cloud-translation/papago.h +++ b/src/translation/cloud-translation/papago.h @@ -1,5 +1,5 @@ #pragma once -#include "iTranslator.h" +#include "ITranslator.h" #include class CurlHelper; // Forward declaration From f42e3674bfdb6038f43315b5516129163c0f9355 Mon Sep 17 00:00:00 2001 From: Roy Shilkrot Date: Thu, 21 Nov 2024 13:29:45 -0500 Subject: [PATCH 05/10] Fix CURL helper function signatures and improve URL encoding --- src/translation/cloud-translation/curl-helper.cpp | 4 ++-- src/translation/cloud-translation/curl-helper.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/translation/cloud-translation/curl-helper.cpp b/src/translation/cloud-translation/curl-helper.cpp index 9073e58..3e6913e 100644 --- a/src/translation/cloud-translation/curl-helper.cpp +++ b/src/translation/cloud-translation/curl-helper.cpp @@ -45,7 +45,7 @@ std::string CurlHelper::urlEncode(CURL *curl, const std::string &value) } std::unique_ptr escaped( - curl_easy_escape(curl, value.c_str(), value.length()), curl_free); + curl_easy_escape(curl, value.c_str(), (int)value.length()), curl_free); if (!escaped) { throw TranslationError("Failed to URL encode string"); @@ -54,7 +54,7 @@ std::string CurlHelper::urlEncode(CURL *curl, const std::string &value) return std::string(escaped.get()); } -struct curl_slist *CurlHelper::createBasicHeaders(CURL *curl, const std::string &content_type) +struct curl_slist *CurlHelper::createBasicHeaders(const std::string &content_type) { struct curl_slist *headers = nullptr; diff --git a/src/translation/cloud-translation/curl-helper.h b/src/translation/cloud-translation/curl-helper.h index a227ef6..8a117a5 100644 --- a/src/translation/cloud-translation/curl-helper.h +++ b/src/translation/cloud-translation/curl-helper.h @@ -18,7 +18,7 @@ class CurlHelper { // Common request builders static struct curl_slist * - createBasicHeaders(CURL *curl, const std::string &content_type = "application/json"); + createBasicHeaders(const std::string &content_type = "application/json"); // Verify HTTPS certificate static void setSSLVerification(CURL *curl, bool verify = true); From c7ae10013d1e7bed49123400fe798567e72afa1b Mon Sep 17 00:00:00 2001 From: Roy Shilkrot Date: Thu, 21 Nov 2024 13:47:44 -0500 Subject: [PATCH 06/10] Fix character type casting in DeepLTranslator for language conversion --- src/translation/cloud-translation/deepl.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/translation/cloud-translation/deepl.cpp b/src/translation/cloud-translation/deepl.cpp index f9c775e..e76af31 100644 --- a/src/translation/cloud-translation/deepl.cpp +++ b/src/translation/cloud-translation/deepl.cpp @@ -31,9 +31,9 @@ std::string DeepLTranslator::translate(const std::string &text, const std::strin std::string upperTarget = target_lang; std::string upperSource = source_lang; for (char &c : upperTarget) - c = std::toupper(c); + c = (char)std::toupper((int)c); for (char &c : upperSource) - c = std::toupper(c); + c = (char)std::toupper((int)c); std::stringstream url; url << "https://api-free.deepl.com/v2/translate" From b2a6ee0ff10a9dfd37d7a77ec71b861792a145ac Mon Sep 17 00:00:00 2001 From: Roy Shilkrot Date: Fri, 22 Nov 2024 11:30:21 -0500 Subject: [PATCH 07/10] Refactor file saving logic in transcription filter to streamline sentence handling and add support for saving translated sentences --- src/transcription-filter-callbacks.cpp | 110 +++++++++++++------------ 1 file changed, 57 insertions(+), 53 deletions(-) diff --git a/src/transcription-filter-callbacks.cpp b/src/transcription-filter-callbacks.cpp index 30b2267..174bb79 100644 --- a/src/transcription-filter-callbacks.cpp +++ b/src/transcription-filter-callbacks.cpp @@ -124,8 +124,8 @@ void send_sentence_to_cloud_translation_async(const std::string &sentence, } void send_sentence_to_file(struct transcription_filter_data *gf, - const DetectionResultWithText &result, const std::string &str_copy, - const std::string &translated_sentence) + const DetectionResultWithText &result, const std::string &sentence, + const std::string &file_path, bool bump_sentence_number) { // Check if we should save the sentence if (gf->save_only_while_recording && !obs_frontend_recording_active()) { @@ -133,20 +133,6 @@ void send_sentence_to_file(struct transcription_filter_data *gf, return; } - std::string translated_file_path = ""; - bool write_translations = gf->translate && !translated_sentence.empty(); - - // if translation is enabled, save the translated sentence to another file - if (write_translations) { - // add a postfix to the file name (without extension) with the translation target language - std::string output_file_path = gf->output_file_path; - std::string file_extension = - output_file_path.substr(output_file_path.find_last_of(".") + 1); - std::string file_name = - output_file_path.substr(0, output_file_path.find_last_of(".")); - translated_file_path = file_name + "_" + gf->target_lang + "." + file_extension; - } - // should the file be truncated? std::ios_base::openmode openmode = std::ios::out; if (gf->truncate_output_file) { @@ -157,15 +143,9 @@ void send_sentence_to_file(struct transcription_filter_data *gf, if (!gf->save_srt) { // Write raw sentence to file try { - std::ofstream output_file(gf->output_file_path, openmode); - output_file << str_copy << std::endl; + std::ofstream output_file(file_path, openmode); + output_file << sentence << std::endl; output_file.close(); - if (write_translations) { - std::ofstream translated_output_file(translated_file_path, - openmode); - translated_output_file << translated_sentence << std::endl; - translated_output_file.close(); - } } catch (const std::ofstream::failure &e) { obs_log(LOG_ERROR, "Exception opening/writing/closing file: %s", e.what()); } @@ -176,9 +156,9 @@ void send_sentence_to_file(struct transcription_filter_data *gf, } obs_log(gf->log_level, "Saving sentence to file %s, sentence #%d", - gf->output_file_path.c_str(), gf->sentence_number); + file_path.c_str(), gf->sentence_number); // Append sentence to file in .srt format - std::ofstream output_file(gf->output_file_path, openmode); + std::ofstream output_file(file_path, openmode); output_file << gf->sentence_number << std::endl; // use the start and end timestamps to calculate the start and end time in srt format auto format_ts_for_srt = [](std::ofstream &output_stream, uint64_t ts) { @@ -199,28 +179,34 @@ void send_sentence_to_file(struct transcription_filter_data *gf, format_ts_for_srt(output_file, result.end_timestamp_ms); output_file << std::endl; - output_file << str_copy << std::endl; + output_file << sentence << std::endl; output_file << std::endl; output_file.close(); - if (write_translations) { - obs_log(gf->log_level, "Saving translation to file %s, sentence #%d", - translated_file_path.c_str(), gf->sentence_number); - - // Append translated sentence to file in .srt format - std::ofstream translated_output_file(translated_file_path, openmode); - translated_output_file << gf->sentence_number << std::endl; - format_ts_for_srt(translated_output_file, result.start_timestamp_ms); - translated_output_file << " --> "; - format_ts_for_srt(translated_output_file, result.end_timestamp_ms); - translated_output_file << std::endl; - - translated_output_file << translated_sentence << std::endl; - translated_output_file << std::endl; - translated_output_file.close(); + if (bump_sentence_number) { + gf->sentence_number++; } + } +} - gf->sentence_number++; +void send_translated_sentence_to_file(struct transcription_filter_data *gf, + const DetectionResultWithText &result, + const std::string &translated_sentence, + const std::string &target_lang) +{ + // if translation is enabled, save the translated sentence to another file + if (translated_sentence.empty()) { + obs_log(gf->log_level, "Translation is empty, not saving to file"); + } else { + // add a postfix to the file name (without extension) with the translation target language + std::string translated_file_path = ""; + std::string output_file_path = gf->output_file_path; + std::string file_extension = + output_file_path.substr(output_file_path.find_last_of(".") + 1); + std::string file_name = + output_file_path.substr(0, output_file_path.find_last_of(".")); + translated_file_path = file_name + "_" + target_lang + "." + file_extension; + send_sentence_to_file(gf, result, translated_sentence, translated_file_path, false); } } @@ -304,6 +290,10 @@ void set_text_callback(struct transcription_filter_data *gf, translated_sentence_local, gf); } } + if (gf->save_to_file && gf->output_file_path != "") { + send_translated_sentence_to_file(gf, result, translated_sentence_local, + gf->target_lang); + } } bool should_translate_cloud = (gf->translate_cloud_only_full_sentences @@ -314,22 +304,36 @@ void set_text_callback(struct transcription_filter_data *gf, if (should_translate_cloud) { send_sentence_to_cloud_translation_async( str_copy, gf, result.language, - [gf](const std::string &translated_sentence_cloud) { + [gf, result](const std::string &translated_sentence_cloud) { if (gf->translate_cloud_output != "none") { send_caption_to_source(gf->translate_cloud_output, translated_sentence_cloud, gf); + } else { + // overwrite the original text with the translated text + send_caption_to_source(gf->text_source_name, + translated_sentence_cloud, gf); + } + if (gf->save_to_file && gf->output_file_path != "") { + send_translated_sentence_to_file( + gf, result, translated_sentence_cloud, + gf->translate_cloud_target_language); } }); } - if (gf->buffered_output) { - gf->captions_monitor.addSentenceFromStdString( - str_copy, get_time_point_from_ms(result.start_timestamp_ms), - get_time_point_from_ms(result.end_timestamp_ms), - result.result == DETECTION_RESULT_PARTIAL); - } else { - // non-buffered output - send the sentence to the selected source - send_caption_to_source(gf->text_source_name, str_copy, gf); + // send the original text to the output + // unless the translation is enabled and set to overwrite the original text + if (!((should_translate_cloud && gf->translate_cloud_output == "none") || + (should_translate_local && gf->translation_output == "none"))) { + if (gf->buffered_output) { + gf->captions_monitor.addSentenceFromStdString( + str_copy, get_time_point_from_ms(result.start_timestamp_ms), + get_time_point_from_ms(result.end_timestamp_ms), + result.result == DETECTION_RESULT_PARTIAL); + } else { + // non-buffered output - send the sentence to the selected source + send_caption_to_source(gf->text_source_name, str_copy, gf); + } } if (gf->caption_to_stream && result.result == DETECTION_RESULT_SPEECH) { @@ -339,7 +343,7 @@ void set_text_callback(struct transcription_filter_data *gf, if (gf->save_to_file && gf->output_file_path != "" && result.result == DETECTION_RESULT_SPEECH) { - send_sentence_to_file(gf, result, str_copy, translated_sentence_local); + send_sentence_to_file(gf, result, str_copy, gf->output_file_path, true); } if (!result.text.empty() && (result.result == DETECTION_RESULT_SPEECH || From cedbdc475ba128f573dfe5171f67acec85c4eade Mon Sep 17 00:00:00 2001 From: Roy Shilkrot Date: Fri, 22 Nov 2024 14:42:05 -0500 Subject: [PATCH 08/10] Add support for Deepl Free API endpoint and enhance cloud translation configuration --- data/locale/en-US.ini | 1 + src/transcription-filter-callbacks.cpp | 2 + src/transcription-filter-data.h | 2 + src/transcription-filter-properties.cpp | 41 ++++++++++-- src/transcription-filter.cpp | 2 + .../cloud-translation/ITranslator.h | 12 ++++ src/translation/cloud-translation/azure.cpp | 4 +- src/translation/cloud-translation/claude.cpp | 46 +------------ src/translation/cloud-translation/claude.h | 2 - src/translation/cloud-translation/deepl.cpp | 65 ++++++++++++------- src/translation/cloud-translation/deepl.h | 3 +- .../cloud-translation/google-cloud.cpp | 4 +- src/translation/cloud-translation/openai.cpp | 46 +------------ src/translation/cloud-translation/openai.h | 2 - .../cloud-translation/translation-cloud.cpp | 4 +- .../cloud-translation/translation-cloud.h | 4 +- src/translation/language_codes.h | 25 +++++++ 17 files changed, 137 insertions(+), 128 deletions(-) diff --git a/data/locale/en-US.ini b/data/locale/en-US.ini index 6afc2de..3f94d4e 100644 --- a/data/locale/en-US.ini +++ b/data/locale/en-US.ini @@ -112,3 +112,4 @@ Deepl-Translate="Deepl" Bing-Translate="Bing Translate" OpenAI-Translate="OpenAI" Claude-Translate="Claude" +translate_cloud_deepl_free="Use Deepl Free API Endpoint" diff --git a/src/transcription-filter-callbacks.cpp b/src/transcription-filter-callbacks.cpp index 174bb79..5938863 100644 --- a/src/transcription-filter-callbacks.cpp +++ b/src/transcription-filter-callbacks.cpp @@ -103,6 +103,8 @@ void send_sentence_to_cloud_translation_async(const std::string &sentence, config.provider = gf->translate_cloud_provider; config.access_key = gf->translate_cloud_api_key; config.secret_key = gf->translate_cloud_secret_key; + config.free = gf->translate_cloud_deepl_free; + config.region = gf->translate_cloud_region; translated_text = translate_cloud(config, sentence, gf->translate_cloud_target_language, diff --git a/src/transcription-filter-data.h b/src/transcription-filter-data.h index 1d248b9..f96c7d9 100644 --- a/src/transcription-filter-data.h +++ b/src/transcription-filter-data.h @@ -99,6 +99,8 @@ struct transcription_filter_data { bool translate_cloud_only_full_sentences = true; std::string last_text_for_cloud_translation; std::string last_text_cloud_translation; + bool translate_cloud_deepl_free; + std::string translate_cloud_region; // Transcription context sentences int n_context_sentences; diff --git a/src/transcription-filter-properties.cpp b/src/transcription-filter-properties.cpp index 77ad89e..7b9c2f4 100644 --- a/src/transcription-filter-properties.cpp +++ b/src/transcription-filter-properties.cpp @@ -43,18 +43,37 @@ bool translation_options_callback(obs_properties_t *props, obs_property_t *prope return true; } +bool translation_cloud_provider_selection_callback(obs_properties_t *props, obs_property_t *p, + obs_data_t *s) +{ + UNUSED_PARAMETER(p); + const char *provider = obs_data_get_string(s, "translate_cloud_provider"); + obs_property_set_visible(obs_properties_get(props, "translate_cloud_deepl_free"), + strcmp(provider, "deepl") == 0); + // show the secret key input for the papago provider only + obs_property_set_visible(obs_properties_get(props, "translate_cloud_secret_key"), + strcmp(provider, "papago") == 0); + // show the region input for the azure provider only + obs_property_set_visible(obs_properties_get(props, "translate_cloud_region"), + strcmp(provider, "azure") == 0); + return true; +} + bool translation_cloud_options_callback(obs_properties_t *props, obs_property_t *property, obs_data_t *settings) { UNUSED_PARAMETER(property); // Show/Hide the cloud translation group options const bool translate_enabled = obs_data_get_bool(settings, "translate_cloud"); - for (const auto &prop : - {"translate_cloud_provider", "translate_cloud_target_language", - "translate_cloud_output", "translate_cloud_api_key", - "translate_cloud_only_full_sentences", "translate_cloud_secret_key"}) { + for (const auto &prop : {"translate_cloud_provider", "translate_cloud_target_language", + "translate_cloud_output", "translate_cloud_api_key", + "translate_cloud_only_full_sentences", + "translate_cloud_secret_key", "translate_cloud_deepl_free"}) { obs_property_set_visible(obs_properties_get(props, prop), translate_enabled); } + if (translate_enabled) { + translation_cloud_provider_selection_callback(props, NULL, settings); + } return true; } @@ -239,6 +258,10 @@ void add_translation_cloud_group_properties(obs_properties_t *ppts) obs_property_list_add_string(prop_translate_cloud_provider, MT_("Claude-Translate"), "claude"); + // add callback to show/hide the free API option for deepl + obs_property_set_modified_callback(prop_translate_cloud_provider, + translation_cloud_provider_selection_callback); + // add target language selection obs_property_t *prop_tgt = obs_properties_add_list( translation_cloud_group, "translate_cloud_target_language", MT_("target_language"), @@ -265,6 +288,14 @@ void add_translation_cloud_group_properties(obs_properties_t *ppts) // add input for secret key obs_properties_add_text(translation_cloud_group, "translate_cloud_secret_key", MT_("translate_cloud_secret_key"), OBS_TEXT_PASSWORD); + + // add boolean option for free API from deepl + obs_properties_add_bool(translation_cloud_group, "translate_cloud_deepl_free", + MT_("translate_cloud_deepl_free")); + + // add translate_cloud_region for azure + obs_properties_add_text(translation_cloud_group, "translate_cloud_region", + MT_("translate_cloud_region"), OBS_TEXT_DEFAULT); } void add_translation_group_properties(obs_properties_t *ppts) @@ -707,6 +738,8 @@ void transcription_filter_defaults(obs_data_t *s) obs_data_set_default_bool(s, "translate_cloud_only_full_sentences", true); obs_data_set_default_string(s, "translate_cloud_api_key", ""); obs_data_set_default_string(s, "translate_cloud_secret_key", ""); + obs_data_set_default_bool(s, "translate_cloud_deepl_free", true); + obs_data_set_default_string(s, "translate_cloud_region", "eastus"); // Whisper parameters obs_data_set_default_int(s, "whisper_sampling_method", WHISPER_SAMPLING_BEAM_SEARCH); diff --git a/src/transcription-filter.cpp b/src/transcription-filter.cpp index bb36635..5e13d52 100644 --- a/src/transcription-filter.cpp +++ b/src/transcription-filter.cpp @@ -340,6 +340,8 @@ void transcription_filter_update(void *data, obs_data_t *s) obs_data_get_bool(s, "translate_cloud_only_full_sentences"); gf->translate_cloud_api_key = obs_data_get_string(s, "translate_cloud_api_key"); gf->translate_cloud_secret_key = obs_data_get_string(s, "translate_cloud_secret_key"); + gf->translate_cloud_deepl_free = obs_data_get_bool(s, "translate_cloud_deepl_free"); + gf->translate_cloud_region = obs_data_get_string(s, "translate_cloud_region"); obs_log(gf->log_level, "update text source"); // update the text source diff --git a/src/translation/cloud-translation/ITranslator.h b/src/translation/cloud-translation/ITranslator.h index 6ac3388..a43ae07 100644 --- a/src/translation/cloud-translation/ITranslator.h +++ b/src/translation/cloud-translation/ITranslator.h @@ -22,3 +22,15 @@ class ITranslator { std::unique_ptr createTranslator(const std::string &provider, const std::string &api_key, const std::string &location = ""); + +inline std::string sanitize_language_code(const std::string &lang_code) +{ + // Remove all non-alphabetic characters + std::string sanitized_code; + for (const char &c : lang_code) { + if (isalpha((int)c)) { + sanitized_code += c; + } + } + return sanitized_code; +} diff --git a/src/translation/cloud-translation/azure.cpp b/src/translation/cloud-translation/azure.cpp index e6671bf..a9513b3 100644 --- a/src/translation/cloud-translation/azure.cpp +++ b/src/translation/cloud-translation/azure.cpp @@ -32,10 +32,10 @@ std::string AzureTranslator::translate(const std::string &text, const std::strin // Construct the route std::stringstream route; route << "/translate?api-version=3.0" - << "&to=" << target_lang; + << "&to=" << sanitize_language_code(target_lang); if (source_lang != "auto") { - route << "&from=" << source_lang; + route << "&from=" << sanitize_language_code(source_lang); } // Create the request body diff --git a/src/translation/cloud-translation/claude.cpp b/src/translation/cloud-translation/claude.cpp index e019be9..0007448 100644 --- a/src/translation/cloud-translation/claude.cpp +++ b/src/translation/cloud-translation/claude.cpp @@ -5,6 +5,8 @@ #include #include +#include "translation/language_codes.h" + using json = nlohmann::json; ClaudeTranslator::ClaudeTranslator(const std::string &api_key, const std::string &model) @@ -16,50 +18,6 @@ ClaudeTranslator::ClaudeTranslator(const std::string &api_key, const std::string ClaudeTranslator::~ClaudeTranslator() = default; -std::string ClaudeTranslator::getLanguageName(const std::string &lang_code) const -{ - static const std::unordered_map language_names = { - {"auto", "auto-detected language"}, - {"en", "English"}, - {"es", "Spanish"}, - {"fr", "French"}, - {"de", "German"}, - {"it", "Italian"}, - {"pt", "Portuguese"}, - {"nl", "Dutch"}, - {"pl", "Polish"}, - {"ru", "Russian"}, - {"ja", "Japanese"}, - {"ko", "Korean"}, - {"zh", "Chinese"}, - {"ar", "Arabic"}, - {"hi", "Hindi"}, - {"bn", "Bengali"}, - {"uk", "Ukrainian"}, - {"vi", "Vietnamese"}, - {"th", "Thai"}, - {"tr", "Turkish"}, - // Add more languages as needed - }; - - auto it = language_names.find(lang_code); - if (it != language_names.end()) { - return it->second; - } - return lang_code; // Return the code itself if no mapping exists -} - -bool ClaudeTranslator::isLanguageSupported(const std::string &lang_code) const -{ - static const std::unordered_set supported_languages = { - "auto", "en", "es", "fr", "de", "it", "pt", "nl", "pl", "ru", "ja", - "ko", "zh", "ar", "hi", "bn", "uk", "vi", "th", "tr" - // Add more supported languages as needed - }; - - return supported_languages.find(lang_code) != supported_languages.end(); -} - std::string ClaudeTranslator::createSystemPrompt(const std::string &target_lang) const { std::string target_language = getLanguageName(target_lang); diff --git a/src/translation/cloud-translation/claude.h b/src/translation/cloud-translation/claude.h index 3761058..7301270 100644 --- a/src/translation/cloud-translation/claude.h +++ b/src/translation/cloud-translation/claude.h @@ -16,8 +16,6 @@ class ClaudeTranslator : public ITranslator { private: std::string parseResponse(const std::string &response_str); std::string createSystemPrompt(const std::string &target_lang) const; - std::string getLanguageName(const std::string &lang_code) const; - bool isLanguageSupported(const std::string &lang_code) const; std::string api_key_; std::string model_; diff --git a/src/translation/cloud-translation/deepl.cpp b/src/translation/cloud-translation/deepl.cpp index e76af31..351d7b6 100644 --- a/src/translation/cloud-translation/deepl.cpp +++ b/src/translation/cloud-translation/deepl.cpp @@ -5,8 +5,9 @@ using json = nlohmann::json; -DeepLTranslator::DeepLTranslator(const std::string &api_key) +DeepLTranslator::DeepLTranslator(const std::string &api_key, bool free) : api_key_(api_key), + free_(free), curl_helper_(std::make_unique()) { } @@ -20,7 +21,7 @@ std::string DeepLTranslator::translate(const std::string &text, const std::strin curl_easy_cleanup); if (!curl) { - throw TranslationError("Failed to initialize CURL session"); + throw TranslationError("DeepL Failed to initialize CURL session"); } std::string response; @@ -28,35 +29,39 @@ std::string DeepLTranslator::translate(const std::string &text, const std::strin try { // Construct URL with parameters // Note: DeepL uses uppercase language codes - std::string upperTarget = target_lang; - std::string upperSource = source_lang; + std::string upperTarget = sanitize_language_code(target_lang); + std::string upperSource = sanitize_language_code(source_lang); for (char &c : upperTarget) c = (char)std::toupper((int)c); for (char &c : upperSource) c = (char)std::toupper((int)c); - std::stringstream url; - url << "https://api-free.deepl.com/v2/translate" - << "?auth_key=" << api_key_ - << "&text=" << CurlHelper::urlEncode(curl.get(), text) - << "&target_lang=" << upperTarget; + json body = {{"text", {text}}, + {"target_lang", upperTarget}, + {"source_lang", upperSource}}; + const std::string body_str = body.dump(); - if (upperSource != "AUTO") { - url << "&source_lang=" << upperSource; + std::string url = "https://api.deepl.com/v2/translate"; + if (free_) { + url = "https://api-free.deepl.com/v2/translate"; } // Set up curl options - curl_easy_setopt(curl.get(), CURLOPT_URL, url.str().c_str()); + curl_easy_setopt(curl.get(), CURLOPT_URL, url.c_str()); curl_easy_setopt(curl.get(), CURLOPT_WRITEFUNCTION, CurlHelper::WriteCallback); curl_easy_setopt(curl.get(), CURLOPT_WRITEDATA, &response); curl_easy_setopt(curl.get(), CURLOPT_SSL_VERIFYPEER, 1L); curl_easy_setopt(curl.get(), CURLOPT_SSL_VERIFYHOST, 2L); curl_easy_setopt(curl.get(), CURLOPT_TIMEOUT, 30L); + curl_easy_setopt(curl.get(), CURLOPT_POSTFIELDS, body_str.c_str()); + curl_easy_setopt(curl.get(), CURLOPT_POSTFIELDSIZE, body_str.size()); + curl_easy_setopt(curl.get(), CURLOPT_POST, 1L); // DeepL requires specific headers struct curl_slist *headers = nullptr; + headers = curl_slist_append(headers, "Content-Type: application/json"); headers = curl_slist_append(headers, - "Content-Type: application/x-www-form-urlencoded"); + ("Authorization: DeepL-Auth-Key " + api_key_).c_str()); curl_easy_setopt(curl.get(), CURLOPT_HTTPHEADER, headers); CURLcode res = curl_easy_perform(curl.get()); @@ -65,19 +70,40 @@ std::string DeepLTranslator::translate(const std::string &text, const std::strin curl_slist_free_all(headers); if (res != CURLE_OK) { - throw TranslationError(std::string("CURL request failed: ") + + throw TranslationError(std::string("DeepL: CURL request failed: ") + curl_easy_strerror(res)); } return parseResponse(response); } catch (const json::exception &e) { - throw TranslationError(std::string("JSON parsing error: ") + e.what()); + throw TranslationError(std::string("DeepL JSON parsing error: ") + e.what() + + ". Response: " + response); } } std::string DeepLTranslator::parseResponse(const std::string &response_str) { + // Handle rate limiting errors + long response_code; + curl_easy_getinfo(curl_easy_init(), CURLINFO_RESPONSE_CODE, &response_code); + if (response_code == 429) { + throw TranslationError("DeepL API Error: Rate limit exceeded"); + } + if (response_code == 456) { + throw TranslationError("DeepL API Error: Quota exceeded"); + } + + /* + { + "translations": [ + { + "detected_source_language": "EN", + "text": "Hallo, Welt!" + } + ] + } + */ json response = json::parse(response_str); // Check for API errors @@ -86,13 +112,6 @@ std::string DeepLTranslator::parseResponse(const std::string &response_str) response["message"].get()); } - // Handle rate limiting errors - long response_code; - curl_easy_getinfo(curl_easy_init(), CURLINFO_RESPONSE_CODE, &response_code); - if (response_code == 429) { - throw TranslationError("DeepL API Error: Rate limit exceeded"); - } - try { // DeepL returns translations array with detected language const auto &translation = response["translations"][0]; @@ -104,6 +123,6 @@ std::string DeepLTranslator::parseResponse(const std::string &response_str) return translation["text"].get(); } catch (const json::exception &) { - throw TranslationError("Unexpected response format from DeepL API"); + throw TranslationError("DeepL: Unexpected response format from DeepL API"); } } diff --git a/src/translation/cloud-translation/deepl.h b/src/translation/cloud-translation/deepl.h index 9f661af..b7e2a54 100644 --- a/src/translation/cloud-translation/deepl.h +++ b/src/translation/cloud-translation/deepl.h @@ -6,7 +6,7 @@ class CurlHelper; // Forward declaration class DeepLTranslator : public ITranslator { public: - explicit DeepLTranslator(const std::string &api_key); + explicit DeepLTranslator(const std::string &api_key, bool free = false); ~DeepLTranslator() override; std::string translate(const std::string &text, const std::string &target_lang, @@ -16,5 +16,6 @@ class DeepLTranslator : public ITranslator { std::string parseResponse(const std::string &response_str); std::string api_key_; + bool free_; std::unique_ptr curl_helper_; }; diff --git a/src/translation/cloud-translation/google-cloud.cpp b/src/translation/cloud-translation/google-cloud.cpp index 65ff7c5..f20c0b1 100644 --- a/src/translation/cloud-translation/google-cloud.cpp +++ b/src/translation/cloud-translation/google-cloud.cpp @@ -30,10 +30,10 @@ std::string GoogleTranslator::translate(const std::string &text, const std::stri std::stringstream url; url << "https://translation.googleapis.com/language/translate/v2" << "?key=" << api_key_ << "&q=" << CurlHelper::urlEncode(curl.get(), text) - << "&target=" << target_lang; + << "&target=" << sanitize_language_code(target_lang); if (source_lang != "auto") { - url << "&source=" << source_lang; + url << "&source=" << sanitize_language_code(source_lang); } // Set up curl options diff --git a/src/translation/cloud-translation/openai.cpp b/src/translation/cloud-translation/openai.cpp index 7d7cda8..088da74 100644 --- a/src/translation/cloud-translation/openai.cpp +++ b/src/translation/cloud-translation/openai.cpp @@ -5,6 +5,8 @@ #include #include +#include "translation/language_codes.h" + using json = nlohmann::json; OpenAITranslator::OpenAITranslator(const std::string &api_key, const std::string &model) @@ -16,50 +18,6 @@ OpenAITranslator::OpenAITranslator(const std::string &api_key, const std::string OpenAITranslator::~OpenAITranslator() = default; -std::string OpenAITranslator::getLanguageName(const std::string &lang_code) const -{ - static const std::unordered_map language_names = { - {"auto", "auto-detected language"}, - {"en", "English"}, - {"es", "Spanish"}, - {"fr", "French"}, - {"de", "German"}, - {"it", "Italian"}, - {"pt", "Portuguese"}, - {"nl", "Dutch"}, - {"pl", "Polish"}, - {"ru", "Russian"}, - {"ja", "Japanese"}, - {"ko", "Korean"}, - {"zh", "Chinese"}, - {"ar", "Arabic"}, - {"hi", "Hindi"}, - {"bn", "Bengali"}, - {"uk", "Ukrainian"}, - {"vi", "Vietnamese"}, - {"th", "Thai"}, - {"tr", "Turkish"}, - // Add more languages as needed - }; - - auto it = language_names.find(lang_code); - if (it != language_names.end()) { - return it->second; - } - return lang_code; // Return the code itself if no mapping exists -} - -bool OpenAITranslator::isLanguageSupported(const std::string &lang_code) const -{ - static const std::unordered_set supported_languages = { - "auto", "en", "es", "fr", "de", "it", "pt", "nl", "pl", "ru", "ja", - "ko", "zh", "ar", "hi", "bn", "uk", "vi", "th", "tr" - // Add more supported languages as needed - }; - - return supported_languages.find(lang_code) != supported_languages.end(); -} - std::string OpenAITranslator::createSystemPrompt(const std::string &target_lang) const { std::string target_language = getLanguageName(target_lang); diff --git a/src/translation/cloud-translation/openai.h b/src/translation/cloud-translation/openai.h index a82f811..a7787cf 100644 --- a/src/translation/cloud-translation/openai.h +++ b/src/translation/cloud-translation/openai.h @@ -16,8 +16,6 @@ class OpenAITranslator : public ITranslator { private: std::string parseResponse(const std::string &response_str); std::string createSystemPrompt(const std::string &target_lang) const; - std::string getLanguageName(const std::string &lang_code) const; - bool isLanguageSupported(const std::string &lang_code) const; std::string api_key_; std::string model_; diff --git a/src/translation/cloud-translation/translation-cloud.cpp b/src/translation/cloud-translation/translation-cloud.cpp index a1f4537..6120698 100644 --- a/src/translation/cloud-translation/translation-cloud.cpp +++ b/src/translation/cloud-translation/translation-cloud.cpp @@ -21,9 +21,9 @@ std::unique_ptr createTranslator(const CloudTranslatorConfig &confi if (config.provider == "google") { return std::make_unique(config.access_key); } else if (config.provider == "deepl") { - return std::make_unique(config.access_key); + return std::make_unique(config.access_key, config.free); } else if (config.provider == "azure") { - return std::make_unique(config.access_key, config.location); + return std::make_unique(config.access_key, config.region); // } else if (config.provider == "aws") { // return std::make_unique(config.access_key, config.secret_key, config.region); } else if (config.provider == "papago") { diff --git a/src/translation/cloud-translation/translation-cloud.h b/src/translation/cloud-translation/translation-cloud.h index 4659042..f90de17 100644 --- a/src/translation/cloud-translation/translation-cloud.h +++ b/src/translation/cloud-translation/translation-cloud.h @@ -6,9 +6,9 @@ struct CloudTranslatorConfig { std::string provider; std::string access_key; // Main API key/Client ID std::string secret_key; // Secret key/Client secret - std::string region; // For AWS - std::string location; // For Azure + std::string region; // For AWS / Azure std::string model; // For Claude + bool free; // For Deepl }; std::string translate_cloud(const CloudTranslatorConfig &config, const std::string &text, diff --git a/src/translation/language_codes.h b/src/translation/language_codes.h index fb4890e..8cafb94 100644 --- a/src/translation/language_codes.h +++ b/src/translation/language_codes.h @@ -9,4 +9,29 @@ extern std::map language_codes_reverse; extern std::map language_codes_from_whisper; extern std::map language_codes_to_whisper; +inline bool isLanguageSupported(const std::string &lang_code) +{ + return language_codes.find(lang_code) != language_codes.end() || + language_codes_from_whisper.find(lang_code) != language_codes_from_whisper.end(); +} + +inline std::string getLanguageName(const std::string &lang_code) +{ + auto it = language_codes.find(lang_code); + if (it != language_codes.end()) { + return it->second; + } + // check if it's a whisper language code + it = language_codes_from_whisper.find(lang_code); + if (it != language_codes_from_whisper.end()) { + // convert to the language code + const std::string &whisper_code = it->second; + it = language_codes.find(whisper_code); + if (it != language_codes.end()) { + return it->second; + } + } + return lang_code; // Return the code itself if no mapping exists +} + #endif // LANGUAGE_CODES_H From b2a540c8328e93cdc69692731dcfbe2d96f67f81 Mon Sep 17 00:00:00 2001 From: Roy Shilkrot Date: Fri, 22 Nov 2024 14:55:55 -0500 Subject: [PATCH 09/10] Add ccache detection to ICU build configuration for improved compilation speed --- cmake/BuildICU.cmake | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/cmake/BuildICU.cmake b/cmake/BuildICU.cmake index 2c48f87..48e0b2c 100644 --- a/cmake/BuildICU.cmake +++ b/cmake/BuildICU.cmake @@ -48,6 +48,14 @@ if(WIN32) "${ICU_LIB_${lib}}") endforeach() else() + # Add ccache detection at the start + find_program(CCACHE_PROGRAM ccache) + if(CCACHE_PROGRAM) + message(STATUS "Found ccache: ${CCACHE_PROGRAM}") + set(CMAKE_C_COMPILER_LAUNCHER "${CCACHE_PROGRAM}") + set(CMAKE_CXX_COMPILER_LAUNCHER "${CCACHE_PROGRAM}") + endif() + set(ICU_URL "https://github.com/unicode-org/icu/releases/download/release-${ICU_VERSION_DASH}/icu4c-${ICU_VERSION_UNDERSCORE}-src.tgz" ) @@ -55,10 +63,20 @@ else() if(APPLE) set(ICU_PLATFORM "MacOSX") set(TARGET_ARCH -arch\ $ENV{MACOS_ARCH}) - set(ICU_BUILD_ENV_VARS CFLAGS=${TARGET_ARCH} CXXFLAGS=${TARGET_ARCH} LDFLAGS=${TARGET_ARCH}) + set(ICU_BUILD_ENV_VARS + CFLAGS=${TARGET_ARCH} + CXXFLAGS=${TARGET_ARCH} + LDFLAGS=${TARGET_ARCH} + CC="${CMAKE_C_COMPILER_LAUNCHER} ${CMAKE_C_COMPILER}" + CXX="${CMAKE_CXX_COMPILER_LAUNCHER} ${CMAKE_CXX_COMPILER}") else() set(ICU_PLATFORM "Linux") - set(ICU_BUILD_ENV_VARS CFLAGS=-fPIC CXXFLAGS=-fPIC LDFLAGS=-fPIC) + set(ICU_BUILD_ENV_VARS + CFLAGS=-fPIC + CXXFLAGS=-fPIC + LDFLAGS=-fPIC + CC="${CMAKE_C_COMPILER_LAUNCHER} ${CMAKE_C_COMPILER}" + CXX="${CMAKE_CXX_COMPILER_LAUNCHER} ${CMAKE_CXX_COMPILER}") endif() ExternalProject_Add( From b4b4cb4bb78244f712916e301f482aad37c6dbb5 Mon Sep 17 00:00:00 2001 From: Roy Shilkrot Date: Fri, 22 Nov 2024 15:09:06 -0500 Subject: [PATCH 10/10] Enhance ICU build configuration to use ccache as a compiler wrapper for improved performance --- cmake/BuildICU.cmake | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/cmake/BuildICU.cmake b/cmake/BuildICU.cmake index 48e0b2c..f3ce5f5 100644 --- a/cmake/BuildICU.cmake +++ b/cmake/BuildICU.cmake @@ -52,8 +52,9 @@ else() find_program(CCACHE_PROGRAM ccache) if(CCACHE_PROGRAM) message(STATUS "Found ccache: ${CCACHE_PROGRAM}") - set(CMAKE_C_COMPILER_LAUNCHER "${CCACHE_PROGRAM}") - set(CMAKE_CXX_COMPILER_LAUNCHER "${CCACHE_PROGRAM}") + # Create compiler wrapper commands + set(C_LAUNCHER "${CCACHE_PROGRAM} ${CMAKE_C_COMPILER}") + set(CXX_LAUNCHER "${CCACHE_PROGRAM} ${CMAKE_CXX_COMPILER}") endif() set(ICU_URL @@ -63,20 +64,11 @@ else() if(APPLE) set(ICU_PLATFORM "MacOSX") set(TARGET_ARCH -arch\ $ENV{MACOS_ARCH}) - set(ICU_BUILD_ENV_VARS - CFLAGS=${TARGET_ARCH} - CXXFLAGS=${TARGET_ARCH} - LDFLAGS=${TARGET_ARCH} - CC="${CMAKE_C_COMPILER_LAUNCHER} ${CMAKE_C_COMPILER}" - CXX="${CMAKE_CXX_COMPILER_LAUNCHER} ${CMAKE_CXX_COMPILER}") + set(ICU_BUILD_ENV_VARS CFLAGS=${TARGET_ARCH} CXXFLAGS=${TARGET_ARCH} LDFLAGS=${TARGET_ARCH} CC=${C_LAUNCHER} + CXX=${CXX_LAUNCHER}) else() set(ICU_PLATFORM "Linux") - set(ICU_BUILD_ENV_VARS - CFLAGS=-fPIC - CXXFLAGS=-fPIC - LDFLAGS=-fPIC - CC="${CMAKE_C_COMPILER_LAUNCHER} ${CMAKE_C_COMPILER}" - CXX="${CMAKE_CXX_COMPILER_LAUNCHER} ${CMAKE_CXX_COMPILER}") + set(ICU_BUILD_ENV_VARS CFLAGS=-fPIC CXXFLAGS=-fPIC LDFLAGS=-fPIC CC=${C_LAUNCHER} CXX=${CXX_LAUNCHER}) endif() ExternalProject_Add(