diff --git a/scripts/clickhouse_import_support/clone_mysql_database.sh b/scripts/clickhouse_import_support/clone_mysql_database.sh index 8feac02..5bff0e2 100755 --- a/scripts/clickhouse_import_support/clone_mysql_database.sh +++ b/scripts/clickhouse_import_support/clone_mysql_database.sh @@ -118,7 +118,7 @@ function set_database_table_list() { return 1 fi unset sql_data_array - if ! set_sql_data_array_from_file "$database_table_list_filepath" 0 ; then + if ! set_mysql_sql_data_array_from_file "$database_table_list_filepath" 0 ; then return 1 fi database_table_list=(${sql_data_array[@]}) @@ -146,7 +146,7 @@ function set_source_database_create_table_statement_list() { echo "Warning : failed to execute mysql statement : $statement" >&2 return 1 fi - if ! set_sql_data_array_from_file "$create_table_statement_filepath" 1 ; then + if ! set_mysql_sql_data_array_from_file "$create_table_statement_filepath" 1 ; then return 1 fi source_database_create_table_statement_list+=("${sql_data_array[0]}") @@ -192,7 +192,7 @@ function destination_table_matches_source_table() { local get_record_counts_statement="SELECT count(*) AS record_count from $destination_table_full_name UNION DISTINCT SELECT count(*) as record_count from $source_table_full_name;" execute_sql_statement_via_mysql "$get_record_counts_statement" "$record_count_comparison_filepath" # if recourd_counts match, only one distinct value will be returned. If they differ, there will be 2 values - set_sql_data_array_from_file "$record_count_comparison_filepath" 0 + set_mysql_sql_data_array_from_file "$record_count_comparison_filepath" 0 if [[ "${#sql_data_array[@]}" -ne 1 ]] ; then local record_count_0="${sql_data_array[0]}" local record_count_1="${sql_data_array[1]}" diff --git a/scripts/clickhouse_import_support/copy_mysql_database_tables_to_clickhouse.sh b/scripts/clickhouse_import_support/copy_mysql_database_tables_to_clickhouse.sh index 92adc25..e8f7321 100755 --- a/scripts/clickhouse_import_support/copy_mysql_database_tables_to_clickhouse.sh +++ b/scripts/clickhouse_import_support/copy_mysql_database_tables_to_clickhouse.sh @@ -14,6 +14,7 @@ fi unset this_script_dir function usage() { + # TODO! echo "usage:" } @@ -66,7 +67,7 @@ function clickhouse_is_responding() { while [ $remaining_try_count -ne 0 ] ; do if execute_sql_statement_via_sling "$statement" "clickhouse" "$clickhouse_is_responsive_filepath" ; then unset sql_data_array - if set_sql_data_array_from_file "$clickhouse_is_responsive_filepath" 0 ; then + if set_sling_sql_data_array_from_file "$clickhouse_is_responsive_filepath" 0 ; then local clickhouse_response=${sql_data_array[0]} if [ "$clickhouse_response" == "1" ] ; then return 0 @@ -101,7 +102,7 @@ function set_database_table_list() { return 1 fi unset sql_data_array - if ! set_sql_data_array_from_file "$database_table_list_filepath" 0 ; then + if ! set_sling_sql_data_array_from_file "$database_table_list_filepath" 0 ; then return 1 fi database_table_list=(${sql_data_array[@]}) @@ -117,7 +118,7 @@ function set_mysql_table_record_count() { return 1 fi unset sql_data_array - if ! set_sql_data_array_from_file "$mysql_table_record_count_filepath" 0 ; then + if ! set_sling_sql_data_array_from_file "$mysql_table_record_count_filepath" 0 ; then return 1 fi mysql_table_record_count=${sql_data_array[0]} @@ -192,7 +193,7 @@ function destination_table_matches_source_table() { return 1 fi unset sql_data_array - if ! set_sql_data_array_from_file "$clickhouse_table_record_count_filepath" 0 ; then + if ! set_sling_sql_data_array_from_file "$clickhouse_table_record_count_filepath" 0 ; then return 1 fi local clickhouse_table_record_count=${sql_data_array[0]} diff --git a/scripts/clickhouse_import_support/create_derived_tables_in_clickhouse_database.sh b/scripts/clickhouse_import_support/create_derived_tables_in_clickhouse_database.sh index c8b27c5..e8a6b33 100755 --- a/scripts/clickhouse_import_support/create_derived_tables_in_clickhouse_database.sh +++ b/scripts/clickhouse_import_support/create_derived_tables_in_clickhouse_database.sh @@ -59,7 +59,7 @@ function clickhouse_is_responding() { while [ $remaining_try_count -ne 0 ] ; do if execute_sql_statement_via_clickhouse_client "$statement" "$clickhouse_is_responsive_filepath" ; then unset sql_data_array - if set_sql_data_array_from_file "$clickhouse_is_responsive_filepath" 0 ; then + if set_clickhouse_sql_data_array_from_file "$clickhouse_is_responsive_filepath" 0 ; then local clickhouse_response=${sql_data_array[0]} if [ "$clickhouse_response" == "1" ] ; then return 0 diff --git a/scripts/clickhouse_import_support/drop_tables_in_clickhouse_database.sh b/scripts/clickhouse_import_support/drop_tables_in_clickhouse_database.sh index d83fc03..d20f498 100755 --- a/scripts/clickhouse_import_support/drop_tables_in_clickhouse_database.sh +++ b/scripts/clickhouse_import_support/drop_tables_in_clickhouse_database.sh @@ -62,7 +62,7 @@ function clickhouse_is_responding() { while [ $remaining_try_count -ne 0 ] ; do if execute_sql_statement_via_clickhouse_client "$statement" "$clickhouse_is_responsive_filepath" ; then unset sql_data_array - if set_sql_data_array_from_file "$clickhouse_is_responsive_filepath" 0 ; then + if set_clickhouse_sql_data_array_from_file "$clickhouse_is_responsive_filepath" 0 ; then local clickhouse_response=${sql_data_array[0]} if [ "$clickhouse_response" == "1" ] ; then return 0 @@ -93,7 +93,7 @@ function set_database_table_list() { return 1 fi unset sql_data_array - if ! set_sql_data_array_from_file "$database_table_list_filepath" 0 ; then + if ! set_clickhouse_sql_data_array_from_file "$database_table_list_filepath" 0 ; then return 1 fi database_table_list=(${sql_data_array[@]}) diff --git a/scripts/clickhouse_import_support/mysql_command_line_functions.sh b/scripts/clickhouse_import_support/mysql_command_line_functions.sh index e049f8a..d2e71a5 100644 --- a/scripts/clickhouse_import_support/mysql_command_line_functions.sh +++ b/scripts/clickhouse_import_support/mysql_command_line_functions.sh @@ -58,7 +58,7 @@ function execute_sql_statement_via_mysql() { mysql --defaults-extra-file="$configured_mysql_defaults_config_file_path" --batch $extra_args <<< "$statement" > $output_filepath } -function set_sql_data_field_value_from_record() { +function set_mysql_sql_data_field_value_from_record() { local record_string=$1 local column_number=$2 unset sql_data_field_value @@ -138,7 +138,7 @@ function set_sql_data_field_value_from_record() { sql_data_field_value="$parsed_value" } -function set_sql_data_array_from_file() { +function set_mysql_sql_data_array_from_file() { local filepath=$1 local column_number=$2 unset sql_data_array @@ -152,7 +152,7 @@ function set_sql_data_array_from_file() { if [ "$headers_have_been_parsed" -eq 0 ] ; then headers_have_been_parsed=1 else - set_sql_data_field_value_from_record "$line" "$column_number" + set_mysql_sql_data_field_value_from_record "$line" "$column_number" sql_data_array+=("$sql_data_field_value") fi done < "$filepath" @@ -165,7 +165,7 @@ function database_exists() { echo "Warning : unable to determine if database $database_name exists using : $statement" >&2 return 1 fi - set_sql_data_array_from_file "$database_exists_filepath" 0 + set_mysql_sql_data_array_from_file "$database_exists_filepath" 0 if [[ "${#sql_data_array[@]}" -ne 1 ]] ; then echo "Warning : database $database_name not present on database server, or there are multiple listings for that name" >&2 return 2 @@ -191,7 +191,7 @@ function database_is_empty() { echo "Warning : unable to retrieve table list from database $database_name using : $statement" >&2 return 1 fi - set_sql_data_array_from_file "$database_table_list_filepath" 0 + set_mysql_sql_data_array_from_file "$database_table_list_filepath" 0 if [[ "${#sql_data_array[@]}" -ne 0 ]] ; then echo "Warning : database $database_name has tables (is not empty as required)" >&2 return 2 diff --git a/scripts/clickhouse_import_support/sling_command_line_functions.sh b/scripts/clickhouse_import_support/sling_command_line_functions.sh index a7f93b4..15e972c 100644 --- a/scripts/clickhouse_import_support/sling_command_line_functions.sh +++ b/scripts/clickhouse_import_support/sling_command_line_functions.sh @@ -143,7 +143,7 @@ function transfer_table_data_via_sling() { ) } -# set_sql_data_field_value_from_record +# set_sling_sql_data_field_value_from_record # This function currently assumes that all queries are going to produce simple results # which do not cause the quotation of value fields. Output from sling is in comma separated # value format, so any value which contains a comma, or which contains a quotation mark or @@ -152,7 +152,7 @@ function transfer_table_data_via_sling() { # or record counts from tables. If use of this function expands to more general value retrieval # it may be necessary to add proper parsing of quoted values. The presence of quotation marks # on the line currently cuases a failure to parse. -function set_sql_data_field_value_from_record() { +function set_sling_sql_data_field_value_from_record() { local record_string=$1 local column_number=$2 unset sql_data_field_value @@ -200,7 +200,7 @@ function set_sql_data_field_value_from_record() { sql_data_field_value="$parsed_value" } -function set_sql_data_array_from_file() { +function set_sling_sql_data_array_from_file() { local filepath=$1 local column_number=$2 unset sql_data_array @@ -214,7 +214,7 @@ function set_sql_data_array_from_file() { if [ "$headers_have_been_parsed" -eq 0 ] ; then headers_have_been_parsed=1 else - set_sql_data_field_value_from_record "$line" "$column_number" + set_sling_sql_data_field_value_from_record "$line" "$column_number" sql_data_array+=("$sql_data_field_value") fi done < "$filepath" @@ -227,7 +227,7 @@ function clickhouse_database_exists() { echo "Warning : unable to determine if database $database_name exists using : $statement" >&2 return 1 fi - set_sql_data_array_from_file "$sling_database_exists_filepath" 0 + set_sling_sql_data_array_from_file "$sling_database_exists_filepath" 0 if [[ "${sql_data_array[0]}" -ne 1 ]] ; then echo "Warning : database $database_name not present on database server, or there are multiple listings for that name" >&2 return 2 @@ -243,7 +243,7 @@ function clickhouse_database_is_empty() { echo "Warning : unable to retrieve table/view list from database $database_name using : $statement" >&2 return 1 fi - set_sql_data_array_from_file "$sling_database_table_list_filepath" 0 + set_sling_sql_data_array_from_file "$sling_database_table_list_filepath" 0 if [[ "${sql_data_array[0]}" -ne 0 ]] ; then echo "Warning : database $database_name has tables or views (is not empty as required)" >&2 return 2 diff --git a/scripts/clickhouse_import_support/synchronize_user_tables_between_databases.sh b/scripts/clickhouse_import_support/synchronize_user_tables_between_databases.sh index e9793fb..cb7996b 100755 --- a/scripts/clickhouse_import_support/synchronize_user_tables_between_databases.sh +++ b/scripts/clickhouse_import_support/synchronize_user_tables_between_databases.sh @@ -11,17 +11,24 @@ if ! source "$this_script_dir/mysql_command_line_functions.sh" ; then echo "Error : unable to load dependency : $this_script_dir/mysql_command_line_functions.sh" >&2 exit 1 fi +if ! source "$this_script_dir/clickhouse_client_command_line_functions.sh" ; then + echo "Error : unable to load dependency : $this_script_dir/clickhouse_client_command_line_functions.sh" >&2 + exit 1 +fi unset this_script_dir # other non-local environment variables in use unset my_properties -unset source_database_name -unset destination_database_name +unset mysql_source_database_name +unset mysql_destination_database_name +unset clickhouse_destination_database_name declare -A my_properties missing_user_records_result_filepath="$(pwd)/missing_user_records.txt" missing_authority_records_result_filepath="$(pwd)/missing_authority_records.txt" insert_user_record_result_filepath="$(pwd)/insert_user_result.txt" insert_authority_record_result_filepath="$(pwd)/insert_authority_result.txt" +clickhouse_is_responsive_filepath="$(pwd)/cdtcd_cmd_clickhouse_is_responsive.txt" +SECONDS_BETWEEN_RESPONSIVENESS_RETRY=$((60)) function usage() { echo "usage: synchronize_user_tables_between_databases.sh properties_filepath database_to_synchronize_from database_to_synchronize_to" >&2 @@ -40,12 +47,16 @@ function initialize_main() { usage return 1 fi + if ! initialize_clickhouse_client_command_line_functions "$database_to_synchronize_to" ; then + usage + return 1 + fi remove_credentials_from_properties my_properties # no longer needed - remove for security if [ "$database_to_synchronize_from" == "blue" ] ; then - source_database_name="${my_properties['mysql_blue_database_name']}" + mysql_source_database_name="${my_properties['mysql_blue_database_name']}" else if [ "$database_to_synchronize_from" == "green" ] ; then - source_database_name="${my_properties['mysql_green_database_name']}" + mysql_source_database_name="${my_properties['mysql_green_database_name']}" else echo "Error : database_to_synchronize_from must be one of {blue, green}" >&2 usage @@ -53,10 +64,12 @@ function initialize_main() { fi fi if [ "$database_to_synchronize_to" == "blue" ] ; then - destination_database_name="${my_properties['mysql_blue_database_name']}" + mysql_destination_database_name="${my_properties['mysql_blue_database_name']}" + clickhouse_destination_database_name="${my_properties['clickhouse_blue_database_name']}" else if [ "$database_to_synchronize_to" == "green" ] ; then - destination_database_name="${my_properties['mysql_green_database_name']}" + mysql_destination_database_name="${my_properties['mysql_green_database_name']}" + clickhouse_destination_database_name="${my_properties['clickhouse_green_database_name']}" else echo "Error : database_to_synchronize_to must be one of {blue, green}" >&2 usage @@ -75,23 +88,56 @@ function delete_output_stream_files() { rm -f "$missing_authority_records_result_filepath" rm -f "$insert_user_record_result_filepath" rm -f "$insert_authority_record_result_filepath" + rm -f "$clickhouse_is_responsive_filepath" } function shutdown_main_and_clean_up() { shutdown_mysql_command_line_functions delete_output_stream_files unset my_properties - unset source_database_name - unset destination_database_name + unset mysql_source_database_name + unset mysql_destination_database_name + unset clickhouse_destination_database_name unset missing_user_records_result_filepath unset missing_authority_records_result_filepath unset insert_user_record_result_filepath unset insert_authority_record_result_filepath + unset clickhouse_is_responsive_filepath + unset SECONDS_BETWEEN_RESPONSIVENESS_RETRY +} + +function mysql_destination_database_exists() { + if ! database_exists "$mysql_destination_database_name" ; then + echo "Error : could not proceed with database cloning because destination database does not exist: $mysql_destination_database_name" >&2 + return 1 + fi + return 0 +} + +function clickhouse_is_responding() { + local remaining_try_count=3 + local statement="SELECT 1" + while [ $remaining_try_count -ne 0 ] ; do + if execute_sql_statement_via_clickhouse_client "$statement" "$clickhouse_is_responsive_filepath" ; then + unset sql_data_array + if set_clickhouse_sql_data_array_from_file "$clickhouse_is_responsive_filepath" 0 ; then + local clickhouse_response=${sql_data_array[0]} + if [ "$clickhouse_response" == "1" ] ; then + return 0 + fi + fi + fi + remaining_try_count=$((remaining_try_count-1)) + if [ $remaining_try_count -gt 0 ] ; then + sleep $SECONDS_BETWEEN_RESPONSIVENESS_RETRY + fi + done + return 1 } -function destination_database_exists() { - if ! database_exists "$destination_database_name" ; then - echo "Error : could not proceed with database cloning because destination database does not exist: $destination_database_name" >&2 +function clickhouse_destination_database_exists() { + if ! clickhouse_database_exists "$clickhouse_destination_database_name"; then + echo "Error : could not proceed with creation of derived tables because database does not exist: $clickhouse_destination_database_name" >&2 return 1 fi return 0 @@ -99,8 +145,8 @@ function destination_database_exists() { function find_records_missing_in_destination_database_users_table() { local table_name="users" - local source_table_full_name="\`$source_database_name\`.\`$table_name\`" - local destination_table_full_name="\`$destination_database_name\`.\`$table_name\`" + local source_table_full_name="\`$mysql_source_database_name\`.\`$table_name\`" + local destination_table_full_name="\`$mysql_destination_database_name\`.\`$table_name\`" local find_records_missing_in_destination_database_table_statement="SELECT $source_table_full_name.EMAIL, $source_table_full_name.NAME, $source_table_full_name.ENABLED FROM $source_table_full_name LEFT JOIN $destination_table_full_name ON $source_table_full_name.EMAIL=$destination_table_full_name.EMAIL WHERE $destination_table_full_name.EMAIL IS NULL;" if ! execute_sql_statement_via_mysql "$find_records_missing_in_destination_database_table_statement" "$missing_user_records_result_filepath" ; then return 1 @@ -110,8 +156,8 @@ function find_records_missing_in_destination_database_users_table() { function find_records_missing_in_destination_database_authorities_table() { local table_name="authorities" - local source_table_full_name="\`$source_database_name\`.\`$table_name\`" - local destination_table_full_name="\`$destination_database_name\`.\`$table_name\`" + local source_table_full_name="\`$mysql_source_database_name\`.\`$table_name\`" + local destination_table_full_name="\`$mysql_destination_database_name\`.\`$table_name\`" local find_records_missing_in_destination_database_table_statement="SELECT $source_table_full_name.EMAIL, $source_table_full_name.AUTHORITY FROM $source_table_full_name LEFT JOIN $destination_table_full_name ON $source_table_full_name.EMAIL=$destination_table_full_name.EMAIL AND $source_table_full_name.AUTHORITY=$destination_table_full_name.AUTHORITY WHERE $destination_table_full_name.EMAIL IS NULL;" if ! execute_sql_statement_via_mysql "$find_records_missing_in_destination_database_table_statement" "$missing_authority_records_result_filepath" ; then return 1 @@ -119,44 +165,69 @@ function find_records_missing_in_destination_database_authorities_table() { return 0 } -function insert_missing_user_record_into_destination_database() { +function insert_missing_user_record_into_mysql_destination_database() { + local email=$1 + local name=$2 + local enabled=$3 + local table_name="users" + local destination_table_full_name="\`$mysql_destination_database_name\`.\`$table_name\`" + local insert_missing_user_record_into_mysql_destination_database_table_statement="INSERT INTO $destination_table_full_name (EMAIL, NAME, ENABLED) VALUES ('$email', '$name', '$enabled');" + if ! execute_sql_statement_via_mysql "$insert_missing_user_record_into_mysql_destination_database_table_statement" "$insert_user_record_result_filepath" ; then + return 1 + fi + return 0 +} + +function insert_missing_authority_record_into_mysql_destination_database() { + local email=$1 + local authority=$2 + local table_name="authorities" + local destination_table_full_name="\`$mysql_destination_database_name\`.\`$table_name\`" + local insert_missing_authority_record_into_mysql_destination_database_table_statement="INSERT INTO $destination_table_full_name (EMAIL, AUTHORITY) VALUES ('$email', '$authority');" + if ! execute_sql_statement_via_mysql "$insert_missing_authority_record_into_mysql_destination_database_table_statement" "$insert_authority_record_result_filepath" ; then + return 1 + fi + return 0 +} + +function insert_missing_user_record_into_clickhouse_destination_database() { local email=$1 local name=$2 local enabled=$3 local table_name="users" - local destination_table_full_name="\`$destination_database_name\`.\`$table_name\`" - local insert_missing_user_record_into_destination_database_table_statement="INSERT INTO $destination_table_full_name (EMAIL, NAME, ENABLED) VALUES ('$email', '$name', '$enabled');" - if ! execute_sql_statement_via_mysql "$insert_missing_user_record_into_destination_database_table_statement" "$insert_user_record_result_filepath" ; then + local destination_table_full_name="\`$clickhouse_destination_database_name\`.\`$table_name\`" + local insert_missing_user_record_into_clickhouse_destination_database_table_statement="INSERT INTO $destination_table_full_name (email, name, enabled) VALUES ('$email', '$name', '$enabled');" + if ! execute_sql_statement_via_clickhouse_client "$insert_missing_user_record_into_clickhouse_destination_database_table_statement" "$insert_user_record_result_filepath" ; then return 1 fi return 0 } -function insert_missing_authority_record_into_destination_database() { +function insert_missing_authority_record_into_clickhouse_destination_database() { local email=$1 local authority=$2 local table_name="authorities" - local destination_table_full_name="\`$destination_database_name\`.\`$table_name\`" - local insert_missing_authority_record_into_destination_database_table_statement="INSERT INTO $destination_table_full_name (EMAIL, AUTHORITY) VALUES ('$email', '$authority');" - if ! execute_sql_statement_via_mysql "$insert_missing_authority_record_into_destination_database_table_statement" "$insert_authority_record_result_filepath" ; then + local destination_table_full_name="\`$clickhouse_destination_database_name\`.\`$table_name\`" + local insert_missing_authority_record_into_clickhouse_destination_database_table_statement="INSERT INTO $destination_table_full_name (email, authority) VALUES ('$email', '$authority');" + if ! execute_sql_statement_via_clickhouse_client "$insert_missing_authority_record_into_clickhouse_destination_database_table_statement" "$insert_authority_record_result_filepath" ; then return 1 fi return 0 } -function insert_missing_records_into_destination_database_users_table() { +function insert_missing_records_into_mysql_destination_database_users_table() { unset sql_data_array - if ! set_sql_data_array_from_file "$missing_user_records_result_filepath" 0 ; then + if ! set_mysql_sql_data_array_from_file "$missing_user_records_result_filepath" 0 ; then return 1 fi local email_list=(${sql_data_array[@]}) unset sql_data_array - if ! set_sql_data_array_from_file "$missing_user_records_result_filepath" 1 ; then + if ! set_mysql_sql_data_array_from_file "$missing_user_records_result_filepath" 1 ; then return 1 fi local name_list=(${sql_data_array[@]}) unset sql_data_array - if ! set_sql_data_array_from_file "$missing_user_records_result_filepath" 2 ; then + if ! set_mysql_sql_data_array_from_file "$missing_user_records_result_filepath" 2 ; then return 1 fi local enabled_list=(${sql_data_array[@]}) @@ -164,21 +235,22 @@ function insert_missing_records_into_destination_database_users_table() { local pos=0 local num_missing_records=${#email_list[@]} while [ $pos -lt $num_missing_records ]; do - echo "Adding user '${name_list[$pos]}' with email '${email_list[$pos]}' to $destination_database_name users table" - insert_missing_user_record_into_destination_database ${email_list[$pos]} ${name_list[$pos]} ${enabled_list[$pos]} + echo "Adding user '${name_list[$pos]}' with email '${email_list[$pos]}' to $mysql_destination_database_name users table" + insert_missing_user_record_into_mysql_destination_database ${email_list[$pos]} ${name_list[$pos]} ${enabled_list[$pos]} + insert_missing_user_record_into_clickhouse_destination_database ${email_list[$pos]} ${name_list[$pos]} ${enabled_list[$pos]} pos=$(($pos+1)) done return 0 } -function insert_missing_records_into_destination_database_authorities_table() { +function insert_missing_records_into_mysql_destination_database_authorities_table() { unset sql_data_array - if ! set_sql_data_array_from_file "$missing_authority_records_result_filepath" 0 ; then + if ! set_mysql_sql_data_array_from_file "$missing_authority_records_result_filepath" 0 ; then return 1 fi local email_list=(${sql_data_array[@]}) unset sql_data_array - if ! set_sql_data_array_from_file "$missing_authority_records_result_filepath" 1 ; then + if ! set_mysql_sql_data_array_from_file "$missing_authority_records_result_filepath" 1 ; then return 1 fi local authority_list=(${sql_data_array[@]}) @@ -186,8 +258,9 @@ function insert_missing_records_into_destination_database_authorities_table() { local pos=0 local num_missing_records=${#email_list[@]} while [ $pos -lt $num_missing_records ]; do - echo "Adding email '${email_list[$pos]}' with authority '${authority_list[$pos]}' to $destination_database_name authorities table" - insert_missing_authority_record_into_destination_database ${email_list[$pos]} ${authority_list[$pos]} + echo "Adding email '${email_list[$pos]}' with authority '${authority_list[$pos]}' to $mysql_destination_database_name authorities table" + insert_missing_authority_record_into_mysql_destination_database ${email_list[$pos]} ${authority_list[$pos]} + insert_missing_authority_record_into_clickhouse_destination_database ${email_list[$pos]} ${authority_list[$pos]} pos=$(($pos+1)) done return 0 @@ -199,11 +272,13 @@ function main() { local database_to_synchronize_to=$3 local exit_status=0 if ! initialize_main "$properties_filepath" "$database_to_synchronize_from" "$database_to_synchronize_to" || - ! destination_database_exists || + ! clickhouse_is_responding || + ! mysql_destination_database_exists || + ! clickhouse_destination_database_exists || ! find_records_missing_in_destination_database_users_table || ! find_records_missing_in_destination_database_authorities_table || - ! insert_missing_records_into_destination_database_users_table || - ! insert_missing_records_into_destination_database_authorities_table; then + ! insert_missing_records_into_mysql_destination_database_users_table || + ! insert_missing_records_into_mysql_destination_database_authorities_table; then exit_status=1 fi shutdown_main_and_clean_up