synchronize users to clickhouse as well as mysql and also renamed ove…

…rloaded functions that clashed in mysql/sling/clickhouse shell scripts

scripts/clickhouse_import_support/clone_mysql_database.sh

@@ -118,7 +118,7 @@ function set_database_table_list() {
         return 1
     fi
     unset sql_data_array
-    if ! set_sql_data_array_from_file "$database_table_list_filepath" 0 ; then
+    if ! set_mysql_sql_data_array_from_file "$database_table_list_filepath" 0 ; then
         return 1
     fi
     database_table_list=(${sql_data_array[@]})
@@ -146,7 +146,7 @@ function set_source_database_create_table_statement_list() {
             echo "Warning : failed to execute mysql statement : $statement" >&2
             return 1
         fi
-        if ! set_sql_data_array_from_file "$create_table_statement_filepath" 1 ; then
+        if ! set_mysql_sql_data_array_from_file "$create_table_statement_filepath" 1 ; then
             return 1
         fi
         source_database_create_table_statement_list+=("${sql_data_array[0]}")
@@ -192,7 +192,7 @@ function destination_table_matches_source_table() {
     local get_record_counts_statement="SELECT count(*) AS record_count from $destination_table_full_name UNION DISTINCT SELECT count(*) as record_count from $source_table_full_name;"
     execute_sql_statement_via_mysql "$get_record_counts_statement" "$record_count_comparison_filepath"
     # if recourd_counts match, only one distinct value will be returned. If they differ, there will be 2 values
-    set_sql_data_array_from_file "$record_count_comparison_filepath" 0
+    set_mysql_sql_data_array_from_file "$record_count_comparison_filepath" 0
     if [[ "${#sql_data_array[@]}" -ne 1 ]] ; then
         local record_count_0="${sql_data_array[0]}"
         local record_count_1="${sql_data_array[1]}"

scripts/clickhouse_import_support/copy_mysql_database_tables_to_clickhouse.sh

@@ -14,6 +14,7 @@ fi
 unset this_script_dir
 
 function usage() {
+    # TODO!
     echo "usage:"
 }
 
@@ -66,7 +67,7 @@ function clickhouse_is_responding() {
     while [ $remaining_try_count -ne 0 ] ; do
         if execute_sql_statement_via_sling "$statement" "clickhouse" "$clickhouse_is_responsive_filepath" ; then
             unset sql_data_array
-            if set_sql_data_array_from_file "$clickhouse_is_responsive_filepath" 0 ; then
+            if set_sling_sql_data_array_from_file "$clickhouse_is_responsive_filepath" 0 ; then
                 local clickhouse_response=${sql_data_array[0]}
                 if [ "$clickhouse_response" == "1" ] ; then
                     return 0
@@ -101,7 +102,7 @@ function set_database_table_list() {
         return 1
     fi
     unset sql_data_array
-    if ! set_sql_data_array_from_file "$database_table_list_filepath" 0 ; then
+    if ! set_sling_sql_data_array_from_file "$database_table_list_filepath" 0 ; then
         return 1
     fi
     database_table_list=(${sql_data_array[@]})
@@ -117,7 +118,7 @@ function set_mysql_table_record_count() {
         return 1
     fi
     unset sql_data_array
-    if ! set_sql_data_array_from_file "$mysql_table_record_count_filepath" 0 ; then
+    if ! set_sling_sql_data_array_from_file "$mysql_table_record_count_filepath" 0 ; then
         return 1
     fi
     mysql_table_record_count=${sql_data_array[0]}
@@ -192,7 +193,7 @@ function destination_table_matches_source_table() {
         return 1
     fi
     unset sql_data_array
-    if ! set_sql_data_array_from_file "$clickhouse_table_record_count_filepath" 0 ; then
+    if ! set_sling_sql_data_array_from_file "$clickhouse_table_record_count_filepath" 0 ; then
         return 1
     fi
     local clickhouse_table_record_count=${sql_data_array[0]}

scripts/clickhouse_import_support/create_derived_tables_in_clickhouse_database.sh

@@ -59,7 +59,7 @@ function clickhouse_is_responding() {
     while [ $remaining_try_count -ne 0 ] ; do
         if execute_sql_statement_via_clickhouse_client "$statement" "$clickhouse_is_responsive_filepath" ; then
             unset sql_data_array
-            if set_sql_data_array_from_file "$clickhouse_is_responsive_filepath" 0 ; then
+            if set_clickhouse_sql_data_array_from_file "$clickhouse_is_responsive_filepath" 0 ; then
                 local clickhouse_response=${sql_data_array[0]}
                 if [ "$clickhouse_response" == "1" ] ; then
                     return 0

scripts/clickhouse_import_support/drop_tables_in_clickhouse_database.sh

@@ -62,7 +62,7 @@ function clickhouse_is_responding() {
     while [ $remaining_try_count -ne 0 ] ; do
         if execute_sql_statement_via_clickhouse_client "$statement" "$clickhouse_is_responsive_filepath" ; then
             unset sql_data_array
-            if set_sql_data_array_from_file "$clickhouse_is_responsive_filepath" 0 ; then
+            if set_clickhouse_sql_data_array_from_file "$clickhouse_is_responsive_filepath" 0 ; then
                 local clickhouse_response=${sql_data_array[0]}
                 if [ "$clickhouse_response" == "1" ] ; then
                     return 0
@@ -93,7 +93,7 @@ function set_database_table_list() {
         return 1
     fi
     unset sql_data_array
-    if ! set_sql_data_array_from_file "$database_table_list_filepath" 0 ; then
+    if ! set_clickhouse_sql_data_array_from_file "$database_table_list_filepath" 0 ; then
         return 1
     fi
     database_table_list=(${sql_data_array[@]})

scripts/clickhouse_import_support/mysql_command_line_functions.sh

@@ -58,7 +58,7 @@ function execute_sql_statement_via_mysql() {
     mysql --defaults-extra-file="$configured_mysql_defaults_config_file_path" --batch $extra_args <<< "$statement" > $output_filepath 
 }
 
-function set_sql_data_field_value_from_record() {
+function set_mysql_sql_data_field_value_from_record() {
     local record_string=$1
     local column_number=$2
     unset sql_data_field_value
@@ -138,7 +138,7 @@ function set_sql_data_field_value_from_record() {
     sql_data_field_value="$parsed_value"
 }
 
-function set_sql_data_array_from_file() {
+function set_mysql_sql_data_array_from_file() {
     local filepath=$1
     local column_number=$2
     unset sql_data_array
@@ -152,7 +152,7 @@ function set_sql_data_array_from_file() {
         if [ "$headers_have_been_parsed" -eq 0 ] ; then
             headers_have_been_parsed=1
         else
-            set_sql_data_field_value_from_record "$line" "$column_number"
+            set_mysql_sql_data_field_value_from_record "$line" "$column_number"
             sql_data_array+=("$sql_data_field_value")
         fi
     done < "$filepath"
@@ -165,7 +165,7 @@ function database_exists() {
         echo "Warning : unable to determine if database $database_name exists using : $statement" >&2
         return 1
     fi
-    set_sql_data_array_from_file "$database_exists_filepath" 0
+    set_mysql_sql_data_array_from_file "$database_exists_filepath" 0
     if [[ "${#sql_data_array[@]}" -ne 1 ]] ; then
         echo "Warning : database $database_name not present on database server, or there are multiple listings for that name" >&2
         return 2
@@ -191,7 +191,7 @@ function database_is_empty() {
         echo "Warning : unable to retrieve table list from database $database_name using : $statement" >&2
         return 1
     fi
-    set_sql_data_array_from_file "$database_table_list_filepath" 0
+    set_mysql_sql_data_array_from_file "$database_table_list_filepath" 0
     if [[ "${#sql_data_array[@]}" -ne 0 ]] ; then
         echo "Warning : database $database_name has tables (is not empty as required)" >&2
         return 2

scripts/clickhouse_import_support/sling_command_line_functions.sh

@@ -143,7 +143,7 @@ function transfer_table_data_via_sling() {
     )
 }
 
-# set_sql_data_field_value_from_record
+# set_sling_sql_data_field_value_from_record
 # This function currently assumes that all queries are going to produce simple results
 # which do not cause the quotation of value fields. Output from sling is in comma separated
 # value format, so any value which contains a comma, or which contains a quotation mark or
@@ -152,7 +152,7 @@ function transfer_table_data_via_sling() {
 # or record counts from tables. If use of this function expands to more general value retrieval
 # it may be necessary to add proper parsing of quoted values. The presence of quotation marks
 # on the line currently cuases a failure to parse.
-function set_sql_data_field_value_from_record() {
+function set_sling_sql_data_field_value_from_record() {
     local record_string=$1
     local column_number=$2
     unset sql_data_field_value
@@ -200,7 +200,7 @@ function set_sql_data_field_value_from_record() {
     sql_data_field_value="$parsed_value"
 }
 
-function set_sql_data_array_from_file() {
+function set_sling_sql_data_array_from_file() {
     local filepath=$1
     local column_number=$2
     unset sql_data_array
@@ -214,7 +214,7 @@ function set_sql_data_array_from_file() {
         if [ "$headers_have_been_parsed" -eq 0 ] ; then
             headers_have_been_parsed=1
         else
-            set_sql_data_field_value_from_record "$line" "$column_number"
+            set_sling_sql_data_field_value_from_record "$line" "$column_number"
             sql_data_array+=("$sql_data_field_value")
         fi
     done < "$filepath"
@@ -227,7 +227,7 @@ function clickhouse_database_exists() {
         echo "Warning : unable to determine if database $database_name exists using : $statement" >&2
         return 1
     fi
-    set_sql_data_array_from_file "$sling_database_exists_filepath" 0
+    set_sling_sql_data_array_from_file "$sling_database_exists_filepath" 0
     if [[ "${sql_data_array[0]}" -ne 1 ]] ; then
         echo "Warning : database $database_name not present on database server, or there are multiple listings for that name" >&2
         return 2
@@ -243,7 +243,7 @@ function clickhouse_database_is_empty() {
         echo "Warning : unable to retrieve table/view list from database $database_name using : $statement" >&2
         return 1
     fi
-    set_sql_data_array_from_file "$sling_database_table_list_filepath" 0
+    set_sling_sql_data_array_from_file "$sling_database_table_list_filepath" 0
     if [[ "${sql_data_array[0]}" -ne 0 ]] ; then
         echo "Warning : database $database_name has tables or views (is not empty as required)" >&2
         return 2