diff --git a/data_compare_pack/main.py b/data_compare_pack/main.py index 0659cea..854a22b 100644 --- a/data_compare_pack/main.py +++ b/data_compare_pack/main.py @@ -37,7 +37,7 @@ # Combine compare_col_list and id_columns while removing duplicates combined_columns_list = list(dict.fromkeys(compare_col_list + id_columns)) -if len(id_columns) == 0 : +if len(id_columns) == 0: id_columns = compare_col_list # Creating subsets for source and target data with no repeated columns @@ -229,19 +229,38 @@ ] ) - # Extracting column labels columnLabels = df_all_mismatch.columns.tolist() -# Converting the DataFrame into the desired format without row labels +# Dictionary to map the old suffix to the new one +suffix_mapping = {"_df1": "_source", "_df2": "_target"} + +# Revise the loop to correctly process replacement without duplication +new_columnLabels = [ + ( + col + if not any(col.endswith(suffix) for suffix in suffix_mapping.keys()) + else next( + col.replace(suffix, replacement) + for suffix, replacement in suffix_mapping.items() + if col.endswith(suffix) + ) + ) + for col in columnLabels +] + +# Assuming `df_all_mismatch` is your DataFrame, rename its columns with the new labels +df_all_mismatch.columns = new_columnLabels + +# Since you've updated column names, you don't need to change the way you convert the DataFrame data_formatted = [ [{"value": row[col]} for col in df_all_mismatch.columns] for index, row in df_all_mismatch.iterrows() ] -# The formatted data structure, now without rowLabels +# The formatted data structure, now with renamed labels format_structure = { - "columnLabels": columnLabels, + "columnLabels": new_columnLabels, # Use the new column labels "data": data_formatted, } @@ -250,7 +269,7 @@ [ { "key": "recommendation_levels_mismatches", - "value": {"info": "<=0.5", "warning": ">0.5", "high": ">0.8"}, + "value": {"info": "0", "warning": "0.5", "high": "0.8"}, "scope": {"perimeter": "dataset", "value": pack.source_config["name"]}, }, { diff --git a/data_compare_pack/pack_conf.json b/data_compare_pack/pack_conf.json index 6637385..02bb045 100644 --- a/data_compare_pack/pack_conf.json +++ b/data_compare_pack/pack_conf.json @@ -1,36 +1,40 @@ { - "job": { - "compare_col_list": [], - "id_columns": [], - "abs_tol": 0.0001, - "rel_tol": 0, - "source": { - "skiprows": 0 - } - }, - "charts": { - "overview": [ - { - "metric_key": "score", - "chart_type": "text", - "display_title": true, - "justify": true - }, - { - "metric_key": "recommendation_levels_mismatches", - "chart_type": "recommendation_level_indicator", - "display_title": true - }, - { - "metric_key": "check_column", - "chart_type": "check_table", - "display_title": true - }, - { - "metric_key": "mismatches_table", - "chart_type": "table", - "display_title": true - } - ] + "job": { + "compare_col_list": [], + "id_columns": [], + "abs_tol": 0.0001, + "rel_tol": 0, + "source": { + "skiprows": 0 } -} \ No newline at end of file + }, + "charts": { + "overview": [ + { + "metric_key": "score", + "chart_type": "text", + "display_title": true, + "justify": true + }, + { + "metric_key": "recommendation_levels_mismatches", + "chart_type": "recommendation_level_indicator", + "tooltip": { + "title": "Recommendation level's importance mapping", + "content": "Gives the recommendation level for proportions of mismatches" + }, + "display_title": true + }, + { + "metric_key": "check_column", + "chart_type": "check_table", + "display_title": true + }, + { + "metric_key": "mismatches_table", + "chart_type": "table", + "display_title": true + } + ] + } +} diff --git a/data_compare_pack/properties.yaml b/data_compare_pack/properties.yaml index fbe02f5..eccd6fc 100644 --- a/data_compare_pack/properties.yaml +++ b/data_compare_pack/properties.yaml @@ -3,5 +3,5 @@ icon: icon.png name: data_compare type: consistency url: https://github.com/qalita-io/packs/tree/main/data_compare_pack -version: 2.0.12 +version: 2.0.21 visibility: public