Merge pull request #530 from DataRecce/feature/drc-945-bug-unable-to-…

…execute-profile-diff

[Bug] Add extra process in get_columns for databricks

recce/adapter/dbt_adapter/__init__.py

@@ -259,20 +259,41 @@ def print_lineage_info(self):
 
     def get_columns(self, model: str, base=False) -> List[Column]:
         relation = self.create_relation(model, base)
+        get_columns_macro = 'get_columns_in_relation'
+        if self.adapter.connections.TYPE == 'databricks':
+            get_columns_macro = 'get_columns_comments'
+
         if dbt_version < 'v1.8':
-            return self.adapter.execute_macro(
-                'get_columns_in_relation',
+            columns = self.adapter.execute_macro(
+                get_columns_macro,
                 kwargs={"relation": relation},
                 manifest=self.manifest)
         else:
             from dbt.context.providers import generate_runtime_macro_context
             macro_manifest = MacroManifest(self.manifest.macros)
             self.adapter.set_macro_resolver(macro_manifest)
             self.adapter.set_macro_context_generator(generate_runtime_macro_context)
-            return self.adapter.execute_macro(
-                'get_columns_in_relation',
+            columns = self.adapter.execute_macro(
+                get_columns_macro,
                 kwargs={"relation": relation})
 
+        if self.adapter.connections.TYPE == 'databricks':
+            # reference: get_columns_in_relation (dbt/adapters/databricks/impl.py)
+            from dbt.adapters.databricks import DatabricksColumn
+            rows = columns
+            columns = []
+            for row in rows:
+                if row["col_name"].startswith("#"):
+                    break
+                columns.append(
+                    DatabricksColumn(
+                        column=row["col_name"], dtype=row["data_type"], comment=row["comment"]
+                    )
+                )
+            return columns
+        else:
+            return columns
+
     def get_model(self, model_id: str, base=False):
         manifest = self.curr_manifest if base is False else self.base_manifest
         manifest_dict = manifest.to_dict()