fix: closing databricks connection after fetching the data in duc (#15)

ASML-Labs · Jan 22, 2025 · 71a42e0 · 71a42e0
1 parent 76594ca
commit 71a42e0
Show file tree

Hide file tree

Showing 5 changed files with 9 additions and 5 deletions.
diff --git a/libraries/dagster-delta-polars/pyproject.toml b/libraries/dagster-delta-polars/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "dagster-delta-polars"
-version = "0.1.4"
+version = "0.1.5"
 description = "Polars deltalake IO Managers for Dagster with optional LakeFS support"
 readme = "README.md"
 requires-python = ">=3.9"

diff --git a/libraries/dagster-delta/pyproject.toml b/libraries/dagster-delta/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "dagster-delta"
-version = "0.1.4"
+version = "0.1.5"
 description = "base deltalake IO Managers for Dagster"
 readme = "README.md"
 requires-python = ">=3.9"

diff --git a/libraries/dagster-unity-catalog-polars/dagster_unity_catalog_polars/utils.py b/libraries/dagster-unity-catalog-polars/dagster_unity_catalog_polars/utils.py
@@ -46,4 +46,8 @@ def read_uc_table(
     )
     cursor.execute(query)
 
-    return pl.DataFrame(cursor.fetchall_arrow())
+    df = pl.DataFrame(cursor.fetchall_arrow())
+
+    conn.close()
+
+    return df
diff --git a/libraries/dagster-unity-catalog-polars/pyproject.toml b/libraries/dagster-unity-catalog-polars/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "dagster-unity-catalog-polars"
-version = "0.1.4"
+version = "0.1.5"
 description = "Polars Unity Catalog IO Managers for Dagster"
 readme = "README.md"
 requires-python = ">=3.9"

diff --git a/requirements.txt b/requirements.txt
@@ -3,7 +3,7 @@ polars
 dagster
 lakefs
 pydantic>=2
-pyarrow
+pyarrow==18.0.0
 databricks-sdk
 databricks-sql-connector
 deltalake