From 56cf470d79dafae8ed2ca62c4b964da2062a112f Mon Sep 17 00:00:00 2001 From: Han Wang Date: Tue, 25 Jan 2022 06:47:01 +0000 Subject: [PATCH 1/2] update --- README.md | 4 ++++ fuggle_version/__init__.py | 2 +- setup.py | 6 +++--- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 32fd30c..605a4dd 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,10 @@ Fugue for Kaggle users ## Release History +### 0.3.4 + +* Use Fugue 0.6.5 + ### 0.3.3 * Use Fugue 0.6.3 diff --git a/fuggle_version/__init__.py b/fuggle_version/__init__.py index e19434e..334b899 100644 --- a/fuggle_version/__init__.py +++ b/fuggle_version/__init__.py @@ -1 +1 @@ -__version__ = "0.3.3" +__version__ = "0.3.4" diff --git a/setup.py b/setup.py index 4754bbd..931e803 100644 --- a/setup.py +++ b/setup.py @@ -18,13 +18,13 @@ keywords="fugue kaggle sql spark dask pandas", url="http://github.com/fugue-project/fuggle", install_requires=[ - "fugue[spark,dask,sql]==0.6.3", - "tune[all]==0.0.9.dev1", + "fugue[spark,dask,sql,duckdb]==0.6.5", + "tune[all]==0.0.9", "notebook", "kaggle", "seaborn", "qpd", - "dask[dataframe]", + "dask[dataframe,distributed]", "pandavro", "psutil", ], From 84cbe05e7d7ece362f52fe28a5abbe9f958fff9e Mon Sep 17 00:00:00 2001 From: Han Wang Date: Tue, 25 Jan 2022 07:48:58 +0000 Subject: [PATCH 2/2] update --- fuggle/execution_engine.py | 13 +++++++++++-- tests/test_execution_engine.py | 14 ++++++-------- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/fuggle/execution_engine.py b/fuggle/execution_engine.py index cbb485b..abbc144 100644 --- a/fuggle/execution_engine.py +++ b/fuggle/execution_engine.py @@ -15,7 +15,10 @@ register_default_execution_engine, register_execution_engine, ) -from fugue.constants import FUGUE_CONF_WORKFLOW_CHECKPOINT_PATH +from fugue.constants import ( + FUGUE_CONF_SQL_IGNORE_CASE, + FUGUE_CONF_WORKFLOW_CHECKPOINT_PATH, +) from fugue_dask._constants import FUGUE_DASK_CONF_DATAFRAME_DEFAULT_PARTITIONS from fugue_dask.execution_engine import DaskExecutionEngine, QPDDaskEngine from fugue_notebook import NotebookSetup @@ -44,7 +47,13 @@ def __init__(self, execution_engine: ExecutionEngine): def select(self, dfs: DataFrames, statement: str) -> DataFrame: pd_dfs = {k: self.execution_engine.to_df(v).as_pandas() for k, v in dfs.items()} - df = run_sql_on_pandas(statement, pd_dfs) + df = run_sql_on_pandas( + statement, + pd_dfs, + ignore_case=self.execution_engine.compile_conf.get( + FUGUE_CONF_SQL_IGNORE_CASE, False + ), + ) return PandasDataFrame(df) diff --git a/tests/test_execution_engine.py b/tests/test_execution_engine.py index 925d0ea..171fbc0 100644 --- a/tests/test_execution_engine.py +++ b/tests/test_execution_engine.py @@ -32,8 +32,8 @@ def make_engine(self): ) return e - def dag(self) -> FugueSQLWorkflow: - return FugueSQLWorkflow(self.engine) + def dag(self, conf=None) -> FugueSQLWorkflow: + return FugueSQLWorkflow(self.engine, conf) def test_sqlite(self): with self.dag() as dag: @@ -65,8 +65,6 @@ def test__join_outer_pandas_incompatible(self): def test_sample_n(self): pass - - class KaggleSparkExecutionEngineBuiltInTests(BuiltInTests.Tests): @pytest.fixture(autouse=True) @@ -82,8 +80,8 @@ def make_engine(self): ) return e - def dag(self) -> FugueSQLWorkflow: - return FugueSQLWorkflow(self.engine) + def dag(self, conf=None) -> FugueSQLWorkflow: + return FugueSQLWorkflow(self.engine, conf) def test_callback(self): pass @@ -132,8 +130,8 @@ def make_engine(self): ) return e - def dag(self) -> FugueSQLWorkflow: - return FugueSQLWorkflow(self.engine) + def dag(self, conf=None) -> FugueSQLWorkflow: + return FugueSQLWorkflow(self.engine, conf) def test_sqlite(self): with self.dag() as dag: