Skip to content

Commit

Permalink
fix sql global variable import (#17)
Browse files Browse the repository at this point in the history
  • Loading branch information
goodwanghan authored Jan 1, 2021
1 parent 81737a3 commit d102ace
Show file tree
Hide file tree
Showing 3 changed files with 13 additions and 4 deletions.
4 changes: 4 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,10 @@ Fugue for Kaggle users

## Release History

### 0.1.6

* Fix sql global variable import

### 0.1.5

* Disable pandas udf (kaggle upgraded to java 11, spark is hard to config)
Expand Down
11 changes: 8 additions & 3 deletions fuggle/_setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
from typing import Any, Dict, List, Optional

import pandas as pd
from fugue import Schema
from fugue import Schema, FugueWorkflow, WorkflowDataFrame
from fugue.extensions._builtins.outputters import Show
from fugue_sql import FugueSQLWorkflow
from IPython.core.magic import register_cell_magic
Expand Down Expand Up @@ -54,8 +54,8 @@ def register_magic(default_engine: Any, conf: Any) -> None:
def fsql(line: Any, cell: Any) -> None: # type: ignore
start = datetime.now()
try:
global_vars = _get_caller_global_vars()
dag = FugueSQLWorkflow()
global_vars = _get_caller_global_vars(dag)
dag(cell, global_vars)
dag.run(engine if line == "" else ENGINE_FACTORY.make_engine(line, conf))
finally:
Expand All @@ -64,6 +64,7 @@ def fsql(line: Any, cell: Any) -> None: # type: ignore


def _get_caller_global_vars(
dag: FugueWorkflow,
global_vars: Optional[Dict[str, Any]] = None,
max_depth: int = 10,
) -> Dict[str, Any]:
Expand All @@ -76,7 +77,11 @@ def _get_caller_global_vars(
global_vars.update(stack.f_globals)
stack = stack.f_back # type: ignore
max_depth -= 1
return global_vars # type: ignore
return {
k: v
for k, v in global_vars.items() # type: ignore
if not isinstance(v, WorkflowDataFrame) or v.workflow is dag
}


def set_print_hook() -> None:
Expand Down
2 changes: 1 addition & 1 deletion fuggle_version/__init__.py
Original file line number Diff line number Diff line change
@@ -1 +1 @@
__version__ = "0.1.5"
__version__ = "0.1.6"

0 comments on commit d102ace

Please sign in to comment.