From 4a61593118ea0aa358883440c195e4d5af44f281 Mon Sep 17 00:00:00 2001 From: Bibo Hao Date: Wed, 15 May 2024 20:09:38 +0800 Subject: [PATCH 1/3] feat - duckdb --- src/aloha/db/duckdb.py | 44 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 src/aloha/db/duckdb.py diff --git a/src/aloha/db/duckdb.py b/src/aloha/db/duckdb.py new file mode 100644 index 0000000..98b001c --- /dev/null +++ b/src/aloha/db/duckdb.py @@ -0,0 +1,44 @@ +__all__ = ('DuckOperator',) + +import duckdb +from sqlalchemy import create_engine +from sqlalchemy.sql import text + +from ..logger import LOG + +LOG.debug('duckdb: duckdb version = %s' % duckdb.__version__) + + +class DuckOperator: + def __init__(self, db_config, **kwargs): + self._config = { + 'path': db_config['path'], + 'dbname': db_config['dbname'], + } + connect_args = {} + if 'schema' in db_config: + connect_args['options'] = '-csearch_path={}'.format(db_config['schema']) + + try: + self.engine = create_engine( + 'postgresql+psycopg2://{user}:{password}@{host}:{port}/{dbname}'.format(**self._config), + connect_args=connect_args, client_encoding='utf8', encoding='utf-8', + pool_size=20, max_overflow=10, pool_pre_ping=True, **kwargs + ) + LOG.debug("PostgresSQL connected: {host}:{port}/{dbname}".format(**self._config)) + except Exception as e: + LOG.error(e) + raise RuntimeError('Failed to connect to PostgresSQL') + + @property + def connection(self): + return self.engine + + def execute_query(self, sql, *args, **kwargs): + with self.engine.connect() as conn: + cur = conn.execute(text(sql), *args, **kwargs) + return cur + + @property + def connection_str(self) -> str: + return "duckdb:///{user}:{password}@{host}:{port}/{dbname}".format(**self._config) From e1a3b4f137bd74b738d2f53cc5f7fb89b9df6a42 Mon Sep 17 00:00:00 2001 From: Bibo Hao Date: Sat, 2 Nov 2024 19:28:46 +0000 Subject: [PATCH 2/3] fix subfolders of exclude folder --- src/aloha/script/compile.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/aloha/script/compile.py b/src/aloha/script/compile.py index 2438c6e..552a064 100644 --- a/src/aloha/script/compile.py +++ b/src/aloha/script/compile.py @@ -49,11 +49,19 @@ def build(base: str = None, dist: str = 'build', exclude: list = None, keep: lis for dir_path, dir_names, file_names in os.walk(path_base): dir_name = dir_path.split(os.sep)[-1] # name of the current directory - if dir_path.startswith(path_build) or dir_path in files_exclude: - continue # skip: folder for build output, and excluded folders - + flag_skip: bool = False if dir_name.startswith('.') or (os.sep + '.' in dir_path): - continue # hidden folders and sub-folders + flag_skip = True # hidden folders and sub-folders + elif dir_path.startswith(path_build): + flag_skip = True # skip: folder for build output, and excluded folders + else: + for f in files_exclude: + if dir_path.startswith(f): + flag_skip = True + break + + if flag_skip: + continue for file in file_names: (name, extension), path = os.path.splitext(file), os.path.join(dir_path, file) From 50726ca50216dd5094628bed0176c36c15e01cee Mon Sep 17 00:00:00 2001 From: Bibo Hao Date: Sun, 3 Nov 2024 04:01:24 +0800 Subject: [PATCH 3/3] Update postgres.py Signed-off-by: Bibo Hao --- src/aloha/db/postgres.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aloha/db/postgres.py b/src/aloha/db/postgres.py index a0e8a81..e7394ca 100644 --- a/src/aloha/db/postgres.py +++ b/src/aloha/db/postgres.py @@ -7,7 +7,7 @@ from .base import PasswordVault from ..logger import LOG -LOG.debug('postgres: psycopg2 version = %s' % psycopg2.__version__) +LOG.debug('postgres: psycopg version = %s' % psycopg2.__version__) class PostgresOperator: