update sem_agg with new max_workers setting

guestrin-lab · Dec 26, 2024 · 84141b8 · 84141b8
1 parent 4b0ec96
commit 84141b8
Show file tree

Hide file tree

Showing 2 changed files with 1 addition and 5 deletions.
diff --git a/examples/op_examples/agg_with_grouping.py b/examples/op_examples/agg_with_grouping.py
@@ -8,10 +8,6 @@
 lm = LM(model="gpt-4o-mini")
 
 lotus.settings.configure(lm=lm)
-lotus.settings.configure(enable_multithreading=True)
-
-# turn on lotus debug logging
-lotus.logger.setLevel("DEBUG")
 
 data = {
     "Course Name": [

diff --git a/lotus/sem_ops/sem_agg.py b/lotus/sem_ops/sem_agg.py
@@ -191,7 +191,7 @@ def __call__(
             group_args = [(group, user_instruction, all_cols, suffix, progress_bar_desc) for _, group in grouped]
             from concurrent.futures import ThreadPoolExecutor
 
-            with ThreadPoolExecutor() as executor:
+            with ThreadPoolExecutor(max_workers=lotus.settings.parallel_groupby_max_threads) as executor:
                 return pd.concat(list(executor.map(SemAggDataframe.process_group, group_args)))
 
         # Sort df by partition_id if it exists