Skip to content

Commit

Permalink
fix failing CI test
Browse files Browse the repository at this point in the history
  • Loading branch information
dhruviyer committed Dec 26, 2024
1 parent 69955bf commit 324c81a
Showing 1 changed file with 7 additions and 5 deletions.
12 changes: 7 additions & 5 deletions lotus/sem_ops/sem_topk.py
Original file line number Diff line number Diff line change
Expand Up @@ -433,16 +433,18 @@ def __call__(
(group, user_instruction, K, method, strategy, None, cascade_threshold, return_stats)
for _, group in grouped
]

from concurrent.futures import ThreadPoolExecutor

with ThreadPoolExecutor() as executor:
with ThreadPoolExecutor(max_workers=lotus.settings.parallel_groupby_max_threads) as executor:
results = list(executor.map(SemTopKDataframe.process_group, group_args))

new_df = pd.concat([res[0] for res in results])
stats = {name: res[1] for name, res in zip(grouped.groups.keys(), results)}

if return_stats:
new_df = pd.concat([res[0] for res in results])
stats = {name: res[1] for name, res in zip(grouped.groups.keys(), results)}
return new_df, stats
return new_df
else:
return pd.concat(results)

if method == "quick-sem":
assert len(col_li) == 1, "Only one column can be used for embedding optimization"
Expand Down

0 comments on commit 324c81a

Please sign in to comment.