rename lm_eval => bigcode_eval

bigcode-project · Oct 24, 2023 · 92c81a0 · 92c81a0
1 parent a8fb63d
commit 92c81a0
Show file tree

Hide file tree

Showing 73 changed files with 35 additions and 35 deletions.
diff --git a/lm_eval/__init__.py → bigcode_eval/__init__.py b/lm_eval/__init__.py → bigcode_eval/__init__.py
diff --git a/lm_eval/arguments.py → bigcode_eval/arguments.py b/lm_eval/arguments.py → bigcode_eval/arguments.py
diff --git a/lm_eval/base.py → bigcode_eval/base.py b/lm_eval/base.py → bigcode_eval/base.py
diff --git a/lm_eval/evaluator.py → bigcode_eval/evaluator.py b/lm_eval/evaluator.py → bigcode_eval/evaluator.py
@@ -3,8 +3,8 @@
 import os
 import warnings
 
-from lm_eval import tasks
-from lm_eval.generation import parallel_generations
+from bigcode_eval import tasks
+from bigcode_eval.generation import parallel_generations
 
 _WARNING = """
 ################################################################################

diff --git a/lm_eval/generation.py → bigcode_eval/generation.py b/lm_eval/generation.py → bigcode_eval/generation.py
@@ -5,7 +5,7 @@
 from torch.utils.data.dataloader import DataLoader
 from transformers import StoppingCriteria, StoppingCriteriaList
 
-from lm_eval.utils import TokenizedDataset, complete_code
+from bigcode_eval.utils import TokenizedDataset, complete_code
 
 
 class EndOfFunctionCriteria(StoppingCriteria):

diff --git a/lm_eval/tasks/__init__.py → bigcode_eval/tasks/__init__.py b/lm_eval/tasks/__init__.py → bigcode_eval/tasks/__init__.py
diff --git a/lm_eval/tasks/apps.py → bigcode_eval/tasks/apps.py b/lm_eval/tasks/apps.py → bigcode_eval/tasks/apps.py
@@ -11,7 +11,7 @@
 
 from evaluate import load
 
-from lm_eval.base import Task
+from bigcode_eval.base import Task
 
 _CITATION = """
 @article{hendrycksapps2021,

diff --git a/lm_eval/tasks/codexglue_code_to_text.py → bigcode_eval/tasks/codexglue_code_to_text.py b/lm_eval/tasks/codexglue_code_to_text.py → bigcode_eval/tasks/codexglue_code_to_text.py
@@ -10,7 +10,7 @@
 import re
 import typing
 
-from lm_eval.base import Task
+from bigcode_eval.base import Task
 
 _CITATION = """
 @article{husain2019codesearchnet,
@@ -63,7 +63,7 @@ def compute_codexglue_code_to_text_bleu(
 
     Taken from: https://github.com/dpfried/lm-evaluation-harness/blob/5d9a6aaaaa929bcad95bb73d85e78fe75eb64b4e/lm_eval/tasks/codexglue_summarization.py#L102
     """
-    from lm_eval.tasks.custom_metrics import codexglue_code_to_text_bleu
+    from bigcode_eval.tasks.custom_metrics import codexglue_code_to_text_bleu
 
     predicted_map = {}
     gold_map = {}

diff --git a/lm_eval/tasks/codexglue_text_to_text.py → bigcode_eval/tasks/codexglue_text_to_text.py b/lm_eval/tasks/codexglue_text_to_text.py → bigcode_eval/tasks/codexglue_text_to_text.py
@@ -11,7 +11,7 @@
 
 from evaluate import load
 
-from lm_eval.base import Task
+from bigcode_eval.base import Task
 
 _CITATION = """
 @article{CodeXGLUE,

diff --git a/lm_eval/tasks/conala.py → bigcode_eval/tasks/conala.py b/lm_eval/tasks/conala.py → bigcode_eval/tasks/conala.py
@@ -13,7 +13,7 @@
 
 from evaluate import load
 
-from lm_eval.base import Task
+from bigcode_eval.base import Task
 
 _CITATION = """
 @inproceedings{yin2018learning,

diff --git a/lm_eval/tasks/concode.py → bigcode_eval/tasks/concode.py b/lm_eval/tasks/concode.py → bigcode_eval/tasks/concode.py
@@ -14,7 +14,7 @@
 
 from evaluate import load
 
-from lm_eval.base import Task
+from bigcode_eval.base import Task
 
 _CITATION = """
 @article{iyer2018mapping,

diff --git a/lm_eval/tasks/custom_metrics/__init__.py → ...ode_eval/tasks/custom_metrics/__init__.py b/lm_eval/tasks/custom_metrics/__init__.py → ...ode_eval/tasks/custom_metrics/__init__.py
diff --git a/...om_metrics/codexglue_code_to_text_bleu.py → ...om_metrics/codexglue_code_to_text_bleu.py b/...om_metrics/codexglue_code_to_text_bleu.py → ...om_metrics/codexglue_code_to_text_bleu.py
diff --git a/lm_eval/tasks/custom_metrics/diff_eval.py → ...de_eval/tasks/custom_metrics/diff_eval.py b/lm_eval/tasks/custom_metrics/diff_eval.py → ...de_eval/tasks/custom_metrics/diff_eval.py
diff --git a/...stom_metrics/multiple_metrics/__init__.py → ...stom_metrics/multiple_metrics/__init__.py b/...stom_metrics/multiple_metrics/__init__.py → ...stom_metrics/multiple_metrics/__init__.py
diff --git a/...cs/multiple_metrics/containerized_eval.py → ...cs/multiple_metrics/containerized_eval.py b/...cs/multiple_metrics/containerized_eval.py → ...cs/multiple_metrics/containerized_eval.py
diff --git a/...stom_metrics/multiple_metrics/eval_cpp.py → ...stom_metrics/multiple_metrics/eval_cpp.py b/...stom_metrics/multiple_metrics/eval_cpp.py → ...stom_metrics/multiple_metrics/eval_cpp.py
diff --git a/...ustom_metrics/multiple_metrics/eval_cs.py → ...ustom_metrics/multiple_metrics/eval_cs.py b/...ustom_metrics/multiple_metrics/eval_cs.py → ...ustom_metrics/multiple_metrics/eval_cs.py
diff --git a/...om_metrics/multiple_metrics/eval_dlang.py → ...om_metrics/multiple_metrics/eval_dlang.py b/...om_metrics/multiple_metrics/eval_dlang.py → ...om_metrics/multiple_metrics/eval_dlang.py
diff --git a/...ustom_metrics/multiple_metrics/eval_go.py → ...ustom_metrics/multiple_metrics/eval_go.py b/...ustom_metrics/multiple_metrics/eval_go.py → ...ustom_metrics/multiple_metrics/eval_go.py
diff --git a/...tom_metrics/multiple_metrics/eval_java.py → ...tom_metrics/multiple_metrics/eval_java.py b/...tom_metrics/multiple_metrics/eval_java.py → ...tom_metrics/multiple_metrics/eval_java.py
diff --git a/...trics/multiple_metrics/eval_javascript.py → ...trics/multiple_metrics/eval_javascript.py b/...trics/multiple_metrics/eval_javascript.py → ...trics/multiple_metrics/eval_javascript.py
diff --git a/...om_metrics/multiple_metrics/eval_julia.py → ...om_metrics/multiple_metrics/eval_julia.py b/...om_metrics/multiple_metrics/eval_julia.py → ...om_metrics/multiple_metrics/eval_julia.py
diff --git a/...stom_metrics/multiple_metrics/eval_lua.py → ...stom_metrics/multiple_metrics/eval_lua.py b/...stom_metrics/multiple_metrics/eval_lua.py → ...stom_metrics/multiple_metrics/eval_lua.py
diff --git a/...stom_metrics/multiple_metrics/eval_php.py → ...stom_metrics/multiple_metrics/eval_php.py b/...stom_metrics/multiple_metrics/eval_php.py → ...stom_metrics/multiple_metrics/eval_php.py
diff --git a/...ustom_metrics/multiple_metrics/eval_pl.py → ...ustom_metrics/multiple_metrics/eval_pl.py b/...ustom_metrics/multiple_metrics/eval_pl.py → ...ustom_metrics/multiple_metrics/eval_pl.py
diff --git a/...m_metrics/multiple_metrics/eval_python.py → ...m_metrics/multiple_metrics/eval_python.py b/...m_metrics/multiple_metrics/eval_python.py → ...m_metrics/multiple_metrics/eval_python.py
diff --git a/...custom_metrics/multiple_metrics/eval_r.py → ...custom_metrics/multiple_metrics/eval_r.py b/...custom_metrics/multiple_metrics/eval_r.py → ...custom_metrics/multiple_metrics/eval_r.py
diff --git a/...m_metrics/multiple_metrics/eval_racket.py → ...m_metrics/multiple_metrics/eval_racket.py b/...m_metrics/multiple_metrics/eval_racket.py → ...m_metrics/multiple_metrics/eval_racket.py
diff --git a/...tom_metrics/multiple_metrics/eval_ruby.py → ...tom_metrics/multiple_metrics/eval_ruby.py b/...tom_metrics/multiple_metrics/eval_ruby.py → ...tom_metrics/multiple_metrics/eval_ruby.py
diff --git a/...tom_metrics/multiple_metrics/eval_rust.py → ...tom_metrics/multiple_metrics/eval_rust.py b/...tom_metrics/multiple_metrics/eval_rust.py → ...tom_metrics/multiple_metrics/eval_rust.py
diff --git a/...om_metrics/multiple_metrics/eval_scala.py → ...om_metrics/multiple_metrics/eval_scala.py b/...om_metrics/multiple_metrics/eval_scala.py → ...om_metrics/multiple_metrics/eval_scala.py
diff --git a/...ustom_metrics/multiple_metrics/eval_sh.py → ...ustom_metrics/multiple_metrics/eval_sh.py b/...ustom_metrics/multiple_metrics/eval_sh.py → ...ustom_metrics/multiple_metrics/eval_sh.py
diff --git a/...om_metrics/multiple_metrics/eval_swift.py → ...om_metrics/multiple_metrics/eval_swift.py b/...om_metrics/multiple_metrics/eval_swift.py → ...om_metrics/multiple_metrics/eval_swift.py
diff --git a/...ustom_metrics/multiple_metrics/eval_ts.py → ...ustom_metrics/multiple_metrics/eval_ts.py b/...ustom_metrics/multiple_metrics/eval_ts.py → ...ustom_metrics/multiple_metrics/eval_ts.py
diff --git a/...om_metrics/multiple_metrics/evaluation.py → ...om_metrics/multiple_metrics/evaluation.py b/...om_metrics/multiple_metrics/evaluation.py → ...om_metrics/multiple_metrics/evaluation.py
diff --git a/..._metrics/multiple_metrics/generic_eval.py → ..._metrics/multiple_metrics/generic_eval.py b/..._metrics/multiple_metrics/generic_eval.py → ..._metrics/multiple_metrics/generic_eval.py
diff --git a/...ustom_metrics/multiple_metrics/libeval.py → ...ustom_metrics/multiple_metrics/libeval.py b/...ustom_metrics/multiple_metrics/libeval.py → ...ustom_metrics/multiple_metrics/libeval.py
diff --git a/...ltiple_metrics/safe_subprocess/.gitignore → ...ltiple_metrics/safe_subprocess/.gitignore b/...ltiple_metrics/safe_subprocess/.gitignore → ...ltiple_metrics/safe_subprocess/.gitignore
diff --git a/...tiple_metrics/safe_subprocess/__init__.py → ...tiple_metrics/safe_subprocess/__init__.py b/...tiple_metrics/safe_subprocess/__init__.py → ...tiple_metrics/safe_subprocess/__init__.py
diff --git a/...bprocess/evil_programs/block_on_inputs.py → ...bprocess/evil_programs/block_on_inputs.py b/...bprocess/evil_programs/block_on_inputs.py → ...bprocess/evil_programs/block_on_inputs.py
diff --git a/...subprocess/evil_programs/close_outputs.py → ...subprocess/evil_programs/close_outputs.py b/...subprocess/evil_programs/close_outputs.py → ...subprocess/evil_programs/close_outputs.py
diff --git a/...afe_subprocess/evil_programs/fork_bomb.py → ...afe_subprocess/evil_programs/fork_bomb.py b/...afe_subprocess/evil_programs/fork_bomb.py → ...afe_subprocess/evil_programs/fork_bomb.py
diff --git a/...afe_subprocess/evil_programs/fork_once.py → ...afe_subprocess/evil_programs/fork_once.py b/...afe_subprocess/evil_programs/fork_once.py → ...afe_subprocess/evil_programs/fork_once.py
diff --git a/...subprocess/evil_programs/sleep_forever.py → ...subprocess/evil_programs/sleep_forever.py b/...subprocess/evil_programs/sleep_forever.py → ...subprocess/evil_programs/sleep_forever.py
diff --git a/...process/evil_programs/unbounded_output.py → ...process/evil_programs/unbounded_output.py b/...process/evil_programs/unbounded_output.py → ...process/evil_programs/unbounded_output.py
diff --git a/...le_metrics/safe_subprocess/module_test.py → ...le_metrics/safe_subprocess/module_test.py b/...le_metrics/safe_subprocess/module_test.py → ...le_metrics/safe_subprocess/module_test.py
diff --git a/...tiple_metrics/single_experiment_pass_k.py → ...tiple_metrics/single_experiment_pass_k.py b/...tiple_metrics/single_experiment_pass_k.py → ...tiple_metrics/single_experiment_pass_k.py
diff --git a/...sks/custom_metrics/pal_metric/__init__.py → ...sks/custom_metrics/pal_metric/__init__.py b/...sks/custom_metrics/pal_metric/__init__.py → ...sks/custom_metrics/pal_metric/__init__.py
diff --git a/...ustom_metrics/pal_metric/pal_code_exec.py → ...ustom_metrics/pal_metric/pal_code_exec.py b/...ustom_metrics/pal_metric/pal_code_exec.py → ...ustom_metrics/pal_metric/pal_code_exec.py
@@ -3,7 +3,7 @@
 from collections import Counter, defaultdict
 from concurrent.futures import ThreadPoolExecutor, as_completed
 
-from lm_eval.tasks.custom_metrics.pal_metric.python_executor import run_program
+from bigcode_eval.tasks.custom_metrics.pal_metric.python_executor import run_program
 
 # adapted from https://github.com/huggingface/evaluate/blob/main/metrics/code_eval/code_eval.py
 

diff --git a/...tom_metrics/pal_metric/python_executor.py → ...tom_metrics/pal_metric/python_executor.py b/...tom_metrics/pal_metric/python_executor.py → ...tom_metrics/pal_metric/python_executor.py
diff --git a/lm_eval/tasks/ds1000.py → bigcode_eval/tasks/ds1000.py b/lm_eval/tasks/ds1000.py → bigcode_eval/tasks/ds1000.py
@@ -19,7 +19,7 @@
 import requests
 import tqdm
 
-from lm_eval.base import Task
+from bigcode_eval.base import Task
 
 _CITATION = """
 @article{Lai2022DS1000,

diff --git a/...exglue_text_to_text_few_shot_prompts.json → ...exglue_text_to_text_few_shot_prompts.json b/...exglue_text_to_text_few_shot_prompts.json → ...exglue_text_to_text_few_shot_prompts.json
diff --git a/...hot_examples/conala_few_shot_prompts.json → ...hot_examples/conala_few_shot_prompts.json b/...hot_examples/conala_few_shot_prompts.json → ...hot_examples/conala_few_shot_prompts.json
diff --git a/...ot_examples/concode_few_shot_prompts.json → ...ot_examples/concode_few_shot_prompts.json b/...ot_examples/concode_few_shot_prompts.json → ...ot_examples/concode_few_shot_prompts.json
diff --git a/...shot_examples/gsm8k_few_shot_prompts.json → ...shot_examples/gsm8k_few_shot_prompts.json b/...shot_examples/gsm8k_few_shot_prompts.json → ...shot_examples/gsm8k_few_shot_prompts.json
diff --git a/lm_eval/tasks/gsm.py → bigcode_eval/tasks/gsm.py b/lm_eval/tasks/gsm.py → bigcode_eval/tasks/gsm.py
@@ -18,8 +18,8 @@
 
 from evaluate import load
 
-from lm_eval.base import Task
-from lm_eval.tasks.custom_metrics.pal_metric.pal_code_exec import compute
+from bigcode_eval.base import Task
+from bigcode_eval.tasks.custom_metrics.pal_metric.pal_code_exec import compute
 
 _CITATION = """
 @article{gao2022pal,

diff --git a/lm_eval/tasks/humaneval.py → bigcode_eval/tasks/humaneval.py b/lm_eval/tasks/humaneval.py → bigcode_eval/tasks/humaneval.py
@@ -12,7 +12,7 @@
 
 from evaluate import load
 
-from lm_eval.base import Task
+from bigcode_eval.base import Task
 
 _CITATION = """
 @misc{chen2021evaluating,

diff --git a/lm_eval/tasks/humanevalpack.py → bigcode_eval/tasks/humanevalpack.py b/lm_eval/tasks/humanevalpack.py → bigcode_eval/tasks/humanevalpack.py
@@ -2,7 +2,7 @@
 import re
 
 from evaluate import load
-from lm_eval.base import Task
+from bigcode_eval.base import Task
 
 _CITATION = """
 @article{muennighoff2023octopack,
@@ -357,7 +357,7 @@ def process_results(self, generations, references):
                         fixed_code = ""
                     gen[i] = fixed_code
         elif self.prompt == "diff-carper":
-            from lm_eval.tasks.custom_metrics.diff_eval import apply_diff
+            from bigcode_eval.tasks.custom_metrics.diff_eval import apply_diff
             ds = self.get_dataset().select(range(len(generations)))
             for gen, doc in zip(generations, ds):
                 prompt_base = self.get_prompt_base(doc)
@@ -522,7 +522,7 @@ def postprocess_generation(self, generation, idx):
             # Only remove final stopwords like <MSG>
             generation = self.remove_last_block(generation[len(prompt):].rstrip())
             generation = prompt + generation
-            from lm_eval.tasks.custom_metrics.diff_eval import split_diff
+            from bigcode_eval.tasks.custom_metrics.diff_eval import split_diff
             # From https://github.com/CarperAI/OpenELM/blob/e6402a0696096011572152334ccbe049f89c332e/src/openelm/benchmarks/benchmark_bugs.py#L93
             end_of_diff = re.compile("\n[^ +-@]+")
             parsed: dict = split_diff(generation)

diff --git a/lm_eval/tasks/humanevalpack_openai.py → bigcode_eval/tasks/humanevalpack_openai.py b/lm_eval/tasks/humanevalpack_openai.py → bigcode_eval/tasks/humanevalpack_openai.py
diff --git a/lm_eval/tasks/instruct_humaneval.py → bigcode_eval/tasks/instruct_humaneval.py b/lm_eval/tasks/instruct_humaneval.py → bigcode_eval/tasks/instruct_humaneval.py
@@ -9,8 +9,8 @@
 """
 
 from evaluate import load
-from lm_eval.base import Task
-from lm_eval.utils import remove_after_return
+from bigcode_eval.base import Task
+from bigcode_eval.utils import remove_after_return
 
 _CITATION = ""
 

diff --git a/lm_eval/tasks/instruct_wizard_humaneval.py → ...e_eval/tasks/instruct_wizard_humaneval.py b/lm_eval/tasks/instruct_wizard_humaneval.py → ...e_eval/tasks/instruct_wizard_humaneval.py
@@ -13,7 +13,7 @@
 
 from evaluate import load
 
-from lm_eval.base import Task
+from bigcode_eval.base import Task
 
 _CITATION = """
 @misc{chen2021evaluating,

diff --git a/lm_eval/tasks/mbpp.py → bigcode_eval/tasks/mbpp.py b/lm_eval/tasks/mbpp.py → bigcode_eval/tasks/mbpp.py
@@ -14,7 +14,7 @@
 
 from evaluate import load
 
-from lm_eval.base import Task
+from bigcode_eval.base import Task
 
 _CITATION = """
 @article{austin2021program,

diff --git a/lm_eval/tasks/multiple.py → bigcode_eval/tasks/multiple.py b/lm_eval/tasks/multiple.py → bigcode_eval/tasks/multiple.py
@@ -19,10 +19,10 @@
 from datasets import load_dataset
 from tqdm import tqdm
 
-from lm_eval.base import Task
-from lm_eval.tasks.custom_metrics.multiple_metrics.evaluation import \
+from bigcode_eval.base import Task
+from bigcode_eval.tasks.custom_metrics.multiple_metrics.evaluation import \
     evaluate_problem
-from lm_eval.tasks.custom_metrics.multiple_metrics.single_experiment_pass_k import \
+from bigcode_eval.tasks.custom_metrics.multiple_metrics.single_experiment_pass_k import \
     for_file
 
 _CITATION = """

diff --git a/lm_eval/tasks/parity.py → bigcode_eval/tasks/parity.py b/lm_eval/tasks/parity.py → bigcode_eval/tasks/parity.py
@@ -4,7 +4,7 @@
 import re
 
 from evaluate import load
-from lm_eval.base import Task
+from bigcode_eval.base import Task
 import tqdm
 
 def mutate_code(

diff --git a/lm_eval/tasks/python_bugs.py → bigcode_eval/tasks/python_bugs.py b/lm_eval/tasks/python_bugs.py → bigcode_eval/tasks/python_bugs.py
@@ -10,7 +10,7 @@
 import re
 
 from evaluate import load
-from lm_eval.base import Task
+from bigcode_eval.base import Task
 import tqdm
 
 _CITATION = """

diff --git a/lm_eval/tasks/quixbugs.py → bigcode_eval/tasks/quixbugs.py b/lm_eval/tasks/quixbugs.py → bigcode_eval/tasks/quixbugs.py
@@ -3,7 +3,7 @@
 import re
 
 from evaluate import load
-from lm_eval.base import Task
+from bigcode_eval.base import Task
 
 _CITATION = """
 @inproceedings{lin2017quixbugs,

diff --git a/lm_eval/tasks/recode.py → bigcode_eval/tasks/recode.py b/lm_eval/tasks/recode.py → bigcode_eval/tasks/recode.py
@@ -6,7 +6,7 @@
 This task allows to run the released perturbed HumanEval benchmark, and compute the robust-pass-at-k metric.
 """
 from collections import defaultdict
-from lm_eval.base import Task
+from bigcode_eval.base import Task
 
 from evaluate import load
 

diff --git a/lm_eval/utils.py → bigcode_eval/utils.py b/lm_eval/utils.py → bigcode_eval/utils.py
diff --git a/main.py b/main.py
@@ -13,9 +13,9 @@
     HfArgumentParser,
 )
 
-from lm_eval.arguments import EvalArguments
-from lm_eval.evaluator import Evaluator
-from lm_eval.tasks import ALL_TASKS
+from bigcode_eval.arguments import EvalArguments
+from bigcode_eval.evaluator import Evaluator
+from bigcode_eval.tasks import ALL_TASKS
 
 
 class MultiChoice:

diff --git a/templates/new_task.py b/templates/new_task.py
@@ -7,7 +7,7 @@
 TODO: Write a Short Description of the task.
 Homepage: TODO: Add the URL to the task's Homepage here.
 """
-from lm_eval.base import Task
+from bigcode_eval.base import Task
 
 # TODO: Add the BibTeX citation for the task.
 _CITATION = """

diff --git a/tests/test_generation_evaluation.py b/tests/test_generation_evaluation.py
@@ -6,8 +6,8 @@
 from accelerate.utils import write_basic_config
 from transformers import AutoModelForCausalLM, AutoTokenizer, set_seed
 
-from lm_eval.arguments import EvalArguments
-from lm_eval.evaluator import Evaluator
+from bigcode_eval.arguments import EvalArguments
+from bigcode_eval.evaluator import Evaluator
 
 # TODO add more tasks
 

diff --git a/tests/test_prompts.py b/tests/test_prompts.py
@@ -1,6 +1,6 @@
 import json
 
-from lm_eval import tasks
+from bigcode_eval import tasks
 
 TASKS = ["pal-gsm8k-greedy"]