Revert "add the gate function of softmoe"

This reverts commit c71a7f9.
pjlab-sys4nlp · Oct 7, 2023 · f8a56b4 · f8a56b4
1 parent c71a7f9
commit f8a56b4
Show file tree

Hide file tree

Showing 14 changed files with 50,011 additions and 755 deletions.
diff --git a/models/124M/encoder.json b/models/124M/encoder.json
diff --git a/models/124M/vocab.bpe b/models/124M/vocab.bpe
diff --git a/requirements.txt b/requirements.txt
@@ -4,7 +4,6 @@ coverage==7.2.7
 datasets==2.14.1
 debugpy==1.6.7
 deepspeed==0.10.0
-einops==0.6.1
 flake8==6.0.0
 huggingface-hub==0.16.4
 isort==5.12.0

diff --git a/scripts/examples/create_soft_llama_moe.sh b/scripts/examples/create_soft_llama_moe.sh
diff --git a/scripts/examples/create_switch_llama_moe.sh b/scripts/examples/create_switch_llama_moe.sh
@@ -8,7 +8,7 @@ model_type=LlamaMoEForCausalLM #  LlamaMoEModel  LlamaMoEForCausalLM  LlamaMoEFo
 
 tokenizer_path=/mnt/petrelfs/share_data/quxiaoye/models/${base_model}/
 
-gpus=1
+gpus=0
 cpus=16
 OMP_NUM_THREADS=8 srun --partition=MoE --job-name=test --mpi=pmi2 --gres=gpu:${gpus} -n1 --ntasks-per-node=1 -c ${cpus} --job-name=example --kill-on-bad-exit=1 \
   python -m smoe.entrypoint.examples.create_switch_llama_moe \

diff --git a/scripts/moefication/run_select.py b/scripts/moefication/run_select.py
diff --git a/smoe/entrypoint/examples/create_soft_llama_moe.py b/smoe/entrypoint/examples/create_soft_llama_moe.py
-Original file line number
+Diff line change
@@ Expand Up @@
     tokenizer_path=/mnt/petrelfs/share_data/quxiaoye/models/${base_model}/
-    gpus=1
+    gpus=0
     cpus=16
     OMP_NUM_THREADS=8 srun --partition=MoE --job-name=test --mpi=pmi2 --gres=gpu:${gpus} -n1 --ntasks-per-node=1 -c ${cpus} --job-name=example --kill-on-bad-exit=1 \
       python -m smoe.entrypoint.examples.create_switch_llama_moe \
@@ Expand Down @@