From 970c971527f15bfdca407edc7f25dcae35d66605 Mon Sep 17 00:00:00 2001 From: dbogunowicz Date: Tue, 9 Apr 2024 11:37:51 +0000 Subject: [PATCH 1/8] initial commit --- .../{ => transformers}/test_clear_ml.py | 17 ----------------- 1 file changed, 17 deletions(-) rename tests/sparseml/{ => transformers}/test_clear_ml.py (78%) diff --git a/tests/sparseml/test_clear_ml.py b/tests/sparseml/transformers/test_clear_ml.py similarity index 78% rename from tests/sparseml/test_clear_ml.py rename to tests/sparseml/transformers/test_clear_ml.py index 987d15a15fe..24ec38e2b51 100644 --- a/tests/sparseml/test_clear_ml.py +++ b/tests/sparseml/transformers/test_clear_ml.py @@ -16,27 +16,11 @@ from clearml import Task from sparseml.transformers import apply -from sparseml.utils import is_package_available - - -is_torch_available = is_package_available("torch") -if is_torch_available: - import torch - - torch_err = None -else: - torch = object - torch_err = ModuleNotFoundError( - "`torch` is not installed, use `pip install torch` to log to Weights and Biases" - ) def test_oneshot_and_finetune(tmp_path: Path): recipe_str = "tests/sparseml/transformers/finetune/test_alternate_recipe.yaml" model = "Xenova/llama2.c-stories15M" - device = "cuda:0" - if is_torch_available and not torch.cuda.is_available(): - device = "cpu" dataset = "wikitext" dataset_config_name = "wikitext-2-raw-v1" concatenate_data = True @@ -59,5 +43,4 @@ def test_oneshot_and_finetune(tmp_path: Path): max_steps=max_steps, concatenate_data=concatenate_data, splits=splits, - oneshot_device=device, ) From 7e85d5fd4d42e6d7ed7323b92ead574fdb764d9d Mon Sep 17 00:00:00 2001 From: George Ohashi Date: Mon, 15 Apr 2024 14:49:14 +0000 Subject: [PATCH 2/8] add secrets --- .github/workflows/test-check.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/test-check.yaml b/.github/workflows/test-check.yaml index 65bcb815467..578763be211 100644 --- a/.github/workflows/test-check.yaml +++ b/.github/workflows/test-check.yaml @@ -215,6 +215,11 @@ jobs: runs-on: ubuntu-22.04 env: SPARSEZOO_TEST_MODE: "true" + CLEARML_WEB_HOST: ${{ secrets.CLEARML_WEB_HOST }} + CLEARML_API_HOST: ${{ secrets.CLEARML_API_HOST }} + CLEARML_API_ACCESS_KEY: ${{ secrets.CLEARML_API_ACCESS_KEY }} + CLEARML_FILES_HOST: ${{ secrets.CLEARML_FILES_HOST }} + CLEARML_API_SECRET_KEY: ${{ secrets.CLEARML_API_SECRET_KEY }} needs: test-setup if: ${{needs.test-setup.outputs.transformers == 1}} steps: From 7940ac029d187719d914b09d9db91143c2fef9e2 Mon Sep 17 00:00:00 2001 From: George Ohashi Date: Mon, 15 Apr 2024 16:51:24 +0000 Subject: [PATCH 3/8] add secrets to tests that use clearml --- .github/workflows/test-check.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/test-check.yaml b/.github/workflows/test-check.yaml index 578763be211..400cc751a23 100644 --- a/.github/workflows/test-check.yaml +++ b/.github/workflows/test-check.yaml @@ -143,6 +143,11 @@ jobs: runs-on: ubuntu-22.04 env: SPARSEZOO_TEST_MODE: "true" + CLEARML_WEB_HOST: ${{ secrets.CLEARML_WEB_HOST }} + CLEARML_API_HOST: ${{ secrets.CLEARML_API_HOST }} + CLEARML_API_ACCESS_KEY: ${{ secrets.CLEARML_API_ACCESS_KEY }} + CLEARML_FILES_HOST: ${{ secrets.CLEARML_FILES_HOST }} + CLEARML_API_SECRET_KEY: ${{ secrets.CLEARML_API_SECRET_KEY }} needs: test-setup if: ${{needs.test-setup.outputs.pytorch == 1}} steps: @@ -167,6 +172,11 @@ jobs: runs-on: ubuntu-22.04 env: SPARSEZOO_TEST_MODE: "true" + CLEARML_WEB_HOST: ${{ secrets.CLEARML_WEB_HOST }} + CLEARML_API_HOST: ${{ secrets.CLEARML_API_HOST }} + CLEARML_API_ACCESS_KEY: ${{ secrets.CLEARML_API_ACCESS_KEY }} + CLEARML_FILES_HOST: ${{ secrets.CLEARML_FILES_HOST }} + CLEARML_API_SECRET_KEY: ${{ secrets.CLEARML_API_SECRET_KEY }} needs: test-setup if: ${{needs.test-setup.outputs.pytorch == 1}} steps: From d34701996eb6f8c215de1260f8de2ed072cc806e Mon Sep 17 00:00:00 2001 From: George Ohashi Date: Tue, 16 Apr 2024 14:35:40 +0000 Subject: [PATCH 4/8] run tests on whitelisted machine --- .github/workflows/test-check.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-check.yaml b/.github/workflows/test-check.yaml index 400cc751a23..9f668546d99 100644 --- a/.github/workflows/test-check.yaml +++ b/.github/workflows/test-check.yaml @@ -140,7 +140,7 @@ jobs: - name: "🔬 Running onnx tests" run: make test TARGETS=onnx pytorch-tests: - runs-on: ubuntu-22.04 + runs-on: aws-avx2-64G env: SPARSEZOO_TEST_MODE: "true" CLEARML_WEB_HOST: ${{ secrets.CLEARML_WEB_HOST }} @@ -169,7 +169,7 @@ jobs: - name: "🔬 Running pytorch tests" run: make test TARGETS=pytorch compat-pytorch-1_9-pytorch-tests: - runs-on: ubuntu-22.04 + runs-on: aws-avx2-64G env: SPARSEZOO_TEST_MODE: "true" CLEARML_WEB_HOST: ${{ secrets.CLEARML_WEB_HOST }} @@ -222,7 +222,7 @@ jobs: - name: "🔬 Running onnx tests" run: make test TARGETS=onnx transformers-tests: - runs-on: ubuntu-22.04 + runs-on: aws-avx2-64G env: SPARSEZOO_TEST_MODE: "true" CLEARML_WEB_HOST: ${{ secrets.CLEARML_WEB_HOST }} From 2fb569392a65458ef64a3fea5a1328d6a788fd60 Mon Sep 17 00:00:00 2001 From: George Ohashi Date: Tue, 16 Apr 2024 17:51:39 +0000 Subject: [PATCH 5/8] get rid of task --- tests/sparseml/transformers/test_clear_ml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/sparseml/transformers/test_clear_ml.py b/tests/sparseml/transformers/test_clear_ml.py index 24ec38e2b51..7cc33bab500 100644 --- a/tests/sparseml/transformers/test_clear_ml.py +++ b/tests/sparseml/transformers/test_clear_ml.py @@ -31,7 +31,7 @@ def test_oneshot_and_finetune(tmp_path: Path): # clearML will automatically log default capturing entries without # explicitly calling logger. Logs accessible in https://app.clear.ml/ - Task.init(project_name="test", task_name="test_oneshot_and_finetune") + # Task.init(project_name="test", task_name="test_oneshot_and_finetune") apply( model=model, From 90349635af846727501a2241cc753d7e5c13f56a Mon Sep 17 00:00:00 2001 From: George Ohashi Date: Wed, 17 Apr 2024 12:20:19 +0000 Subject: [PATCH 6/8] re add Task --- tests/sparseml/transformers/test_clear_ml.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/sparseml/transformers/test_clear_ml.py b/tests/sparseml/transformers/test_clear_ml.py index 7cc33bab500..1751e541052 100644 --- a/tests/sparseml/transformers/test_clear_ml.py +++ b/tests/sparseml/transformers/test_clear_ml.py @@ -14,6 +14,8 @@ from pathlib import Path +import torch + from clearml import Task from sparseml.transformers import apply @@ -21,6 +23,9 @@ def test_oneshot_and_finetune(tmp_path: Path): recipe_str = "tests/sparseml/transformers/finetune/test_alternate_recipe.yaml" model = "Xenova/llama2.c-stories15M" + device = "cuda:0" + if not torch.cuda.is_available(): + device = "cpu" dataset = "wikitext" dataset_config_name = "wikitext-2-raw-v1" concatenate_data = True @@ -31,7 +36,7 @@ def test_oneshot_and_finetune(tmp_path: Path): # clearML will automatically log default capturing entries without # explicitly calling logger. Logs accessible in https://app.clear.ml/ - # Task.init(project_name="test", task_name="test_oneshot_and_finetune") + Task.init(project_name="test", task_name="test_oneshot_and_finetune") apply( model=model, @@ -43,4 +48,5 @@ def test_oneshot_and_finetune(tmp_path: Path): max_steps=max_steps, concatenate_data=concatenate_data, splits=splits, + oneshot_device=device, ) From b10f1daa6564801e126ee97832216e028e47cca1 Mon Sep 17 00:00:00 2001 From: George Ohashi Date: Wed, 17 Apr 2024 12:39:33 +0000 Subject: [PATCH 7/8] use finetune wout recipe --- tests/sparseml/transformers/test_clear_ml.py | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/tests/sparseml/transformers/test_clear_ml.py b/tests/sparseml/transformers/test_clear_ml.py index 1751e541052..31ff945b495 100644 --- a/tests/sparseml/transformers/test_clear_ml.py +++ b/tests/sparseml/transformers/test_clear_ml.py @@ -17,32 +17,26 @@ import torch from clearml import Task -from sparseml.transformers import apply +from sparseml.transformers import train -def test_oneshot_and_finetune(tmp_path: Path): - recipe_str = "tests/sparseml/transformers/finetune/test_alternate_recipe.yaml" +def test_finetune_wout_recipe(tmp_path: Path): + recipe_str = None model = "Xenova/llama2.c-stories15M" device = "cuda:0" if not torch.cuda.is_available(): device = "cpu" - dataset = "wikitext" - dataset_config_name = "wikitext-2-raw-v1" - concatenate_data = True - run_stages = True + dataset = "open_platypus" + concatenate_data = False output_dir = tmp_path max_steps = 50 - splits = {"train": "train[:50%]", "calibration": "train[50%:60%]"} + splits = "train" - # clearML will automatically log default capturing entries without - # explicitly calling logger. Logs accessible in https://app.clear.ml/ Task.init(project_name="test", task_name="test_oneshot_and_finetune") - apply( + train( model=model, dataset=dataset, - dataset_config_name=dataset_config_name, - run_stages=run_stages, output_dir=output_dir, recipe=recipe_str, max_steps=max_steps, From dd68e1acd8e5dd5693212654f1bc29eeb888e61d Mon Sep 17 00:00:00 2001 From: dbogunowicz Date: Wed, 17 Apr 2024 13:44:24 +0000 Subject: [PATCH 8/8] lets make it fly --- tests/sparseml/transformers/finetune/test_finetune.py | 2 +- tests/sparseml/transformers/finetune/test_finetune_helpers.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/sparseml/transformers/finetune/test_finetune.py b/tests/sparseml/transformers/finetune/test_finetune.py index b2a437751da..823ade908a4 100644 --- a/tests/sparseml/transformers/finetune/test_finetune.py +++ b/tests/sparseml/transformers/finetune/test_finetune.py @@ -141,7 +141,7 @@ def test_oneshot_then_finetune(tmp_path: Path): ) -def test_finetune_wout_recipe(tmp_path: Path): +def test_finetune_without_recipe(tmp_path: Path): recipe_str = None model = "Xenova/llama2.c-stories15M" device = "cuda:0" diff --git a/tests/sparseml/transformers/finetune/test_finetune_helpers.py b/tests/sparseml/transformers/finetune/test_finetune_helpers.py index 262788c7af9..3fde66276d9 100644 --- a/tests/sparseml/transformers/finetune/test_finetune_helpers.py +++ b/tests/sparseml/transformers/finetune/test_finetune_helpers.py @@ -26,7 +26,7 @@ def test_apply_recipe_structure(): model = AutoModelForCausalLM.from_pretrained(model_path) assert not qat_active(model) - recipe_with_quant = "tests/sparseml/transformers/obcq/test_tiny.yaml" + recipe_with_quant = "tests/sparseml/transformers/obcq/quant_and_sparse.yaml" apply_recipe_structure_to_model(model, recipe_with_quant, model_path) assert qat_active(model)