From 5913a91133adf1fa19b3e0ed010ff2c5ef2a7b28 Mon Sep 17 00:00:00 2001 From: Jan Cuhel Date: Wed, 27 Dec 2023 02:14:04 +0100 Subject: [PATCH 01/24] Add test structure --- .github/workflows/tests.yaml | 115 ++++++++++++++++++++++ tests/unittests/test_image_generation.py | 17 ++++ tests/unittests/test_prompt_generation.py | 17 ++++ 3 files changed, 149 insertions(+) create mode 100644 .github/workflows/tests.yaml create mode 100644 tests/unittests/test_image_generation.py create mode 100644 tests/unittests/test_prompt_generation.py diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml new file mode 100644 index 0000000..d5fbf84 --- /dev/null +++ b/.github/workflows/tests.yaml @@ -0,0 +1,115 @@ +name: Tests + +on: + pull_request: + branches: [ dev, main ] + paths: + - 'src/datadreamer/**/**.py' + - 'tests/**/**.py' + - .github/workflows/tests.yaml + +jobs: + run_tests: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macOS-latest] + version: ['3.10', '3.11'] + + runs-on: ${{ matrix.os }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ github.head_ref }} + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.version }} + cache: pip + + - name: Install dependencies [Ubuntu] + if: matrix.os == 'ubuntu-latest' + run: | + sudo apt update + sudo apt install -y pandoc + pip install -e .[dev] + + - name: Install dependencies [Windows] + if: matrix.os == 'windows-latest' + run: pip install -e .[dev] + + - name: Install dependencies [macOS] + if: matrix.os == 'macOS-latest' + run: pip install -e .[dev] + + - name: Run tests with coverage [Ubuntu] + if: matrix.os == 'ubuntu-latest' && matrix.version == '3.10' + run: pytest tests --cov=datadreamer --cov-report xml --junit-xml pytest.xml + + - name: Run tests [Windows, macOS] + if: matrix.os != 'ubuntu-latest' || matrix.version != '3.10' + run: pytest tests --junit-xml pytest.xml + + - name: Generate coverage badge [Ubuntu] + if: matrix.os == 'ubuntu-latest' && matrix.version == '3.10' + run: coverage-badge -o media/coverage_badge.svg -f + + - name: Generate coverage report [Ubuntu] + if: matrix.os == 'ubuntu-latest' && matrix.version == '3.10' + uses: orgoro/coverage@v3.1 + with: + coverageFile: coverage.xml + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Commit coverage badge [Ubuntu] + if: matrix.os == 'ubuntu-latest' && matrix.version == '3.10' + run: | + git config --global user.name 'GitHub Actions' + git config --global user.email 'actions@github.com' + git diff --quiet media/coverage_badge.svg || { + git add media/coverage_badge.svg + git commit -m "[Automated] Updated coverage badge" + } + + - name: Push changes [Ubuntu] + if: matrix.os == 'ubuntu-latest' && matrix.version == '3.10' + uses: ad-m/github-push-action@master + with: + branch: ${{ github.head_ref }} + + - name: Upload Test Results + if: always() + uses: actions/upload-artifact@v4 + with: + name: Test Results [${{ matrix.os }}] (Python ${{ matrix.version }}) + path: pytest.xml + retention-days: 10 + if-no-files-found: error + + publish-test-results: + name: "Publish Tests Results" + needs: run_tests + runs-on: ubuntu-latest + permissions: + checks: write + pull-requests: write + if: always() + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ github.head_ref }} + + - name: Download Artifacts + uses: actions/download-artifact@v3 + with: + path: artifacts + + - name: Publish Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + with: + files: "artifacts/**/*.xml" \ No newline at end of file diff --git a/tests/unittests/test_image_generation.py b/tests/unittests/test_image_generation.py new file mode 100644 index 0000000..20d6193 --- /dev/null +++ b/tests/unittests/test_image_generation.py @@ -0,0 +1,17 @@ +import pytest + + +def test_clip_image_tester(): + pass + + +def test_sdxl_image_generator(): + pass + + +def test_sdxl_turbo_image_generator(): + pass + + +if __name__ == "__main__": + pytest.main() diff --git a/tests/unittests/test_prompt_generation.py b/tests/unittests/test_prompt_generation.py new file mode 100644 index 0000000..eee3bcb --- /dev/null +++ b/tests/unittests/test_prompt_generation.py @@ -0,0 +1,17 @@ +import pytest + + +def test_simple_prompt_generator(): + pass + + +def test_lm_prompt_generator(): + pass + + +def test_synonym_generator(): + pass + + +if __name__ == "__main__": + pytest.main() From 25d1669adbb7cc37dd0e11691ae2354df44d0a3c Mon Sep 17 00:00:00 2001 From: Jan Cuhel Date: Sun, 31 Dec 2023 16:08:57 +0100 Subject: [PATCH 02/24] Add tests for prompt generation --- tests/unittests/test_prompt_generation.py | 51 +++++++++++++++++++++-- 1 file changed, 48 insertions(+), 3 deletions(-) diff --git a/tests/unittests/test_prompt_generation.py b/tests/unittests/test_prompt_generation.py index eee3bcb..5abf5e1 100644 --- a/tests/unittests/test_prompt_generation.py +++ b/tests/unittests/test_prompt_generation.py @@ -1,16 +1,61 @@ +from datadreamer.prompt_generation.lm_prompt_generator import LMPromptGenerator +from datadreamer.prompt_generation.simple_prompt_generator import SimplePromptGenerator import pytest +from datadreamer.prompt_generation.synonym_generator import SynonymGenerator + def test_simple_prompt_generator(): - pass + class_names = ["dog", "cat", "bird", "tree", "car", "person", "house", "flower"] + prompt_generator = SimplePromptGenerator(class_names, prompts_number=10) + prompts = prompt_generator.generate_prompts() + # Check that the some prompts were generated + assert len(prompts) > 0 + # Iterate through the prompts + for selected_objects, prompt_text in prompts: + # Selected objects aren't empty + assert len(selected_objects) > 0 + # The slected objects are in the range + assert ( + prompt_generator.num_objects_range[0] + <= len(selected_objects) + <= prompt_generator.num_objects_range[1] + ) + # Check the generated text + assert prompt_text == f"A photo of a {', a '.join(selected_objects)}" def test_lm_prompt_generator(): - pass + object_names = ["aeroplane", "bicycle", "bird", "boat"] + prompt_generator = LMPromptGenerator(class_names=object_names) + prompts = prompt_generator.generate_prompts() + # Check that the some prompts were generated + assert len(prompts) > 0 + # Iterate through the prompts + for selected_objects, prompt_text in prompts: + # Selected objects aren't empty + assert len(selected_objects) > 0 + # The slected objects are in the range + assert ( + prompt_generator.num_objects_range[0] + <= len(selected_objects) + <= prompt_generator.num_objects_range[1] + ) + # Check the generated text + assert len(prompt_text) > 0 and any( + [x in prompt_text for x in selected_objects] + ) def test_synonym_generator(): - pass + generator = SynonymGenerator( + "mistralai/Mistral-7B-Instruct-v0.1", synonyms_number=3 + ) + synonyms = generator.generate_synonyms_for_list( + ["astronaut", "cat", "dog", "person", "horse"] + ) + # Check that the some synonyms were generated + assert len(synonyms) > 0 if __name__ == "__main__": From e036d9bfdd879969af8d80e29ec69d459cd97406 Mon Sep 17 00:00:00 2001 From: Jan Cuhel Date: Tue, 2 Jan 2024 23:46:19 +0100 Subject: [PATCH 03/24] Update tests --- .github/workflows/tests.yaml | 2 +- tests/unittests/test_prompt_generation.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index d5fbf84..ffa69bc 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -2,7 +2,7 @@ name: Tests on: pull_request: - branches: [ dev, main ] + branches: [ dev, main, add-tests ] paths: - 'src/datadreamer/**/**.py' - 'tests/**/**.py' diff --git a/tests/unittests/test_prompt_generation.py b/tests/unittests/test_prompt_generation.py index 5abf5e1..d4c8c38 100644 --- a/tests/unittests/test_prompt_generation.py +++ b/tests/unittests/test_prompt_generation.py @@ -27,7 +27,7 @@ def test_simple_prompt_generator(): def test_lm_prompt_generator(): object_names = ["aeroplane", "bicycle", "bird", "boat"] - prompt_generator = LMPromptGenerator(class_names=object_names) + prompt_generator = LMPromptGenerator(class_names=object_names, prompts_number=2) prompts = prompt_generator.generate_prompts() # Check that the some prompts were generated assert len(prompts) > 0 From dc56670ead8160917be5bcdfa22838f09eeeef32 Mon Sep 17 00:00:00 2001 From: Jan Cuhel Date: Wed, 3 Jan 2024 01:09:09 +0100 Subject: [PATCH 04/24] Update prompt tests & add img generation tests, annotators and pipeline structures --- tests/integration/test_pipeline.py | 17 +++++++++++++++++ tests/unittests/test_annotators.py | 11 +++++++++++ tests/unittests/test_image_generation.py | 17 +++++++++++++++-- tests/unittests/test_prompt_generation.py | 9 +++++++-- 4 files changed, 50 insertions(+), 4 deletions(-) create mode 100644 tests/integration/test_pipeline.py create mode 100644 tests/unittests/test_annotators.py diff --git a/tests/integration/test_pipeline.py b/tests/integration/test_pipeline.py new file mode 100644 index 0000000..b3c1734 --- /dev/null +++ b/tests/integration/test_pipeline.py @@ -0,0 +1,17 @@ +import pytest +import torch + + +def test_detection_pipeline(): + device = "cuda" if torch.cuda.is_available() else "cpu" + cmd = "datadreamer --save_dir data-det/ --class_names alien mars cat --prompts_number 2 --prompt_generator simple --num_objects_range 1 2 --image_generator sdxl-turbo --device cpu" + pass + +def test_classification_pipeline(): + device = "cuda" if torch.cuda.is_available() else "cpu" + cmd = "datadreamer --task classification --save_dir data_cls/ --class_names alien mars cat --prompts_number 2 --prompt_generator simple --num_objects_range 1 2 --image_generator sdxl-turbo --device cpu" + pass + + +if __name__ == "__main__": + pytest.main() diff --git a/tests/unittests/test_annotators.py b/tests/unittests/test_annotators.py new file mode 100644 index 0000000..73f0d8b --- /dev/null +++ b/tests/unittests/test_annotators.py @@ -0,0 +1,11 @@ +import pytest +import torch + + +def test_owlv2_annotator(): + device = "cuda" if torch.cuda.is_available() else "cpu" + pass + + +if __name__ == "__main__": + pytest.main() diff --git a/tests/unittests/test_image_generation.py b/tests/unittests/test_image_generation.py index 20d6193..a015533 100644 --- a/tests/unittests/test_image_generation.py +++ b/tests/unittests/test_image_generation.py @@ -1,16 +1,29 @@ import pytest +import torch +from datadreamer.image_generation.sdxl_image_generator import StableDiffusionImageGenerator +from datadreamer.image_generation.sdxl_turbo_image_generator import StableDiffusionTurboImageGenerator +from datadreamer.image_generation.clip_image_tester import ClipImageTester def test_clip_image_tester(): + device = "cuda" if torch.cuda.is_available() else "cpu" pass def test_sdxl_image_generator(): - pass + device = "cuda" if torch.cuda.is_available() else "cpu" + # image_generator = StableDiffusionImageGenerator(device=device) + image_generator = StableDiffusionImageGenerator() + generated_image = image_generator.generate_images(["A photo of a cat, dog"], [["cat", "dog"]]) + image_generator.release(empty_cuda_cache=True if device != 'cpu' else False) def test_sdxl_turbo_image_generator(): - pass + device = "cuda" if torch.cuda.is_available() else "cpu" + # image_generator = StableDiffusionTurboImageGenerator(device=device) + image_generator = StableDiffusionTurboImageGenerator() + generated_image = image_generator.generate_images(["A photo of a cat, dog"], [["cat", "dog"]]) + image_generator.release(empty_cuda_cache=True if device != 'cpu' else False) if __name__ == "__main__": diff --git a/tests/unittests/test_prompt_generation.py b/tests/unittests/test_prompt_generation.py index d4c8c38..de25b6b 100644 --- a/tests/unittests/test_prompt_generation.py +++ b/tests/unittests/test_prompt_generation.py @@ -1,6 +1,7 @@ from datadreamer.prompt_generation.lm_prompt_generator import LMPromptGenerator from datadreamer.prompt_generation.simple_prompt_generator import SimplePromptGenerator import pytest +import torch from datadreamer.prompt_generation.synonym_generator import SynonymGenerator @@ -26,8 +27,9 @@ def test_simple_prompt_generator(): def test_lm_prompt_generator(): + device = "cuda" if torch.cuda.is_available() else "cpu" object_names = ["aeroplane", "bicycle", "bird", "boat"] - prompt_generator = LMPromptGenerator(class_names=object_names, prompts_number=2) + prompt_generator = LMPromptGenerator(class_names=object_names, prompts_number=2, device=device) prompts = prompt_generator.generate_prompts() # Check that the some prompts were generated assert len(prompts) > 0 @@ -45,17 +47,20 @@ def test_lm_prompt_generator(): assert len(prompt_text) > 0 and any( [x in prompt_text for x in selected_objects] ) + prompt_generator.release(empty_cuda_cache=True if device != 'cpu' else False) def test_synonym_generator(): + device = "cuda" if torch.cuda.is_available() else "cpu" generator = SynonymGenerator( - "mistralai/Mistral-7B-Instruct-v0.1", synonyms_number=3 + "mistralai/Mistral-7B-Instruct-v0.1", synonyms_number=3, device=device ) synonyms = generator.generate_synonyms_for_list( ["astronaut", "cat", "dog", "person", "horse"] ) # Check that the some synonyms were generated assert len(synonyms) > 0 + generator.release(empty_cuda_cache=True if device != 'cpu' else False) if __name__ == "__main__": From 0d5698875d60b22394d5efb828a8909253512b82 Mon Sep 17 00:00:00 2001 From: Jan Cuhel Date: Sun, 7 Jan 2024 15:31:46 +0100 Subject: [PATCH 05/24] Add pipeline test --- tests/integration/test_pipeline.py | 31 ++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_pipeline.py b/tests/integration/test_pipeline.py index b3c1734..c0f2b08 100644 --- a/tests/integration/test_pipeline.py +++ b/tests/integration/test_pipeline.py @@ -1,16 +1,39 @@ import pytest import torch +import os +import subprocess def test_detection_pipeline(): device = "cuda" if torch.cuda.is_available() else "cpu" - cmd = "datadreamer --save_dir data-det/ --class_names alien mars cat --prompts_number 2 --prompt_generator simple --num_objects_range 1 2 --image_generator sdxl-turbo --device cpu" - pass + cmd = f"datadreamer --save_dir data-det/ --class_names alien mars cat --prompts_number 2 --prompt_generator simple --num_objects_range 1 2 --image_generator sdxl-turbo --device {device}" + + result = subprocess.run(cmd, shell=True) + assert result.returncode == 0, "Command failed to run" + + assert os.path.isdir("data/data-det"), "Directory not created" + + files = ["annotations.json", "generation_args.json", "image_1.jpg", "image_0.jpg", "prompts.json"] + for file in files: + assert os.path.isfile(os.path.join("data/data_cls", file)), f"{file} not created" + + assert os.path.isdir("data/data_cls/bboxes_visualization"), "bboxes_visualization directory not created" + def test_classification_pipeline(): device = "cuda" if torch.cuda.is_available() else "cpu" - cmd = "datadreamer --task classification --save_dir data_cls/ --class_names alien mars cat --prompts_number 2 --prompt_generator simple --num_objects_range 1 2 --image_generator sdxl-turbo --device cpu" - pass + cmd = f"datadreamer --task classification --save_dir data-cls/ --class_names alien mars cat --prompts_number 2 --prompt_generator simple --num_objects_range 1 2 --image_generator sdxl-turbo --device {device}" + + result = subprocess.run(cmd, shell=True) + assert result.returncode == 0, "Command failed to run" + + assert os.path.isdir("data/data-cls"), "Directory not created" + + files = ["annotations.json", "generation_args.json", "image_1.jpg", "image_0.jpg", "prompts.json"] + for file in files: + assert os.path.isfile(os.path.join("data/data_cls", file)), f"{file} not created" + + assert os.path.isdir("data/data_cls/bboxes_visualization"), "bboxes_visualization directory not created" if __name__ == "__main__": From 5285f654dd5b483eecf55187315bd1e84b4434f4 Mon Sep 17 00:00:00 2001 From: Jan Cuhel Date: Wed, 10 Jan 2024 06:39:18 +0100 Subject: [PATCH 06/24] Update workflow & tests --- .github/workflows/tests.yaml | 17 ++++++++++++++--- tests/unittests/test_annotators.py | 3 ++- tests/unittests/test_image_generation.py | 11 ++++++++--- 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index ffa69bc..a57fd52 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -1,8 +1,14 @@ name: Tests on: + push: + branches: [ master, main ] + paths: + - 'src/datadreamer/**/**.py' + - 'tests/**/**.py' + - .github/workflows/tests.yaml pull_request: - branches: [ dev, main, add-tests ] + branches: [ dev, main ] paths: - 'src/datadreamer/**/**.py' - 'tests/**/**.py' @@ -36,14 +42,19 @@ jobs: sudo apt update sudo apt install -y pandoc pip install -e .[dev] + pip install pytest coverage-badge - name: Install dependencies [Windows] if: matrix.os == 'windows-latest' - run: pip install -e .[dev] + run: | + pip install -e .[dev] + pip install pytest coverage-badge - name: Install dependencies [macOS] if: matrix.os == 'macOS-latest' - run: pip install -e .[dev] + run: | + pip install -e .[dev] + pip install pytest coverage-badge - name: Run tests with coverage [Ubuntu] if: matrix.os == 'ubuntu-latest' && matrix.version == '3.10' diff --git a/tests/unittests/test_annotators.py b/tests/unittests/test_annotators.py index 73f0d8b..4be509f 100644 --- a/tests/unittests/test_annotators.py +++ b/tests/unittests/test_annotators.py @@ -4,7 +4,8 @@ def test_owlv2_annotator(): device = "cuda" if torch.cuda.is_available() else "cpu" - pass + assert device in ["cuda", "cpu"] + if __name__ == "__main__": diff --git a/tests/unittests/test_image_generation.py b/tests/unittests/test_image_generation.py index a015533..09e9879 100644 --- a/tests/unittests/test_image_generation.py +++ b/tests/unittests/test_image_generation.py @@ -1,5 +1,6 @@ import pytest import torch +from PIL import Image from datadreamer.image_generation.sdxl_image_generator import StableDiffusionImageGenerator from datadreamer.image_generation.sdxl_turbo_image_generator import StableDiffusionTurboImageGenerator from datadreamer.image_generation.clip_image_tester import ClipImageTester @@ -7,14 +8,16 @@ def test_clip_image_tester(): device = "cuda" if torch.cuda.is_available() else "cpu" - pass + assert device in ["cuda", "cpu"] def test_sdxl_image_generator(): device = "cuda" if torch.cuda.is_available() else "cpu" # image_generator = StableDiffusionImageGenerator(device=device) image_generator = StableDiffusionImageGenerator() - generated_image = image_generator.generate_images(["A photo of a cat, dog"], [["cat", "dog"]]) + for generated_image in image_generator.generate_images(["A photo of a cat, dog"], [["cat", "dog"]]): + assert generated_image is not None + assert isinstance(generated_image, Image.Image) image_generator.release(empty_cuda_cache=True if device != 'cpu' else False) @@ -22,7 +25,9 @@ def test_sdxl_turbo_image_generator(): device = "cuda" if torch.cuda.is_available() else "cpu" # image_generator = StableDiffusionTurboImageGenerator(device=device) image_generator = StableDiffusionTurboImageGenerator() - generated_image = image_generator.generate_images(["A photo of a cat, dog"], [["cat", "dog"]]) + for generated_image in image_generator.generate_images(["A photo of a cat, dog"], [["cat", "dog"]]): + assert generated_image is not None + assert isinstance(generated_image, Image.Image) image_generator.release(empty_cuda_cache=True if device != 'cpu' else False) From 9e1d0a52b19eb0d3843950bd22c959245231fb8e Mon Sep 17 00:00:00 2001 From: Jan Cuhel Date: Wed, 10 Jan 2024 06:40:59 +0100 Subject: [PATCH 07/24] Update workflow --- .github/workflows/tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index a57fd52..97fe35a 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -2,7 +2,7 @@ name: Tests on: push: - branches: [ master, main ] + branches: [ add-tests ] paths: - 'src/datadreamer/**/**.py' - 'tests/**/**.py' From f33fb341336ef6dbbf004cf29f6583cf4478f2fe Mon Sep 17 00:00:00 2001 From: Jan Cuhel Date: Wed, 10 Jan 2024 07:20:32 +0100 Subject: [PATCH 08/24] Update tests --- tests/integration/test_pipeline.py | 8 ++++---- tests/unittests/test_image_generation.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/integration/test_pipeline.py b/tests/integration/test_pipeline.py index c0f2b08..7fe38e9 100644 --- a/tests/integration/test_pipeline.py +++ b/tests/integration/test_pipeline.py @@ -5,8 +5,8 @@ def test_detection_pipeline(): - device = "cuda" if torch.cuda.is_available() else "cpu" - cmd = f"datadreamer --save_dir data-det/ --class_names alien mars cat --prompts_number 2 --prompt_generator simple --num_objects_range 1 2 --image_generator sdxl-turbo --device {device}" + device = "cpu" # "cuda" if torch.cuda.is_available() else "cpu" + cmd = f"datadreamer --save_dir data/data-det/ --class_names alien mars cat --prompts_number 2 --prompt_generator simple --num_objects_range 1 2 --image_generator sdxl-turbo --device {device}" result = subprocess.run(cmd, shell=True) assert result.returncode == 0, "Command failed to run" @@ -21,8 +21,8 @@ def test_detection_pipeline(): def test_classification_pipeline(): - device = "cuda" if torch.cuda.is_available() else "cpu" - cmd = f"datadreamer --task classification --save_dir data-cls/ --class_names alien mars cat --prompts_number 2 --prompt_generator simple --num_objects_range 1 2 --image_generator sdxl-turbo --device {device}" + device = "cpu" # "cuda" if torch.cuda.is_available() else "cpu" + cmd = f"datadreamer --task classification --save_dir data/data-cls/ --class_names alien mars cat --prompts_number 2 --prompt_generator simple --num_objects_range 1 2 --image_generator sdxl-turbo --device {device}" result = subprocess.run(cmd, shell=True) assert result.returncode == 0, "Command failed to run" diff --git a/tests/unittests/test_image_generation.py b/tests/unittests/test_image_generation.py index 09e9879..434edf2 100644 --- a/tests/unittests/test_image_generation.py +++ b/tests/unittests/test_image_generation.py @@ -12,7 +12,7 @@ def test_clip_image_tester(): def test_sdxl_image_generator(): - device = "cuda" if torch.cuda.is_available() else "cpu" + device = "cpu" # "cuda" if torch.cuda.is_available() else "cpu" # image_generator = StableDiffusionImageGenerator(device=device) image_generator = StableDiffusionImageGenerator() for generated_image in image_generator.generate_images(["A photo of a cat, dog"], [["cat", "dog"]]): @@ -22,7 +22,7 @@ def test_sdxl_image_generator(): def test_sdxl_turbo_image_generator(): - device = "cuda" if torch.cuda.is_available() else "cpu" + device = "cpu" # "cuda" if torch.cuda.is_available() else "cpu" # image_generator = StableDiffusionTurboImageGenerator(device=device) image_generator = StableDiffusionTurboImageGenerator() for generated_image in image_generator.generate_images(["A photo of a cat, dog"], [["cat", "dog"]]): From 45e85fd3035537344094116fe2fca92965bfd841 Mon Sep 17 00:00:00 2001 From: Jan Cuhel Date: Wed, 10 Jan 2024 09:04:51 +0100 Subject: [PATCH 09/24] Add annotator test & clip test & update pipeline test --- .github/workflows/tests.yaml | 12 +++--- tests/integration/test_pipeline.py | 52 ++++++++++++++++------- tests/unittests/test_annotators.py | 25 ++++++++++- tests/unittests/test_image_generation.py | 42 ++++++++++++++---- tests/unittests/test_prompt_generation.py | 26 +++++++++--- 5 files changed, 117 insertions(+), 40 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 97fe35a..3827dad 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -1,12 +1,12 @@ name: Tests on: - push: - branches: [ add-tests ] - paths: - - 'src/datadreamer/**/**.py' - - 'tests/**/**.py' - - .github/workflows/tests.yaml + # push: + # branches: [ add-tests ] + # paths: + # - 'src/datadreamer/**/**.py' + # - 'tests/**/**.py' + # - .github/workflows/tests.yaml pull_request: branches: [ dev, main ] paths: diff --git a/tests/integration/test_pipeline.py b/tests/integration/test_pipeline.py index 7fe38e9..e9b962f 100644 --- a/tests/integration/test_pipeline.py +++ b/tests/integration/test_pipeline.py @@ -5,35 +5,55 @@ def test_detection_pipeline(): - device = "cpu" # "cuda" if torch.cuda.is_available() else "cpu" + device = "cpu" # "cuda" if torch.cuda.is_available() else "cpu" cmd = f"datadreamer --save_dir data/data-det/ --class_names alien mars cat --prompts_number 2 --prompt_generator simple --num_objects_range 1 2 --image_generator sdxl-turbo --device {device}" - + result = subprocess.run(cmd, shell=True) assert result.returncode == 0, "Command failed to run" - assert os.path.isdir("data/data-det"), "Directory not created" - - files = ["annotations.json", "generation_args.json", "image_1.jpg", "image_0.jpg", "prompts.json"] + assert os.path.isdir("data/data-det/"), "Directory not created" + + files = [ + "annotations.json", + "generation_args.json", + "image_1.jpg", + "image_0.jpg", + "prompts.json", + ] for file in files: - assert os.path.isfile(os.path.join("data/data_cls", file)), f"{file} not created" - - assert os.path.isdir("data/data_cls/bboxes_visualization"), "bboxes_visualization directory not created" + assert os.path.isfile( + os.path.join("data/data-det/", file) + ), f"{file} not created" + + assert os.path.isdir( + "data/data-det/bboxes_visualization" + ), "bboxes_visualization directory not created" def test_classification_pipeline(): - device = "cpu" # "cuda" if torch.cuda.is_available() else "cpu" + device = "cpu" # "cuda" if torch.cuda.is_available() else "cpu" cmd = f"datadreamer --task classification --save_dir data/data-cls/ --class_names alien mars cat --prompts_number 2 --prompt_generator simple --num_objects_range 1 2 --image_generator sdxl-turbo --device {device}" - + result = subprocess.run(cmd, shell=True) assert result.returncode == 0, "Command failed to run" - assert os.path.isdir("data/data-cls"), "Directory not created" - - files = ["annotations.json", "generation_args.json", "image_1.jpg", "image_0.jpg", "prompts.json"] + assert os.path.isdir("data/data-cls/"), "Directory not created" + + files = [ + "annotations.json", + "generation_args.json", + "image_1.jpg", + "image_0.jpg", + "prompts.json", + ] for file in files: - assert os.path.isfile(os.path.join("data/data_cls", file)), f"{file} not created" - - assert os.path.isdir("data/data_cls/bboxes_visualization"), "bboxes_visualization directory not created" + assert os.path.isfile( + os.path.join("data/data-cls/", file) + ), f"{file} not created" + + assert os.path.isdir( + "data/data-cls/bboxes_visualization" + ), "bboxes_visualization directory not created" if __name__ == "__main__": diff --git a/tests/unittests/test_annotators.py b/tests/unittests/test_annotators.py index 4be509f..f3f4634 100644 --- a/tests/unittests/test_annotators.py +++ b/tests/unittests/test_annotators.py @@ -1,11 +1,32 @@ import pytest import torch +from PIL import Image +import requests +from datadreamer.dataset_annotation.owlv2_annotator import OWLv2Annotator def test_owlv2_annotator(): + url = "https://ultralytics.com/images/bus.jpg" + im = Image.open(requests.get(url, stream=True).raw) device = "cuda" if torch.cuda.is_available() else "cpu" - assert device in ["cuda", "cpu"] - + annotator = OWLv2Annotator(device=device) + final_boxes, final_scores, final_labels = annotator.annotate(im, ["bus", "people"]) + # Assert that the boxes, scores and labels are tensors + assert type(final_boxes) == torch.Tensor + assert type(final_scores) == torch.Tensor + assert type(final_labels) == torch.Tensor + # Get the number of objects detected + num_objects = final_boxes.shape[0] + # Check that the boxes has correct shape + assert final_boxes.shape == (num_objects, 4) + # Check that the scores has correct shape + assert final_scores.shape == (num_objects,) + # Check that the labels has correct shape + assert final_labels.shape == (num_objects,) + # Check that the scores are not zero + assert torch.all(final_scores > 0) + # Check that the labels are bigger or equal to zero + assert torch.all(final_labels >= 0) if __name__ == "__main__": diff --git a/tests/unittests/test_image_generation.py b/tests/unittests/test_image_generation.py index 434edf2..0dc0380 100644 --- a/tests/unittests/test_image_generation.py +++ b/tests/unittests/test_image_generation.py @@ -1,34 +1,58 @@ import pytest import torch from PIL import Image -from datadreamer.image_generation.sdxl_image_generator import StableDiffusionImageGenerator -from datadreamer.image_generation.sdxl_turbo_image_generator import StableDiffusionTurboImageGenerator +import requests +from datadreamer.image_generation.sdxl_image_generator import ( + StableDiffusionImageGenerator, +) +from datadreamer.image_generation.sdxl_turbo_image_generator import ( + StableDiffusionTurboImageGenerator, +) from datadreamer.image_generation.clip_image_tester import ClipImageTester def test_clip_image_tester(): + url = "https://ultralytics.com/images/bus.jpg" + im = Image.open(requests.get(url, stream=True).raw) device = "cuda" if torch.cuda.is_available() else "cpu" - assert device in ["cuda", "cpu"] + tester = ClipImageTester(device=device) + passed, probs, num_passed = tester.test_image(im, ["bus"]) + # Check that the image passed the test + assert passed is True + # Check that the number of objects passed is correct + assert num_passed == 1 + # Check that the probability has correct shape + assert probs.shape == (1, 1) + # Check that the probability is not zero + assert probs[0, 0] > 0 + # Release the tester + tester.release(empty_cuda_cache=True if device != "cpu" else False) def test_sdxl_image_generator(): - device = "cpu" # "cuda" if torch.cuda.is_available() else "cpu" + device = "cuda" if torch.cuda.is_available() else "cpu" # image_generator = StableDiffusionImageGenerator(device=device) image_generator = StableDiffusionImageGenerator() - for generated_image in image_generator.generate_images(["A photo of a cat, dog"], [["cat", "dog"]]): + for generated_image in image_generator.generate_images( + ["A photo of a cat, dog"], [["cat", "dog"]] + ): assert generated_image is not None assert isinstance(generated_image, Image.Image) - image_generator.release(empty_cuda_cache=True if device != 'cpu' else False) + # Release the generator + image_generator.release(empty_cuda_cache=True if device != "cpu" else False) def test_sdxl_turbo_image_generator(): - device = "cpu" # "cuda" if torch.cuda.is_available() else "cpu" + device = "cuda" if torch.cuda.is_available() else "cpu" # image_generator = StableDiffusionTurboImageGenerator(device=device) image_generator = StableDiffusionTurboImageGenerator() - for generated_image in image_generator.generate_images(["A photo of a cat, dog"], [["cat", "dog"]]): + for generated_image in image_generator.generate_images( + ["A photo of a cat, dog"], [["cat", "dog"]] + ): assert generated_image is not None assert isinstance(generated_image, Image.Image) - image_generator.release(empty_cuda_cache=True if device != 'cpu' else False) + # Release the generator + image_generator.release(empty_cuda_cache=True if device != "cpu" else False) if __name__ == "__main__": diff --git a/tests/unittests/test_prompt_generation.py b/tests/unittests/test_prompt_generation.py index de25b6b..5f07282 100644 --- a/tests/unittests/test_prompt_generation.py +++ b/tests/unittests/test_prompt_generation.py @@ -29,7 +29,9 @@ def test_simple_prompt_generator(): def test_lm_prompt_generator(): device = "cuda" if torch.cuda.is_available() else "cpu" object_names = ["aeroplane", "bicycle", "bird", "boat"] - prompt_generator = LMPromptGenerator(class_names=object_names, prompts_number=2, device=device) + prompt_generator = LMPromptGenerator( + class_names=object_names, prompts_number=2, device=device + ) prompts = prompt_generator.generate_prompts() # Check that the some prompts were generated assert len(prompts) > 0 @@ -47,20 +49,30 @@ def test_lm_prompt_generator(): assert len(prompt_text) > 0 and any( [x in prompt_text for x in selected_objects] ) - prompt_generator.release(empty_cuda_cache=True if device != 'cpu' else False) + prompt_generator.release(empty_cuda_cache=True if device != "cpu" else False) def test_synonym_generator(): device = "cuda" if torch.cuda.is_available() else "cpu" + synonyms_num = 3 generator = SynonymGenerator( - "mistralai/Mistral-7B-Instruct-v0.1", synonyms_number=3, device=device - ) - synonyms = generator.generate_synonyms_for_list( - ["astronaut", "cat", "dog", "person", "horse"] + "mistralai/Mistral-7B-Instruct-v0.1", + synonyms_number=synonyms_num, + device=device, ) + synonyms = generator.generate_synonyms_for_list(["astronaut", "cat", "dog"]) # Check that the some synonyms were generated assert len(synonyms) > 0 - generator.release(empty_cuda_cache=True if device != 'cpu' else False) + # Iterate through the synonyms + for word, synonym_list in synonyms.items(): + # Check that the word is not empty + assert len(word) > 0 + # Check that the synonym list is not empty and has the correct number of synonyms + assert len(synonym_list) > 0 and len(synonym_list) == synonyms_num + # Check that the synonyms are not empty + for synonym in synonym_list: + assert len(synonym) > 0 + generator.release(empty_cuda_cache=True if device != "cpu" else False) if __name__ == "__main__": From 09efd51cfbe46fd8683791d09cd68ec8a16ae093 Mon Sep 17 00:00:00 2001 From: Jan Cuhel Date: Thu, 11 Jan 2024 23:25:23 +0100 Subject: [PATCH 10/24] Update the test workflow & add arg checks for the pipeline & add several tests and skips --- .github/workflows/tests.yaml | 8 +- tests/integration/test_pipeline.py | 505 ++++++++++++++++++++-- tests/unittests/test_annotators.py | 12 +- tests/unittests/test_image_generation.py | 47 +- tests/unittests/test_prompt_generation.py | 37 +- 5 files changed, 539 insertions(+), 70 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 3827dad..17bc8bf 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -42,23 +42,23 @@ jobs: sudo apt update sudo apt install -y pandoc pip install -e .[dev] - pip install pytest coverage-badge + pip install coverage-badge>=1.1.0 pytest-cov>=4.1.0 - name: Install dependencies [Windows] if: matrix.os == 'windows-latest' run: | pip install -e .[dev] - pip install pytest coverage-badge + pip install coverage-badge>=1.1.0 pytest-cov>=4.1.0 - name: Install dependencies [macOS] if: matrix.os == 'macOS-latest' run: | pip install -e .[dev] - pip install pytest coverage-badge + pip install coverage-badge>=1.1.0 pytest-cov>=4.1.0 - name: Run tests with coverage [Ubuntu] if: matrix.os == 'ubuntu-latest' && matrix.version == '3.10' - run: pytest tests --cov=datadreamer --cov-report xml --junit-xml pytest.xml + run: pytest tests --cov=src --cov-report xml --junit-xml pytest.xml - name: Run tests [Windows, macOS] if: matrix.os != 'ubuntu-latest' || matrix.version != '3.10' diff --git a/tests/integration/test_pipeline.py b/tests/integration/test_pipeline.py index e9b962f..08f9189 100644 --- a/tests/integration/test_pipeline.py +++ b/tests/integration/test_pipeline.py @@ -2,59 +2,500 @@ import torch import os import subprocess +import psutil -def test_detection_pipeline(): - device = "cpu" # "cuda" if torch.cuda.is_available() else "cpu" - cmd = f"datadreamer --save_dir data/data-det/ --class_names alien mars cat --prompts_number 2 --prompt_generator simple --num_objects_range 1 2 --image_generator sdxl-turbo --device {device}" +# Get the total memory in GB +total_memory = psutil.virtual_memory().total / (1024 ** 3) + +def _check_detection_pipeline(cmd: str, target_folder: str): + # Run the command result = subprocess.run(cmd, shell=True) assert result.returncode == 0, "Command failed to run" - - assert os.path.isdir("data/data-det/"), "Directory not created" - + # Check that the target folder is a folder + assert os.path.isdir(target_folder), "Directory not created" files = [ "annotations.json", "generation_args.json", - "image_1.jpg", "image_0.jpg", "prompts.json", ] + # Check that all the files were created for file in files: assert os.path.isfile( - os.path.join("data/data-det/", file) + os.path.join(target_folder, file) ), f"{file} not created" - + # Check that the "bboxes_visualization" folder was created assert os.path.isdir( - "data/data-det/bboxes_visualization" + os.path.join(target_folder, "bboxes_visualization") ), "bboxes_visualization directory not created" -def test_classification_pipeline(): - device = "cpu" # "cuda" if torch.cuda.is_available() else "cpu" - cmd = f"datadreamer --task classification --save_dir data/data-cls/ --class_names alien mars cat --prompts_number 2 --prompt_generator simple --num_objects_range 1 2 --image_generator sdxl-turbo --device {device}" +def _check_wrong_argument_choice(cmd: str): + with pytest.raises(subprocess.CalledProcessError): + subprocess.check_call(cmd, shell=True) - result = subprocess.run(cmd, shell=True) - assert result.returncode == 0, "Command failed to run" - assert os.path.isdir("data/data-cls/"), "Directory not created" +def _check_wrong_value(cmd: str): + with pytest.raises(ValueError): + try: + subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as e: + raise ValueError(e.output.decode()) - files = [ - "annotations.json", - "generation_args.json", - "image_1.jpg", - "image_0.jpg", - "prompts.json", - ] - for file in files: - assert os.path.isfile( - os.path.join("data/data-cls/", file) - ), f"{file} not created" - assert os.path.isdir( - "data/data-cls/bboxes_visualization" - ), "bboxes_visualization directory not created" +# ========================================================= +# ARGUMENTS CHECKS +# ========================================================= +def test_invalid_task_value(): + # Define the cmd + cmd = "datadreamer --task invalid_task" + _check_wrong_argument_choice(cmd) + + +def test_invalid_save_dir(): + # Define the cmd + cmd = "datadreamer --save_dir 1" + _check_wrong_argument_choice(cmd) + + +def test_invalid_prompts_number_type(): + # Define the cmd + cmd = "datadreamer --prompts_number value" + _check_wrong_argument_choice(cmd) + + +def test_invalid_num_objects_range_type(): + # Define the cmd + cmd = "datadreamer --num_objects_range value" + _check_wrong_argument_choice(cmd) + + +def test_invalid_conf_threshold_range_type(): + # Define the cmd + cmd = "datadreamer --conf_threshold value" + _check_wrong_argument_choice(cmd) + + +def test_invalid_image_tester_patience_type(): + # Define the cmd + cmd = "datadreamer --image_tester_patience value" + _check_wrong_argument_choice(cmd) + + +def test_invalid_seed_type(): + # Define the cmd + cmd = "datadreamer --seed value --device cpu" + _check_wrong_argument_choice(cmd) + + +def test_invalid_prompt_generator(): + # Define the cmd + cmd = "datadreamer --prompt_generator invalide_value" + _check_wrong_argument_choice(cmd) + + +def test_invalid_image_generator(): + # Define the cmd + cmd = "datadreamer --image_generator invalide_value" + _check_wrong_argument_choice(cmd) + + +def test_invalid_image_annotator(): + # Define the cmd + cmd = "datadreamer --image_annotator invalide_value" + _check_wrong_argument_choice(cmd) + + +def test_invalid_device(): + # Define the cmd + cmd = "datadreamer --device invalide_value" + _check_wrong_argument_choice(cmd) + + +def test_empty_class_names(): + # Define the cmd + cmd = "datadreamer --class_names []" + _check_wrong_value(cmd) + + +def test_invalid_class_names(): + # Define the cmd + cmd = "datadreamer --class_names [2, -1]" + _check_wrong_value(cmd) + + +def test_invalid_prompts_number(): + # Define the cmd + cmd = "datadreamer --prompts_number -1" + _check_wrong_value(cmd) + + +def test_negative_conf_threshold(): + # Define the cmd + cmd = "datadreamer --conf_threshold -1" + _check_wrong_value(cmd) + + +def test_big_conf_threshold(): + # Define the cmd + cmd = "datadreamer --conf_threshold 10" + _check_wrong_value(cmd) + + +def test_invalid_image_tester_patience(): + # Define the cmd + cmd = "datadreamer --image_tester_patience -1" + _check_wrong_value(cmd) + + +def test_invalid_seed(): + # Define the cmd + cmd = "datadreamer --seed -1 --device cpu" + _check_wrong_value(cmd) + + +def test_invalid_num_objects_range(): + # Define the cmd + cmd = "datadreamer --num_objects_range 1" + _check_wrong_value(cmd) + + +def test_many_num_objects_range(): + # Define the cmd + cmd = "datadreamer --num_objects_range 1 2 3" + _check_wrong_value(cmd) + + +def test_desc_num_objects_range(): + # Define the cmd + cmd = "datadreamer --num_objects_range 3 1" + _check_wrong_value(cmd) + + +def test_negative_num_objects_range(): + # Define the cmd + cmd = "datadreamer --num_objects_range -3 1" + _check_wrong_value(cmd) + + +# ========================================================= +# DETECTION - SIMPLE LM +# ========================================================= +def test_cpu_simple_sdxl_turbo_detection_pipeline(): + # Define target folder + target_folder = "data/data-det-cpu-simple-sdxl-turbo/" + # Define the command to run the datadreamer + cmd = ( + f"datadreamer --save_dir {target_folder} " + f"--class_names alien mars cat " + f"--prompts_number 1 " + f"--prompt_generator simple " + f"--num_objects_range 1 2 " + f"--image_generator sdxl-turbo " + f"--use_image_tester " + f"--device cpu" + ) + # Check the run of the pipeline + _check_detection_pipeline(cmd, target_folder) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="Test requires GPU") +def test_cuda_simple_sdxl_turbo_detection_pipeline(): + # Define target folder + target_folder = "data/data-det-cuda-simple-sdxl-turbo/" + # Define the command to run the datadreamer + cmd = ( + f"datadreamer --save_dir {target_folder} " + f"--class_names alien mars cat " + f"--prompts_number 1 " + f"--prompt_generator simple " + f"--num_objects_range 1 2 " + f"--image_generator sdxl-turbo " + f"--use_image_tester " + f"--device cuda" + ) + # Check the run of the pipeline + _check_detection_pipeline(cmd, target_folder) + + +def test_cpu_simple_sdxl_detection_pipeline(): + # Define target folder + target_folder = "data/data-det-cpu-simple-sdxl/" + # Define the command to run the datadreamer + cmd = ( + f"datadreamer --save_dir {target_folder} " + f"--class_names alien mars cat " + f"--prompts_number 1 " + f"--prompt_generator simple " + f"--num_objects_range 1 2 " + f"--image_generator sdxl " + f"--use_image_tester " + f"--device cpu" + ) + # Check the run of the pipeline + _check_detection_pipeline(cmd, target_folder) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="Test requires GPU") +def test_cuda_simple_sdxl_detection_pipeline(): + # Define target folder + target_folder = "data/data-det-cuda-simple-sdxl/" + # Define the command to run the datadreamer + cmd = ( + f"datadreamer --save_dir {target_folder} " + f"--class_names alien mars cat " + f"--prompts_number 1 " + f"--prompt_generator simple " + f"--num_objects_range 1 2 " + f"--image_generator sdxl " + f"--use_image_tester " + f"--device cuda" + ) + # Check the run of the pipeline + _check_detection_pipeline(cmd, target_folder) + + +# ========================================================= +# DETECTION - LLM +# ========================================================= +@pytest.mark.skipif(total_memory < 28, reason="Test requires at least 28GB of RAM for running on CPU") +def test_cpu_lm_sdxl_turbo_detection_pipeline(): + # Define target folder + target_folder = "data/data-det-cpu-lm-sdxl-turbo/" + # Define the command to run the datadreamer + cmd = ( + f"datadreamer --save_dir {target_folder} " + f"--class_names alien mars cat " + f"--prompts_number 1 " + f"--prompt_generator lm " + f"--num_objects_range 1 2 " + f"--image_generator sdxl-turbo " + f"--use_image_tester " + f"--device cpu" + ) + # Check the run of the pipeline + _check_detection_pipeline(cmd, target_folder) + + +@pytest.mark.skipif(total_memory < 16 or not torch.cuda.is_available(), reason="Test requires at least 16GB of RAM and CUDA support") +def test_cuda_lm_sdxl_turbo_detection_pipeline(): + # Define target folder + target_folder = "data/data-det-cuda-lm-sdxl-turbo/" + # Define the command to run the datadreamer + cmd = ( + f"datadreamer --save_dir {target_folder} " + f"--class_names alien mars cat " + f"--prompts_number 1 " + f"--prompt_generator lm " + f"--num_objects_range 1 2 " + f"--image_generator sdxl-turbo " + f"--use_image_tester " + f"--device cuda" + ) + # Check the run of the pipeline + _check_detection_pipeline(cmd, target_folder) + + +@pytest.mark.skipif(total_memory < 28, reason="Test requires at least 28GB of RAM for running on CPU") +def test_cpu_lm_sdxl_detection_pipeline(): + # Define target folder + target_folder = "data/data-det-cpu-lm-sdxl/" + # Define the command to run the datadreamer + cmd = ( + f"datadreamer --save_dir {target_folder} " + f"--class_names alien mars cat " + f"--prompts_number 1 " + f"--prompt_generator lm " + f"--num_objects_range 1 2 " + f"--image_generator sdxl " + f"--use_image_tester " + f"--device cpu" + ) + # Check the run of the pipeline + _check_detection_pipeline(cmd, target_folder) + + +@pytest.mark.skipif(total_memory < 16 or not torch.cuda.is_available(), reason="Test requires at least 16GB of RAM and CUDA support") +def test_cuda_lm_sdxl_detection_pipeline(): + # Define target folder + target_folder = "data/data-det-cuda-lm-sdxl/" + # Define the command to run the datadreamer + cmd = ( + f"datadreamer --save_dir {target_folder} " + f"--class_names alien mars cat " + f"--prompts_number 1 " + f"--prompt_generator lm " + f"--num_objects_range 1 2 " + f"--image_generator sdxl " + f"--use_image_tester " + f"--device cuda" + ) + # Check the run of the pipeline + _check_detection_pipeline(cmd, target_folder) + + +# ========================================================= +# CLASSIFICATION - SIMPLE LM +# ========================================================= +def test_cpu_simple_sdxl_turbo_classification_pipeline(): + # Define target folder + target_folder = "data/data-cls-cpu-simple-sdxl-turbo/" + # Define the command to run the datadreamer + cmd = ( + f"datadreamer --task classification " + f"--save_dir {target_folder} " + f"--class_names alien mars cat " + f"--prompts_number 1 " + f"--prompt_generator simple " + f"--num_objects_range 1 2 " + f"--image_generator sdxl-turbo " + f"--use_image_tester " + f"--device cpu" + ) + # Check the run of the pipeline + _check_detection_pipeline(cmd, target_folder) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="Test requires GPU") +def test_cuda_simple_sdxl_turbo_classification_pipeline(): + # Define target folder + target_folder = "data/data-cls-cuda-simple-sdxl-turbo/" + # Define the command to run the datadreamer + cmd = ( + f"datadreamer --task classification " + f"--save_dir {target_folder} " + f"--class_names alien mars cat " + f"--prompts_number 1 " + f"--prompt_generator simple " + f"--num_objects_range 1 2 " + f"--image_generator sdxl-turbo " + f"--use_image_tester " + f"--device cuda" + ) + # Check the run of the pipeline + _check_detection_pipeline(cmd, target_folder) + + + +def test_cpu_simple_sdxl_classification_pipeline(): + # Define target folder + target_folder = "data/data-cls-cpu-simple-sdxl/" + # Define the command to run the datadreamer + cmd = ( + f"datadreamer --task classification " + f"--save_dir {target_folder} " + f"--class_names alien mars cat " + f"--prompts_number 1 " + f"--prompt_generator simple " + f"--num_objects_range 1 2 " + f"--image_generator sdxl " + f"--use_image_tester " + f"--device cpu" + ) + # Check the run of the pipeline + _check_detection_pipeline(cmd, target_folder) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="Test requires GPU") +def test_cuda_simple_sdxl_classification_pipeline(): + # Define target folder + target_folder = "data/data-cls-cuda-simple-sdxl/" + # Define the command to run the datadreamer + cmd = ( + f"datadreamer --task classification " + f"--save_dir {target_folder} " + f"--class_names alien mars cat " + f"--prompts_number 1 " + f"--prompt_generator simple " + f"--num_objects_range 1 2 " + f"--image_generator sdxl " + f"--use_image_tester " + f"--device cuda" + ) + # Check the run of the pipeline + _check_detection_pipeline(cmd, target_folder) + + +# ========================================================= +# CLASSIFICATION - LLM +# ========================================================= +@pytest.mark.skipif(total_memory < 28, reason="Test requires at least 28GB of RAM for running on CPU") +def test_cpu_lm_sdxl_turbo_classification_pipeline(): + # Define target folder + target_folder = "data/data-cls-cpu-lm-sdxl-turbo/" + # Define the command to run the datadreamer + cmd = ( + f"datadreamer --task classification " + f"--save_dir {target_folder} " + f"--class_names alien mars cat " + f"--prompts_number 1 " + f"--prompt_generator lm " + f"--num_objects_range 1 2 " + f"--image_generator sdxl-turbo " + f"--use_image_tester " + f"--device cpu" + ) + # Check the run of the pipeline + _check_detection_pipeline(cmd, target_folder) + + +@pytest.mark.skipif(total_memory < 16 or not torch.cuda.is_available(), reason="Test requires at least 16GB of RAM and CUDA support") +def test_cuda_lm_sdxl_turbo_classification_pipeline(): + # Define target folder + target_folder = "data/data-cls-cuda-lm-sdxl-turbo/" + # Define the command to run the datadreamer + cmd = ( + f"datadreamer --task classification " + f"--save_dir {target_folder} " + f"--class_names alien mars cat " + f"--prompts_number 1 " + f"--prompt_generator lm " + f"--num_objects_range 1 2 " + f"--image_generator sdxl-turbo " + f"--use_image_tester " + f"--device cuda" + ) + # Check the run of the pipeline + _check_detection_pipeline(cmd, target_folder) + + +@pytest.mark.skipif(total_memory < 28, reason="Test requires at least 28GB of RAM for running on CPU") +def test_cpu_lm_sdxl_classification_pipeline(): + # Define target folder + target_folder = "data/data-cls-cpu-lm-sdxl/" + # Define the command to run the datadreamer + cmd = ( + f"datadreamer --task classification " + f"--save_dir {target_folder} " + f"--class_names alien mars cat " + f"--prompts_number 1 " + f"--prompt_generator lm " + f"--num_objects_range 1 2 " + f"--image_generator sdxl " + f"--use_image_tester " + f"--device cpu" + ) + # Check the run of the pipeline + _check_detection_pipeline(cmd, target_folder) -if __name__ == "__main__": - pytest.main() +@pytest.mark.skipif(total_memory < 16 or not torch.cuda.is_available(), reason="Test requires at least 16GB of RAM and CUDA support") +def test_cuda_lm_sdxl_classification_pipeline(): + # Define target folder + target_folder = "data/data-cls-cuda-lm-sdxl/" + # Define the command to run the datadreamer + cmd = ( + f"datadreamer --task classification " + f"--save_dir {target_folder} " + f"--class_names alien mars cat " + f"--prompts_number 1 " + f"--prompt_generator lm " + f"--num_objects_range 1 2 " + f"--image_generator sdxl " + f"--use_image_tester " + f"--device cuda" + ) + # Check the run of the pipeline + _check_detection_pipeline(cmd, target_folder) diff --git a/tests/unittests/test_annotators.py b/tests/unittests/test_annotators.py index f3f4634..cdbfdd1 100644 --- a/tests/unittests/test_annotators.py +++ b/tests/unittests/test_annotators.py @@ -5,10 +5,9 @@ from datadreamer.dataset_annotation.owlv2_annotator import OWLv2Annotator -def test_owlv2_annotator(): +def _check_owlv2_annotator(device: str): url = "https://ultralytics.com/images/bus.jpg" im = Image.open(requests.get(url, stream=True).raw) - device = "cuda" if torch.cuda.is_available() else "cpu" annotator = OWLv2Annotator(device=device) final_boxes, final_scores, final_labels = annotator.annotate(im, ["bus", "people"]) # Assert that the boxes, scores and labels are tensors @@ -29,5 +28,10 @@ def test_owlv2_annotator(): assert torch.all(final_labels >= 0) -if __name__ == "__main__": - pytest.main() +@pytest.mark.skipif(not torch.cuda.is_available(), reason="Test requires GPU") +def test_cuda_owlv2_annotator(): + _check_owlv2_annotator("cuda") + + +def test_cou_owlv2_annotator(): + _check_owlv2_annotator("cpu") diff --git a/tests/unittests/test_image_generation.py b/tests/unittests/test_image_generation.py index 0dc0380..e3cc9cf 100644 --- a/tests/unittests/test_image_generation.py +++ b/tests/unittests/test_image_generation.py @@ -2,6 +2,7 @@ import torch from PIL import Image import requests +from typing import Union, Type from datadreamer.image_generation.sdxl_image_generator import ( StableDiffusionImageGenerator, ) @@ -11,10 +12,9 @@ from datadreamer.image_generation.clip_image_tester import ClipImageTester -def test_clip_image_tester(): +def _check_clip_image_tester(device: str): url = "https://ultralytics.com/images/bus.jpg" im = Image.open(requests.get(url, stream=True).raw) - device = "cuda" if torch.cuda.is_available() else "cpu" tester = ClipImageTester(device=device) passed, probs, num_passed = tester.test_image(im, ["bus"]) # Check that the image passed the test @@ -29,23 +29,18 @@ def test_clip_image_tester(): tester.release(empty_cuda_cache=True if device != "cpu" else False) -def test_sdxl_image_generator(): - device = "cuda" if torch.cuda.is_available() else "cpu" - # image_generator = StableDiffusionImageGenerator(device=device) - image_generator = StableDiffusionImageGenerator() - for generated_image in image_generator.generate_images( - ["A photo of a cat, dog"], [["cat", "dog"]] - ): - assert generated_image is not None - assert isinstance(generated_image, Image.Image) - # Release the generator - image_generator.release(empty_cuda_cache=True if device != "cpu" else False) +@pytest.mark.skipif(not torch.cuda.is_available(), reason="Test requires GPU") +def test_cuda_clip_image_tester(): + _check_clip_image_tester("cuda") + +def test_cpu_clip_image_tester(): + _check_clip_image_tester("cpu") -def test_sdxl_turbo_image_generator(): - device = "cuda" if torch.cuda.is_available() else "cpu" - # image_generator = StableDiffusionTurboImageGenerator(device=device) - image_generator = StableDiffusionTurboImageGenerator() + +def _check_image_generator(image_generator_class: Type[Union[StableDiffusionImageGenerator, StableDiffusionTurboImageGenerator]], device: str): + image_generator = image_generator_class(device=device) + # Generate images and check each of them for generated_image in image_generator.generate_images( ["A photo of a cat, dog"], [["cat", "dog"]] ): @@ -55,5 +50,19 @@ def test_sdxl_turbo_image_generator(): image_generator.release(empty_cuda_cache=True if device != "cpu" else False) -if __name__ == "__main__": - pytest.main() +@pytest.mark.skipif(not torch.cuda.is_available(), reason="Test requires GPU") +def test_cuda_sdxl_image_generator(): + _check_image_generator(StableDiffusionImageGenerator, "cuda") + + +def test_cpu_sdxl_image_generator(): + _check_image_generator(StableDiffusionImageGenerator, "cpu") + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="Test requires GPU") +def test_cuda_sdxl_turbo_image_generator(): + _check_image_generator(StableDiffusionTurboImageGenerator, "cuda") + + +def test_cpu_sdxl_turbo_image_generator(): + _check_image_generator(StableDiffusionTurboImageGenerator, "cpu") diff --git a/tests/unittests/test_prompt_generation.py b/tests/unittests/test_prompt_generation.py index 5f07282..4924408 100644 --- a/tests/unittests/test_prompt_generation.py +++ b/tests/unittests/test_prompt_generation.py @@ -2,10 +2,15 @@ from datadreamer.prompt_generation.simple_prompt_generator import SimplePromptGenerator import pytest import torch +import psutil from datadreamer.prompt_generation.synonym_generator import SynonymGenerator +# Get the total memory in GB +total_memory = psutil.virtual_memory().total / (1024 ** 3) + + def test_simple_prompt_generator(): class_names = ["dog", "cat", "bird", "tree", "car", "person", "house", "flower"] prompt_generator = SimplePromptGenerator(class_names, prompts_number=10) @@ -26,8 +31,7 @@ def test_simple_prompt_generator(): assert prompt_text == f"A photo of a {', a '.join(selected_objects)}" -def test_lm_prompt_generator(): - device = "cuda" if torch.cuda.is_available() else "cpu" +def _check_lm_prompt_generator(device: str): object_names = ["aeroplane", "bicycle", "bird", "boat"] prompt_generator = LMPromptGenerator( class_names=object_names, prompts_number=2, device=device @@ -52,14 +56,19 @@ def test_lm_prompt_generator(): prompt_generator.release(empty_cuda_cache=True if device != "cpu" else False) -def test_synonym_generator(): - device = "cuda" if torch.cuda.is_available() else "cpu" +@pytest.mark.skipif(total_memory < 16 or not torch.cuda.is_available(), reason="Test requires at least 16GB of RAM and CUDA support") +def test_cuda_lm_prompt_generator(): + _check_lm_prompt_generator("cuda") + + +@pytest.mark.skipif(total_memory < 28, reason="Test requires at least 28GB of RAM for running on CPU") +def test_cpu_lm_prompt_generator(): + _check_lm_prompt_generator("cpu") + + +def _check_synonym_generator(device: str): synonyms_num = 3 - generator = SynonymGenerator( - "mistralai/Mistral-7B-Instruct-v0.1", - synonyms_number=synonyms_num, - device=device, - ) + generator = SynonymGenerator(synonyms_number=synonyms_num, device=device) synonyms = generator.generate_synonyms_for_list(["astronaut", "cat", "dog"]) # Check that the some synonyms were generated assert len(synonyms) > 0 @@ -75,5 +84,11 @@ def test_synonym_generator(): generator.release(empty_cuda_cache=True if device != "cpu" else False) -if __name__ == "__main__": - pytest.main() +@pytest.mark.skipif(total_memory < 16 or not torch.cuda.is_available(), reason="Test requires at least 16GB of RAM and CUDA support") +def test_cuda_synonym_generator(): + _check_synonym_generator("cuda") + + +@pytest.mark.skipif(total_memory < 28, reason="Test requires at least 28GB of RAM for running on CPU") +def test_cpu_synonym_generator(): + _check_synonym_generator("cpu") From 44a54d0b71a3c0fb14d74f8cb6eae4725bbee7ea Mon Sep 17 00:00:00 2001 From: Jan Cuhel Date: Thu, 11 Jan 2024 23:26:30 +0100 Subject: [PATCH 11/24] Update test workflow --- .github/workflows/tests.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 17bc8bf..05e658d 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -1,12 +1,12 @@ name: Tests on: - # push: - # branches: [ add-tests ] - # paths: - # - 'src/datadreamer/**/**.py' - # - 'tests/**/**.py' - # - .github/workflows/tests.yaml + push: + branches: [ add-tests ] + paths: + - 'src/datadreamer/**/**.py' + - 'tests/**/**.py' + - .github/workflows/tests.yaml pull_request: branches: [ dev, main ] paths: From 34ebe34dd567b1bd074bc58202e70dd4f62e9621 Mon Sep 17 00:00:00 2001 From: Jan Cuhel Date: Fri, 12 Jan 2024 07:31:34 +0100 Subject: [PATCH 12/24] Comment some tests --- tests/integration/test_pipeline.py | 424 +++++++++++----------- tests/unittests/test_prompt_generation.py | 2 +- 2 files changed, 213 insertions(+), 213 deletions(-) diff --git a/tests/integration/test_pipeline.py b/tests/integration/test_pipeline.py index 08f9189..58ed2e0 100644 --- a/tests/integration/test_pipeline.py +++ b/tests/integration/test_pipeline.py @@ -194,7 +194,7 @@ def test_cpu_simple_sdxl_turbo_detection_pipeline(): f"--prompt_generator simple " f"--num_objects_range 1 2 " f"--image_generator sdxl-turbo " - f"--use_image_tester " + # f"--use_image_tester " f"--device cpu" ) # Check the run of the pipeline @@ -213,127 +213,127 @@ def test_cuda_simple_sdxl_turbo_detection_pipeline(): f"--prompt_generator simple " f"--num_objects_range 1 2 " f"--image_generator sdxl-turbo " - f"--use_image_tester " + # f"--use_image_tester " f"--device cuda" ) # Check the run of the pipeline _check_detection_pipeline(cmd, target_folder) -def test_cpu_simple_sdxl_detection_pipeline(): - # Define target folder - target_folder = "data/data-det-cpu-simple-sdxl/" - # Define the command to run the datadreamer - cmd = ( - f"datadreamer --save_dir {target_folder} " - f"--class_names alien mars cat " - f"--prompts_number 1 " - f"--prompt_generator simple " - f"--num_objects_range 1 2 " - f"--image_generator sdxl " - f"--use_image_tester " - f"--device cpu" - ) - # Check the run of the pipeline - _check_detection_pipeline(cmd, target_folder) - - -@pytest.mark.skipif(not torch.cuda.is_available(), reason="Test requires GPU") -def test_cuda_simple_sdxl_detection_pipeline(): - # Define target folder - target_folder = "data/data-det-cuda-simple-sdxl/" - # Define the command to run the datadreamer - cmd = ( - f"datadreamer --save_dir {target_folder} " - f"--class_names alien mars cat " - f"--prompts_number 1 " - f"--prompt_generator simple " - f"--num_objects_range 1 2 " - f"--image_generator sdxl " - f"--use_image_tester " - f"--device cuda" - ) - # Check the run of the pipeline - _check_detection_pipeline(cmd, target_folder) +# def test_cpu_simple_sdxl_detection_pipeline(): +# # Define target folder +# target_folder = "data/data-det-cpu-simple-sdxl/" +# # Define the command to run the datadreamer +# cmd = ( +# f"datadreamer --save_dir {target_folder} " +# f"--class_names alien mars cat " +# f"--prompts_number 1 " +# f"--prompt_generator simple " +# f"--num_objects_range 1 2 " +# f"--image_generator sdxl " +# f"--use_image_tester " +# f"--device cpu" +# ) +# # Check the run of the pipeline +# _check_detection_pipeline(cmd, target_folder) + + +# @pytest.mark.skipif(not torch.cuda.is_available(), reason="Test requires GPU") +# def test_cuda_simple_sdxl_detection_pipeline(): +# # Define target folder +# target_folder = "data/data-det-cuda-simple-sdxl/" +# # Define the command to run the datadreamer +# cmd = ( +# f"datadreamer --save_dir {target_folder} " +# f"--class_names alien mars cat " +# f"--prompts_number 1 " +# f"--prompt_generator simple " +# f"--num_objects_range 1 2 " +# f"--image_generator sdxl " +# f"--use_image_tester " +# f"--device cuda" +# ) +# # Check the run of the pipeline +# _check_detection_pipeline(cmd, target_folder) # ========================================================= # DETECTION - LLM # ========================================================= -@pytest.mark.skipif(total_memory < 28, reason="Test requires at least 28GB of RAM for running on CPU") -def test_cpu_lm_sdxl_turbo_detection_pipeline(): - # Define target folder - target_folder = "data/data-det-cpu-lm-sdxl-turbo/" - # Define the command to run the datadreamer - cmd = ( - f"datadreamer --save_dir {target_folder} " - f"--class_names alien mars cat " - f"--prompts_number 1 " - f"--prompt_generator lm " - f"--num_objects_range 1 2 " - f"--image_generator sdxl-turbo " - f"--use_image_tester " - f"--device cpu" - ) - # Check the run of the pipeline - _check_detection_pipeline(cmd, target_folder) - - -@pytest.mark.skipif(total_memory < 16 or not torch.cuda.is_available(), reason="Test requires at least 16GB of RAM and CUDA support") -def test_cuda_lm_sdxl_turbo_detection_pipeline(): - # Define target folder - target_folder = "data/data-det-cuda-lm-sdxl-turbo/" - # Define the command to run the datadreamer - cmd = ( - f"datadreamer --save_dir {target_folder} " - f"--class_names alien mars cat " - f"--prompts_number 1 " - f"--prompt_generator lm " - f"--num_objects_range 1 2 " - f"--image_generator sdxl-turbo " - f"--use_image_tester " - f"--device cuda" - ) - # Check the run of the pipeline - _check_detection_pipeline(cmd, target_folder) - - -@pytest.mark.skipif(total_memory < 28, reason="Test requires at least 28GB of RAM for running on CPU") -def test_cpu_lm_sdxl_detection_pipeline(): - # Define target folder - target_folder = "data/data-det-cpu-lm-sdxl/" - # Define the command to run the datadreamer - cmd = ( - f"datadreamer --save_dir {target_folder} " - f"--class_names alien mars cat " - f"--prompts_number 1 " - f"--prompt_generator lm " - f"--num_objects_range 1 2 " - f"--image_generator sdxl " - f"--use_image_tester " - f"--device cpu" - ) - # Check the run of the pipeline - _check_detection_pipeline(cmd, target_folder) - - -@pytest.mark.skipif(total_memory < 16 or not torch.cuda.is_available(), reason="Test requires at least 16GB of RAM and CUDA support") -def test_cuda_lm_sdxl_detection_pipeline(): - # Define target folder - target_folder = "data/data-det-cuda-lm-sdxl/" - # Define the command to run the datadreamer - cmd = ( - f"datadreamer --save_dir {target_folder} " - f"--class_names alien mars cat " - f"--prompts_number 1 " - f"--prompt_generator lm " - f"--num_objects_range 1 2 " - f"--image_generator sdxl " - f"--use_image_tester " - f"--device cuda" - ) - # Check the run of the pipeline - _check_detection_pipeline(cmd, target_folder) +# @pytest.mark.skipif(total_memory < 28, reason="Test requires at least 28GB of RAM for running on CPU") +# def test_cpu_lm_sdxl_turbo_detection_pipeline(): +# # Define target folder +# target_folder = "data/data-det-cpu-lm-sdxl-turbo/" +# # Define the command to run the datadreamer +# cmd = ( +# f"datadreamer --save_dir {target_folder} " +# f"--class_names alien mars cat " +# f"--prompts_number 1 " +# f"--prompt_generator lm " +# f"--num_objects_range 1 2 " +# f"--image_generator sdxl-turbo " +# f"--use_image_tester " +# f"--device cpu" +# ) +# # Check the run of the pipeline +# _check_detection_pipeline(cmd, target_folder) + + +# @pytest.mark.skipif(total_memory < 16 or not torch.cuda.is_available(), reason="Test requires at least 16GB of RAM and CUDA support") +# def test_cuda_lm_sdxl_turbo_detection_pipeline(): +# # Define target folder +# target_folder = "data/data-det-cuda-lm-sdxl-turbo/" +# # Define the command to run the datadreamer +# cmd = ( +# f"datadreamer --save_dir {target_folder} " +# f"--class_names alien mars cat " +# f"--prompts_number 1 " +# f"--prompt_generator lm " +# f"--num_objects_range 1 2 " +# f"--image_generator sdxl-turbo " +# f"--use_image_tester " +# f"--device cuda" +# ) +# # Check the run of the pipeline +# _check_detection_pipeline(cmd, target_folder) + + +# @pytest.mark.skipif(total_memory < 28, reason="Test requires at least 28GB of RAM for running on CPU") +# def test_cpu_lm_sdxl_detection_pipeline(): +# # Define target folder +# target_folder = "data/data-det-cpu-lm-sdxl/" +# # Define the command to run the datadreamer +# cmd = ( +# f"datadreamer --save_dir {target_folder} " +# f"--class_names alien mars cat " +# f"--prompts_number 1 " +# f"--prompt_generator lm " +# f"--num_objects_range 1 2 " +# f"--image_generator sdxl " +# f"--use_image_tester " +# f"--device cpu" +# ) +# # Check the run of the pipeline +# _check_detection_pipeline(cmd, target_folder) + + +# @pytest.mark.skipif(total_memory < 16 or not torch.cuda.is_available(), reason="Test requires at least 16GB of RAM and CUDA support") +# def test_cuda_lm_sdxl_detection_pipeline(): +# # Define target folder +# target_folder = "data/data-det-cuda-lm-sdxl/" +# # Define the command to run the datadreamer +# cmd = ( +# f"datadreamer --save_dir {target_folder} " +# f"--class_names alien mars cat " +# f"--prompts_number 1 " +# f"--prompt_generator lm " +# f"--num_objects_range 1 2 " +# f"--image_generator sdxl " +# f"--use_image_tester " +# f"--device cuda" +# ) +# # Check the run of the pipeline +# _check_detection_pipeline(cmd, target_folder) # ========================================================= @@ -351,7 +351,7 @@ def test_cpu_simple_sdxl_turbo_classification_pipeline(): f"--prompt_generator simple " f"--num_objects_range 1 2 " f"--image_generator sdxl-turbo " - f"--use_image_tester " + # f"--use_image_tester " f"--device cpu" ) # Check the run of the pipeline @@ -371,7 +371,7 @@ def test_cuda_simple_sdxl_turbo_classification_pipeline(): f"--prompt_generator simple " f"--num_objects_range 1 2 " f"--image_generator sdxl-turbo " - f"--use_image_tester " + # f"--use_image_tester " f"--device cuda" ) # Check the run of the pipeline @@ -379,110 +379,110 @@ def test_cuda_simple_sdxl_turbo_classification_pipeline(): -def test_cpu_simple_sdxl_classification_pipeline(): - # Define target folder - target_folder = "data/data-cls-cpu-simple-sdxl/" - # Define the command to run the datadreamer - cmd = ( - f"datadreamer --task classification " - f"--save_dir {target_folder} " - f"--class_names alien mars cat " - f"--prompts_number 1 " - f"--prompt_generator simple " - f"--num_objects_range 1 2 " - f"--image_generator sdxl " - f"--use_image_tester " - f"--device cpu" - ) - # Check the run of the pipeline - _check_detection_pipeline(cmd, target_folder) - - -@pytest.mark.skipif(not torch.cuda.is_available(), reason="Test requires GPU") -def test_cuda_simple_sdxl_classification_pipeline(): - # Define target folder - target_folder = "data/data-cls-cuda-simple-sdxl/" - # Define the command to run the datadreamer - cmd = ( - f"datadreamer --task classification " - f"--save_dir {target_folder} " - f"--class_names alien mars cat " - f"--prompts_number 1 " - f"--prompt_generator simple " - f"--num_objects_range 1 2 " - f"--image_generator sdxl " - f"--use_image_tester " - f"--device cuda" - ) - # Check the run of the pipeline - _check_detection_pipeline(cmd, target_folder) +# def test_cpu_simple_sdxl_classification_pipeline(): +# # Define target folder +# target_folder = "data/data-cls-cpu-simple-sdxl/" +# # Define the command to run the datadreamer +# cmd = ( +# f"datadreamer --task classification " +# f"--save_dir {target_folder} " +# f"--class_names alien mars cat " +# f"--prompts_number 1 " +# f"--prompt_generator simple " +# f"--num_objects_range 1 2 " +# f"--image_generator sdxl " +# f"--use_image_tester " +# f"--device cpu" +# ) +# # Check the run of the pipeline +# _check_detection_pipeline(cmd, target_folder) + + +# @pytest.mark.skipif(not torch.cuda.is_available(), reason="Test requires GPU") +# def test_cuda_simple_sdxl_classification_pipeline(): +# # Define target folder +# target_folder = "data/data-cls-cuda-simple-sdxl/" +# # Define the command to run the datadreamer +# cmd = ( +# f"datadreamer --task classification " +# f"--save_dir {target_folder} " +# f"--class_names alien mars cat " +# f"--prompts_number 1 " +# f"--prompt_generator simple " +# f"--num_objects_range 1 2 " +# f"--image_generator sdxl " +# f"--use_image_tester " +# f"--device cuda" +# ) +# # Check the run of the pipeline +# _check_detection_pipeline(cmd, target_folder) # ========================================================= # CLASSIFICATION - LLM # ========================================================= -@pytest.mark.skipif(total_memory < 28, reason="Test requires at least 28GB of RAM for running on CPU") -def test_cpu_lm_sdxl_turbo_classification_pipeline(): - # Define target folder - target_folder = "data/data-cls-cpu-lm-sdxl-turbo/" - # Define the command to run the datadreamer - cmd = ( - f"datadreamer --task classification " - f"--save_dir {target_folder} " - f"--class_names alien mars cat " - f"--prompts_number 1 " - f"--prompt_generator lm " - f"--num_objects_range 1 2 " - f"--image_generator sdxl-turbo " - f"--use_image_tester " - f"--device cpu" - ) - # Check the run of the pipeline - _check_detection_pipeline(cmd, target_folder) - - -@pytest.mark.skipif(total_memory < 16 or not torch.cuda.is_available(), reason="Test requires at least 16GB of RAM and CUDA support") -def test_cuda_lm_sdxl_turbo_classification_pipeline(): - # Define target folder - target_folder = "data/data-cls-cuda-lm-sdxl-turbo/" - # Define the command to run the datadreamer - cmd = ( - f"datadreamer --task classification " - f"--save_dir {target_folder} " - f"--class_names alien mars cat " - f"--prompts_number 1 " - f"--prompt_generator lm " - f"--num_objects_range 1 2 " - f"--image_generator sdxl-turbo " - f"--use_image_tester " - f"--device cuda" - ) - # Check the run of the pipeline - _check_detection_pipeline(cmd, target_folder) - - -@pytest.mark.skipif(total_memory < 28, reason="Test requires at least 28GB of RAM for running on CPU") -def test_cpu_lm_sdxl_classification_pipeline(): - # Define target folder - target_folder = "data/data-cls-cpu-lm-sdxl/" - # Define the command to run the datadreamer - cmd = ( - f"datadreamer --task classification " - f"--save_dir {target_folder} " - f"--class_names alien mars cat " - f"--prompts_number 1 " - f"--prompt_generator lm " - f"--num_objects_range 1 2 " - f"--image_generator sdxl " - f"--use_image_tester " - f"--device cpu" - ) - # Check the run of the pipeline - _check_detection_pipeline(cmd, target_folder) - - -@pytest.mark.skipif(total_memory < 16 or not torch.cuda.is_available(), reason="Test requires at least 16GB of RAM and CUDA support") -def test_cuda_lm_sdxl_classification_pipeline(): +# @pytest.mark.skipif(total_memory < 28, reason="Test requires at least 28GB of RAM for running on CPU") +# def test_cpu_lm_sdxl_turbo_classification_pipeline(): +# # Define target folder +# target_folder = "data/data-cls-cpu-lm-sdxl-turbo/" +# # Define the command to run the datadreamer +# cmd = ( +# f"datadreamer --task classification " +# f"--save_dir {target_folder} " +# f"--class_names alien mars cat " +# f"--prompts_number 1 " +# f"--prompt_generator lm " +# f"--num_objects_range 1 2 " +# f"--image_generator sdxl-turbo " +# f"--use_image_tester " +# f"--device cpu" +# ) +# # Check the run of the pipeline +# _check_detection_pipeline(cmd, target_folder) + + +# @pytest.mark.skipif(total_memory < 16 or not torch.cuda.is_available(), reason="Test requires at least 16GB of RAM and CUDA support") +# def test_cuda_lm_sdxl_turbo_classification_pipeline(): +# # Define target folder +# target_folder = "data/data-cls-cuda-lm-sdxl-turbo/" +# # Define the command to run the datadreamer +# cmd = ( +# f"datadreamer --task classification " +# f"--save_dir {target_folder} " +# f"--class_names alien mars cat " +# f"--prompts_number 1 " +# f"--prompt_generator lm " +# f"--num_objects_range 1 2 " +# f"--image_generator sdxl-turbo " +# f"--use_image_tester " +# f"--device cuda" +# ) +# # Check the run of the pipeline +# _check_detection_pipeline(cmd, target_folder) + + +# @pytest.mark.skipif(total_memory < 28, reason="Test requires at least 28GB of RAM for running on CPU") +# def test_cpu_lm_sdxl_classification_pipeline(): +# # Define target folder +# target_folder = "data/data-cls-cpu-lm-sdxl/" +# # Define the command to run the datadreamer +# cmd = ( +# f"datadreamer --task classification " +# f"--save_dir {target_folder} " +# f"--class_names alien mars cat " +# f"--prompts_number 1 " +# f"--prompt_generator lm " +# f"--num_objects_range 1 2 " +# f"--image_generator sdxl " +# f"--use_image_tester " +# f"--device cpu" +# ) +# # Check the run of the pipeline +# _check_detection_pipeline(cmd, target_folder) + + +# @pytest.mark.skipif(total_memory < 16 or not torch.cuda.is_available(), reason="Test requires at least 16GB of RAM and CUDA support") +# def test_cuda_lm_sdxl_classification_pipeline(): # Define target folder target_folder = "data/data-cls-cuda-lm-sdxl/" # Define the command to run the datadreamer diff --git a/tests/unittests/test_prompt_generation.py b/tests/unittests/test_prompt_generation.py index 4924408..277ada8 100644 --- a/tests/unittests/test_prompt_generation.py +++ b/tests/unittests/test_prompt_generation.py @@ -87,7 +87,7 @@ def _check_synonym_generator(device: str): @pytest.mark.skipif(total_memory < 16 or not torch.cuda.is_available(), reason="Test requires at least 16GB of RAM and CUDA support") def test_cuda_synonym_generator(): _check_synonym_generator("cuda") - + @pytest.mark.skipif(total_memory < 28, reason="Test requires at least 28GB of RAM for running on CPU") def test_cpu_synonym_generator(): From 856d32a71e01f12b06b2c70793b137d27e396d12 Mon Sep 17 00:00:00 2001 From: Jan Cuhel Date: Fri, 12 Jan 2024 14:07:29 +0100 Subject: [PATCH 13/24] Comment test --- tests/unittests/test_image_generation.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/unittests/test_image_generation.py b/tests/unittests/test_image_generation.py index e3cc9cf..4dc90b5 100644 --- a/tests/unittests/test_image_generation.py +++ b/tests/unittests/test_image_generation.py @@ -29,13 +29,13 @@ def _check_clip_image_tester(device: str): tester.release(empty_cuda_cache=True if device != "cpu" else False) -@pytest.mark.skipif(not torch.cuda.is_available(), reason="Test requires GPU") -def test_cuda_clip_image_tester(): - _check_clip_image_tester("cuda") +# @pytest.mark.skipif(not torch.cuda.is_available(), reason="Test requires GPU") +# def test_cuda_clip_image_tester(): +# _check_clip_image_tester("cuda") -def test_cpu_clip_image_tester(): - _check_clip_image_tester("cpu") +# def test_cpu_clip_image_tester(): +# _check_clip_image_tester("cpu") def _check_image_generator(image_generator_class: Type[Union[StableDiffusionImageGenerator, StableDiffusionTurboImageGenerator]], device: str): @@ -50,13 +50,13 @@ def _check_image_generator(image_generator_class: Type[Union[StableDiffusionImag image_generator.release(empty_cuda_cache=True if device != "cpu" else False) -@pytest.mark.skipif(not torch.cuda.is_available(), reason="Test requires GPU") -def test_cuda_sdxl_image_generator(): - _check_image_generator(StableDiffusionImageGenerator, "cuda") +# @pytest.mark.skipif(not torch.cuda.is_available(), reason="Test requires GPU") +# def test_cuda_sdxl_image_generator(): +# _check_image_generator(StableDiffusionImageGenerator, "cuda") -def test_cpu_sdxl_image_generator(): - _check_image_generator(StableDiffusionImageGenerator, "cpu") +# def test_cpu_sdxl_image_generator(): +# _check_image_generator(StableDiffusionImageGenerator, "cpu") @pytest.mark.skipif(not torch.cuda.is_available(), reason="Test requires GPU") From 52dd12796781c33cba5e3ce50a7fb2f26222947e Mon Sep 17 00:00:00 2001 From: Jan Cuhel Date: Fri, 12 Jan 2024 16:24:10 +0100 Subject: [PATCH 14/24] Update tests --- media/coverage_badge.svg | 21 ++ tests/integration/test_pipeline.py | 248 +++++++++++------------ tests/unittests/test_image_generation.py | 10 +- 3 files changed, 150 insertions(+), 129 deletions(-) create mode 100644 media/coverage_badge.svg diff --git a/media/coverage_badge.svg b/media/coverage_badge.svg new file mode 100644 index 0000000..12876e6 --- /dev/null +++ b/media/coverage_badge.svg @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + coverage + coverage + 78% + 78% + + diff --git a/tests/integration/test_pipeline.py b/tests/integration/test_pipeline.py index 58ed2e0..f236472 100644 --- a/tests/integration/test_pipeline.py +++ b/tests/integration/test_pipeline.py @@ -260,80 +260,80 @@ def test_cuda_simple_sdxl_turbo_detection_pipeline(): # ========================================================= # DETECTION - LLM # ========================================================= -# @pytest.mark.skipif(total_memory < 28, reason="Test requires at least 28GB of RAM for running on CPU") -# def test_cpu_lm_sdxl_turbo_detection_pipeline(): -# # Define target folder -# target_folder = "data/data-det-cpu-lm-sdxl-turbo/" -# # Define the command to run the datadreamer -# cmd = ( -# f"datadreamer --save_dir {target_folder} " -# f"--class_names alien mars cat " -# f"--prompts_number 1 " -# f"--prompt_generator lm " -# f"--num_objects_range 1 2 " -# f"--image_generator sdxl-turbo " -# f"--use_image_tester " -# f"--device cpu" -# ) -# # Check the run of the pipeline -# _check_detection_pipeline(cmd, target_folder) +@pytest.mark.skipif(total_memory < 28, reason="Test requires at least 28GB of RAM for running on CPU") +def test_cpu_lm_sdxl_turbo_detection_pipeline(): + # Define target folder + target_folder = "data/data-det-cpu-lm-sdxl-turbo/" + # Define the command to run the datadreamer + cmd = ( + f"datadreamer --save_dir {target_folder} " + f"--class_names alien mars cat " + f"--prompts_number 1 " + f"--prompt_generator lm " + f"--num_objects_range 1 2 " + f"--image_generator sdxl-turbo " + f"--use_image_tester " + f"--device cpu" + ) + # Check the run of the pipeline + _check_detection_pipeline(cmd, target_folder) -# @pytest.mark.skipif(total_memory < 16 or not torch.cuda.is_available(), reason="Test requires at least 16GB of RAM and CUDA support") -# def test_cuda_lm_sdxl_turbo_detection_pipeline(): -# # Define target folder -# target_folder = "data/data-det-cuda-lm-sdxl-turbo/" -# # Define the command to run the datadreamer -# cmd = ( -# f"datadreamer --save_dir {target_folder} " -# f"--class_names alien mars cat " -# f"--prompts_number 1 " -# f"--prompt_generator lm " -# f"--num_objects_range 1 2 " -# f"--image_generator sdxl-turbo " -# f"--use_image_tester " -# f"--device cuda" -# ) -# # Check the run of the pipeline -# _check_detection_pipeline(cmd, target_folder) +@pytest.mark.skipif(total_memory < 16 or not torch.cuda.is_available(), reason="Test requires at least 16GB of RAM and CUDA support") +def test_cuda_lm_sdxl_turbo_detection_pipeline(): + # Define target folder + target_folder = "data/data-det-cuda-lm-sdxl-turbo/" + # Define the command to run the datadreamer + cmd = ( + f"datadreamer --save_dir {target_folder} " + f"--class_names alien mars cat " + f"--prompts_number 1 " + f"--prompt_generator lm " + f"--num_objects_range 1 2 " + f"--image_generator sdxl-turbo " + f"--use_image_tester " + f"--device cuda" + ) + # Check the run of the pipeline + _check_detection_pipeline(cmd, target_folder) -# @pytest.mark.skipif(total_memory < 28, reason="Test requires at least 28GB of RAM for running on CPU") -# def test_cpu_lm_sdxl_detection_pipeline(): -# # Define target folder -# target_folder = "data/data-det-cpu-lm-sdxl/" -# # Define the command to run the datadreamer -# cmd = ( -# f"datadreamer --save_dir {target_folder} " -# f"--class_names alien mars cat " -# f"--prompts_number 1 " -# f"--prompt_generator lm " -# f"--num_objects_range 1 2 " -# f"--image_generator sdxl " -# f"--use_image_tester " -# f"--device cpu" -# ) -# # Check the run of the pipeline -# _check_detection_pipeline(cmd, target_folder) +@pytest.mark.skipif(total_memory < 28, reason="Test requires at least 28GB of RAM for running on CPU") +def test_cpu_lm_sdxl_detection_pipeline(): + # Define target folder + target_folder = "data/data-det-cpu-lm-sdxl/" + # Define the command to run the datadreamer + cmd = ( + f"datadreamer --save_dir {target_folder} " + f"--class_names alien mars cat " + f"--prompts_number 1 " + f"--prompt_generator lm " + f"--num_objects_range 1 2 " + f"--image_generator sdxl " + f"--use_image_tester " + f"--device cpu" + ) + # Check the run of the pipeline + _check_detection_pipeline(cmd, target_folder) -# @pytest.mark.skipif(total_memory < 16 or not torch.cuda.is_available(), reason="Test requires at least 16GB of RAM and CUDA support") -# def test_cuda_lm_sdxl_detection_pipeline(): -# # Define target folder -# target_folder = "data/data-det-cuda-lm-sdxl/" -# # Define the command to run the datadreamer -# cmd = ( -# f"datadreamer --save_dir {target_folder} " -# f"--class_names alien mars cat " -# f"--prompts_number 1 " -# f"--prompt_generator lm " -# f"--num_objects_range 1 2 " -# f"--image_generator sdxl " -# f"--use_image_tester " -# f"--device cuda" -# ) -# # Check the run of the pipeline -# _check_detection_pipeline(cmd, target_folder) +@pytest.mark.skipif(total_memory < 16 or not torch.cuda.is_available(), reason="Test requires at least 16GB of RAM and CUDA support") +def test_cuda_lm_sdxl_detection_pipeline(): + # Define target folder + target_folder = "data/data-det-cuda-lm-sdxl/" + # Define the command to run the datadreamer + cmd = ( + f"datadreamer --save_dir {target_folder} " + f"--class_names alien mars cat " + f"--prompts_number 1 " + f"--prompt_generator lm " + f"--num_objects_range 1 2 " + f"--image_generator sdxl " + f"--use_image_tester " + f"--device cuda" + ) + # Check the run of the pipeline + _check_detection_pipeline(cmd, target_folder) # ========================================================= @@ -421,68 +421,68 @@ def test_cuda_simple_sdxl_turbo_classification_pipeline(): # ========================================================= # CLASSIFICATION - LLM # ========================================================= -# @pytest.mark.skipif(total_memory < 28, reason="Test requires at least 28GB of RAM for running on CPU") -# def test_cpu_lm_sdxl_turbo_classification_pipeline(): -# # Define target folder -# target_folder = "data/data-cls-cpu-lm-sdxl-turbo/" -# # Define the command to run the datadreamer -# cmd = ( -# f"datadreamer --task classification " -# f"--save_dir {target_folder} " -# f"--class_names alien mars cat " -# f"--prompts_number 1 " -# f"--prompt_generator lm " -# f"--num_objects_range 1 2 " -# f"--image_generator sdxl-turbo " -# f"--use_image_tester " -# f"--device cpu" -# ) -# # Check the run of the pipeline -# _check_detection_pipeline(cmd, target_folder) +@pytest.mark.skipif(total_memory < 28, reason="Test requires at least 28GB of RAM for running on CPU") +def test_cpu_lm_sdxl_turbo_classification_pipeline(): + # Define target folder + target_folder = "data/data-cls-cpu-lm-sdxl-turbo/" + # Define the command to run the datadreamer + cmd = ( + f"datadreamer --task classification " + f"--save_dir {target_folder} " + f"--class_names alien mars cat " + f"--prompts_number 1 " + f"--prompt_generator lm " + f"--num_objects_range 1 2 " + f"--image_generator sdxl-turbo " + f"--use_image_tester " + f"--device cpu" + ) + # Check the run of the pipeline + _check_detection_pipeline(cmd, target_folder) -# @pytest.mark.skipif(total_memory < 16 or not torch.cuda.is_available(), reason="Test requires at least 16GB of RAM and CUDA support") -# def test_cuda_lm_sdxl_turbo_classification_pipeline(): -# # Define target folder -# target_folder = "data/data-cls-cuda-lm-sdxl-turbo/" -# # Define the command to run the datadreamer -# cmd = ( -# f"datadreamer --task classification " -# f"--save_dir {target_folder} " -# f"--class_names alien mars cat " -# f"--prompts_number 1 " -# f"--prompt_generator lm " -# f"--num_objects_range 1 2 " -# f"--image_generator sdxl-turbo " -# f"--use_image_tester " -# f"--device cuda" -# ) -# # Check the run of the pipeline -# _check_detection_pipeline(cmd, target_folder) +@pytest.mark.skipif(total_memory < 16 or not torch.cuda.is_available(), reason="Test requires at least 16GB of RAM and CUDA support") +def test_cuda_lm_sdxl_turbo_classification_pipeline(): + # Define target folder + target_folder = "data/data-cls-cuda-lm-sdxl-turbo/" + # Define the command to run the datadreamer + cmd = ( + f"datadreamer --task classification " + f"--save_dir {target_folder} " + f"--class_names alien mars cat " + f"--prompts_number 1 " + f"--prompt_generator lm " + f"--num_objects_range 1 2 " + f"--image_generator sdxl-turbo " + f"--use_image_tester " + f"--device cuda" + ) + # Check the run of the pipeline + _check_detection_pipeline(cmd, target_folder) -# @pytest.mark.skipif(total_memory < 28, reason="Test requires at least 28GB of RAM for running on CPU") -# def test_cpu_lm_sdxl_classification_pipeline(): -# # Define target folder -# target_folder = "data/data-cls-cpu-lm-sdxl/" -# # Define the command to run the datadreamer -# cmd = ( -# f"datadreamer --task classification " -# f"--save_dir {target_folder} " -# f"--class_names alien mars cat " -# f"--prompts_number 1 " -# f"--prompt_generator lm " -# f"--num_objects_range 1 2 " -# f"--image_generator sdxl " -# f"--use_image_tester " -# f"--device cpu" -# ) -# # Check the run of the pipeline -# _check_detection_pipeline(cmd, target_folder) +@pytest.mark.skipif(total_memory < 28, reason="Test requires at least 28GB of RAM for running on CPU") +def test_cpu_lm_sdxl_classification_pipeline(): + # Define target folder + target_folder = "data/data-cls-cpu-lm-sdxl/" + # Define the command to run the datadreamer + cmd = ( + f"datadreamer --task classification " + f"--save_dir {target_folder} " + f"--class_names alien mars cat " + f"--prompts_number 1 " + f"--prompt_generator lm " + f"--num_objects_range 1 2 " + f"--image_generator sdxl " + f"--use_image_tester " + f"--device cpu" + ) + # Check the run of the pipeline + _check_detection_pipeline(cmd, target_folder) -# @pytest.mark.skipif(total_memory < 16 or not torch.cuda.is_available(), reason="Test requires at least 16GB of RAM and CUDA support") -# def test_cuda_lm_sdxl_classification_pipeline(): +@pytest.mark.skipif(total_memory < 16 or not torch.cuda.is_available(), reason="Test requires at least 16GB of RAM and CUDA support") +def test_cuda_lm_sdxl_classification_pipeline(): # Define target folder target_folder = "data/data-cls-cuda-lm-sdxl/" # Define the command to run the datadreamer diff --git a/tests/unittests/test_image_generation.py b/tests/unittests/test_image_generation.py index 4dc90b5..8e044ad 100644 --- a/tests/unittests/test_image_generation.py +++ b/tests/unittests/test_image_generation.py @@ -29,13 +29,13 @@ def _check_clip_image_tester(device: str): tester.release(empty_cuda_cache=True if device != "cpu" else False) -# @pytest.mark.skipif(not torch.cuda.is_available(), reason="Test requires GPU") -# def test_cuda_clip_image_tester(): -# _check_clip_image_tester("cuda") +@pytest.mark.skipif(not torch.cuda.is_available(), reason="Test requires GPU") +def test_cuda_clip_image_tester(): + _check_clip_image_tester("cuda") -# def test_cpu_clip_image_tester(): -# _check_clip_image_tester("cpu") +def test_cpu_clip_image_tester(): + _check_clip_image_tester("cpu") def _check_image_generator(image_generator_class: Type[Union[StableDiffusionImageGenerator, StableDiffusionTurboImageGenerator]], device: str): From b937ac8578992c02f86144446d968a04359be3c9 Mon Sep 17 00:00:00 2001 From: Jan Cuhel Date: Fri, 12 Jan 2024 23:57:20 +0100 Subject: [PATCH 15/24] Update --- tests/integration/test_pipeline.py | 1 - tests/unittests/test_image_generation.py | 10 +++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/integration/test_pipeline.py b/tests/integration/test_pipeline.py index f236472..9ece194 100644 --- a/tests/integration/test_pipeline.py +++ b/tests/integration/test_pipeline.py @@ -378,7 +378,6 @@ def test_cuda_simple_sdxl_turbo_classification_pipeline(): _check_detection_pipeline(cmd, target_folder) - # def test_cpu_simple_sdxl_classification_pipeline(): # # Define target folder # target_folder = "data/data-cls-cpu-simple-sdxl/" diff --git a/tests/unittests/test_image_generation.py b/tests/unittests/test_image_generation.py index 8e044ad..e3cc9cf 100644 --- a/tests/unittests/test_image_generation.py +++ b/tests/unittests/test_image_generation.py @@ -50,13 +50,13 @@ def _check_image_generator(image_generator_class: Type[Union[StableDiffusionImag image_generator.release(empty_cuda_cache=True if device != "cpu" else False) -# @pytest.mark.skipif(not torch.cuda.is_available(), reason="Test requires GPU") -# def test_cuda_sdxl_image_generator(): -# _check_image_generator(StableDiffusionImageGenerator, "cuda") +@pytest.mark.skipif(not torch.cuda.is_available(), reason="Test requires GPU") +def test_cuda_sdxl_image_generator(): + _check_image_generator(StableDiffusionImageGenerator, "cuda") -# def test_cpu_sdxl_image_generator(): -# _check_image_generator(StableDiffusionImageGenerator, "cpu") +def test_cpu_sdxl_image_generator(): + _check_image_generator(StableDiffusionImageGenerator, "cpu") @pytest.mark.skipif(not torch.cuda.is_available(), reason="Test requires GPU") From e09e05406ec61cbc619cfb97c3b4a2a7baaf9c4f Mon Sep 17 00:00:00 2001 From: Jan Cuhel Date: Sat, 13 Jan 2024 09:31:06 +0100 Subject: [PATCH 16/24] Finalize tests --- .github/workflows/tests.yaml | 6 - tests/integration/test_pipeline.py | 210 +++++++++++++--------- tests/unittests/test_image_generation.py | 23 ++- tests/unittests/test_prompt_generation.py | 20 ++- 4 files changed, 158 insertions(+), 101 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 05e658d..a6e161c 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -1,12 +1,6 @@ name: Tests on: - push: - branches: [ add-tests ] - paths: - - 'src/datadreamer/**/**.py' - - 'tests/**/**.py' - - .github/workflows/tests.yaml pull_request: branches: [ dev, main ] paths: diff --git a/tests/integration/test_pipeline.py b/tests/integration/test_pipeline.py index 9ece194..787c226 100644 --- a/tests/integration/test_pipeline.py +++ b/tests/integration/test_pipeline.py @@ -6,7 +6,9 @@ # Get the total memory in GB -total_memory = psutil.virtual_memory().total / (1024 ** 3) +total_memory = psutil.virtual_memory().total / (1024**3) +# Get the total disk space in GB +total_disk_space = psutil.disk_usage("/").total / (1024**3) def _check_detection_pipeline(cmd: str, target_folder: str): @@ -23,9 +25,7 @@ def _check_detection_pipeline(cmd: str, target_folder: str): ] # Check that all the files were created for file in files: - assert os.path.isfile( - os.path.join(target_folder, file) - ), f"{file} not created" + assert os.path.isfile(os.path.join(target_folder, file)), f"{file} not created" # Check that the "bboxes_visualization" folder was created assert os.path.isdir( os.path.join(target_folder, "bboxes_visualization") @@ -194,7 +194,7 @@ def test_cpu_simple_sdxl_turbo_detection_pipeline(): f"--prompt_generator simple " f"--num_objects_range 1 2 " f"--image_generator sdxl-turbo " - # f"--use_image_tester " + f"--use_image_tester " f"--device cpu" ) # Check the run of the pipeline @@ -213,54 +213,63 @@ def test_cuda_simple_sdxl_turbo_detection_pipeline(): f"--prompt_generator simple " f"--num_objects_range 1 2 " f"--image_generator sdxl-turbo " - # f"--use_image_tester " + f"--use_image_tester " f"--device cuda" ) # Check the run of the pipeline _check_detection_pipeline(cmd, target_folder) -# def test_cpu_simple_sdxl_detection_pipeline(): -# # Define target folder -# target_folder = "data/data-det-cpu-simple-sdxl/" -# # Define the command to run the datadreamer -# cmd = ( -# f"datadreamer --save_dir {target_folder} " -# f"--class_names alien mars cat " -# f"--prompts_number 1 " -# f"--prompt_generator simple " -# f"--num_objects_range 1 2 " -# f"--image_generator sdxl " -# f"--use_image_tester " -# f"--device cpu" -# ) -# # Check the run of the pipeline -# _check_detection_pipeline(cmd, target_folder) - - -# @pytest.mark.skipif(not torch.cuda.is_available(), reason="Test requires GPU") -# def test_cuda_simple_sdxl_detection_pipeline(): -# # Define target folder -# target_folder = "data/data-det-cuda-simple-sdxl/" -# # Define the command to run the datadreamer -# cmd = ( -# f"datadreamer --save_dir {target_folder} " -# f"--class_names alien mars cat " -# f"--prompts_number 1 " -# f"--prompt_generator simple " -# f"--num_objects_range 1 2 " -# f"--image_generator sdxl " -# f"--use_image_tester " -# f"--device cuda" -# ) -# # Check the run of the pipeline -# _check_detection_pipeline(cmd, target_folder) +@pytest.mark.skipif( + total_memory < 16 or total_disk_space < 16, + reason="Test requires at least 16GB of RAM and 16GB of HDD", +) +def test_cpu_simple_sdxl_detection_pipeline(): + # Define target folder + target_folder = "data/data-det-cpu-simple-sdxl/" + # Define the command to run the datadreamer + cmd = ( + f"datadreamer --save_dir {target_folder} " + f"--class_names alien mars cat " + f"--prompts_number 1 " + f"--prompt_generator simple " + f"--num_objects_range 1 2 " + f"--image_generator sdxl " + f"--use_image_tester " + f"--device cpu" + ) + # Check the run of the pipeline + _check_detection_pipeline(cmd, target_folder) + + +@pytest.mark.skipif( + not torch.cuda.is_available() or total_memory < 16 or total_disk_space < 16, + reason="Test requires GPU, at least 16GB of RAM and 16GB of HDD", +) +def test_cuda_simple_sdxl_detection_pipeline(): + # Define target folder + target_folder = "data/data-det-cuda-simple-sdxl/" + # Define the command to run the datadreamer + cmd = ( + f"datadreamer --save_dir {target_folder} " + f"--class_names alien mars cat " + f"--prompts_number 1 " + f"--prompt_generator simple " + f"--num_objects_range 1 2 " + f"--image_generator sdxl " + f"--use_image_tester " + f"--device cuda" + ) + # Check the run of the pipeline + _check_detection_pipeline(cmd, target_folder) # ========================================================= # DETECTION - LLM # ========================================================= -@pytest.mark.skipif(total_memory < 28, reason="Test requires at least 28GB of RAM for running on CPU") +@pytest.mark.skipif( + total_memory < 28, reason="Test requires at least 28GB of RAM for running on CPU" +) def test_cpu_lm_sdxl_turbo_detection_pipeline(): # Define target folder target_folder = "data/data-det-cpu-lm-sdxl-turbo/" @@ -279,7 +288,10 @@ def test_cpu_lm_sdxl_turbo_detection_pipeline(): _check_detection_pipeline(cmd, target_folder) -@pytest.mark.skipif(total_memory < 16 or not torch.cuda.is_available(), reason="Test requires at least 16GB of RAM and CUDA support") +@pytest.mark.skipif( + total_memory < 16 or not torch.cuda.is_available(), + reason="Test requires at least 16GB of RAM and CUDA support", +) def test_cuda_lm_sdxl_turbo_detection_pipeline(): # Define target folder target_folder = "data/data-det-cuda-lm-sdxl-turbo/" @@ -298,7 +310,9 @@ def test_cuda_lm_sdxl_turbo_detection_pipeline(): _check_detection_pipeline(cmd, target_folder) -@pytest.mark.skipif(total_memory < 28, reason="Test requires at least 28GB of RAM for running on CPU") +@pytest.mark.skipif( + total_memory < 28, reason="Test requires at least 28GB of RAM for running on CPU" +) def test_cpu_lm_sdxl_detection_pipeline(): # Define target folder target_folder = "data/data-det-cpu-lm-sdxl/" @@ -317,7 +331,10 @@ def test_cpu_lm_sdxl_detection_pipeline(): _check_detection_pipeline(cmd, target_folder) -@pytest.mark.skipif(total_memory < 16 or not torch.cuda.is_available(), reason="Test requires at least 16GB of RAM and CUDA support") +@pytest.mark.skipif( + total_memory < 16 or not torch.cuda.is_available(), + reason="Test requires at least 16GB of RAM and CUDA support", +) def test_cuda_lm_sdxl_detection_pipeline(): # Define target folder target_folder = "data/data-det-cuda-lm-sdxl/" @@ -351,7 +368,7 @@ def test_cpu_simple_sdxl_turbo_classification_pipeline(): f"--prompt_generator simple " f"--num_objects_range 1 2 " f"--image_generator sdxl-turbo " - # f"--use_image_tester " + f"--use_image_tester " f"--device cpu" ) # Check the run of the pipeline @@ -371,56 +388,65 @@ def test_cuda_simple_sdxl_turbo_classification_pipeline(): f"--prompt_generator simple " f"--num_objects_range 1 2 " f"--image_generator sdxl-turbo " - # f"--use_image_tester " + f"--use_image_tester " f"--device cuda" ) # Check the run of the pipeline _check_detection_pipeline(cmd, target_folder) -# def test_cpu_simple_sdxl_classification_pipeline(): -# # Define target folder -# target_folder = "data/data-cls-cpu-simple-sdxl/" -# # Define the command to run the datadreamer -# cmd = ( -# f"datadreamer --task classification " -# f"--save_dir {target_folder} " -# f"--class_names alien mars cat " -# f"--prompts_number 1 " -# f"--prompt_generator simple " -# f"--num_objects_range 1 2 " -# f"--image_generator sdxl " -# f"--use_image_tester " -# f"--device cpu" -# ) -# # Check the run of the pipeline -# _check_detection_pipeline(cmd, target_folder) - - -# @pytest.mark.skipif(not torch.cuda.is_available(), reason="Test requires GPU") -# def test_cuda_simple_sdxl_classification_pipeline(): -# # Define target folder -# target_folder = "data/data-cls-cuda-simple-sdxl/" -# # Define the command to run the datadreamer -# cmd = ( -# f"datadreamer --task classification " -# f"--save_dir {target_folder} " -# f"--class_names alien mars cat " -# f"--prompts_number 1 " -# f"--prompt_generator simple " -# f"--num_objects_range 1 2 " -# f"--image_generator sdxl " -# f"--use_image_tester " -# f"--device cuda" -# ) -# # Check the run of the pipeline -# _check_detection_pipeline(cmd, target_folder) +@pytest.mark.skipif( + total_memory < 16 or total_disk_space < 16, + reason="Test requires at least 16GB of RAM and 16GB of HDD", +) +def test_cpu_simple_sdxl_classification_pipeline(): + # Define target folder + target_folder = "data/data-cls-cpu-simple-sdxl/" + # Define the command to run the datadreamer + cmd = ( + f"datadreamer --task classification " + f"--save_dir {target_folder} " + f"--class_names alien mars cat " + f"--prompts_number 1 " + f"--prompt_generator simple " + f"--num_objects_range 1 2 " + f"--image_generator sdxl " + f"--use_image_tester " + f"--device cpu" + ) + # Check the run of the pipeline + _check_detection_pipeline(cmd, target_folder) + + +@pytest.mark.skipif( + not torch.cuda.is_available() or total_memory < 16 or total_disk_space < 16, + reason="Test requires GPU, at least 16GB of RAM and 16GB of HDD", +) +def test_cuda_simple_sdxl_classification_pipeline(): + # Define target folder + target_folder = "data/data-cls-cuda-simple-sdxl/" + # Define the command to run the datadreamer + cmd = ( + f"datadreamer --task classification " + f"--save_dir {target_folder} " + f"--class_names alien mars cat " + f"--prompts_number 1 " + f"--prompt_generator simple " + f"--num_objects_range 1 2 " + f"--image_generator sdxl " + f"--use_image_tester " + f"--device cuda" + ) + # Check the run of the pipeline + _check_detection_pipeline(cmd, target_folder) # ========================================================= # CLASSIFICATION - LLM # ========================================================= -@pytest.mark.skipif(total_memory < 28, reason="Test requires at least 28GB of RAM for running on CPU") +@pytest.mark.skipif( + total_memory < 28, reason="Test requires at least 28GB of RAM for running on CPU" +) def test_cpu_lm_sdxl_turbo_classification_pipeline(): # Define target folder target_folder = "data/data-cls-cpu-lm-sdxl-turbo/" @@ -440,7 +466,10 @@ def test_cpu_lm_sdxl_turbo_classification_pipeline(): _check_detection_pipeline(cmd, target_folder) -@pytest.mark.skipif(total_memory < 16 or not torch.cuda.is_available(), reason="Test requires at least 16GB of RAM and CUDA support") +@pytest.mark.skipif( + total_memory < 16 or not torch.cuda.is_available(), + reason="Test requires at least 16GB of RAM and CUDA support", +) def test_cuda_lm_sdxl_turbo_classification_pipeline(): # Define target folder target_folder = "data/data-cls-cuda-lm-sdxl-turbo/" @@ -460,7 +489,9 @@ def test_cuda_lm_sdxl_turbo_classification_pipeline(): _check_detection_pipeline(cmd, target_folder) -@pytest.mark.skipif(total_memory < 28, reason="Test requires at least 28GB of RAM for running on CPU") +@pytest.mark.skipif( + total_memory < 28, reason="Test requires at least 28GB of RAM for running on CPU" +) def test_cpu_lm_sdxl_classification_pipeline(): # Define target folder target_folder = "data/data-cls-cpu-lm-sdxl/" @@ -480,7 +511,10 @@ def test_cpu_lm_sdxl_classification_pipeline(): _check_detection_pipeline(cmd, target_folder) -@pytest.mark.skipif(total_memory < 16 or not torch.cuda.is_available(), reason="Test requires at least 16GB of RAM and CUDA support") +@pytest.mark.skipif( + total_memory < 16 or not torch.cuda.is_available(), + reason="Test requires at least 16GB of RAM and CUDA support", +) def test_cuda_lm_sdxl_classification_pipeline(): # Define target folder target_folder = "data/data-cls-cuda-lm-sdxl/" diff --git a/tests/unittests/test_image_generation.py b/tests/unittests/test_image_generation.py index e3cc9cf..db11d86 100644 --- a/tests/unittests/test_image_generation.py +++ b/tests/unittests/test_image_generation.py @@ -2,6 +2,7 @@ import torch from PIL import Image import requests +import psutil from typing import Union, Type from datadreamer.image_generation.sdxl_image_generator import ( StableDiffusionImageGenerator, @@ -12,6 +13,12 @@ from datadreamer.image_generation.clip_image_tester import ClipImageTester +# Get the total memory in GB +total_memory = psutil.virtual_memory().total / (1024**3) +# Get the total disk space in GB +total_disk_space = psutil.disk_usage("/").total / (1024**3) + + def _check_clip_image_tester(device: str): url = "https://ultralytics.com/images/bus.jpg" im = Image.open(requests.get(url, stream=True).raw) @@ -38,7 +45,12 @@ def test_cpu_clip_image_tester(): _check_clip_image_tester("cpu") -def _check_image_generator(image_generator_class: Type[Union[StableDiffusionImageGenerator, StableDiffusionTurboImageGenerator]], device: str): +def _check_image_generator( + image_generator_class: Type[ + Union[StableDiffusionImageGenerator, StableDiffusionTurboImageGenerator] + ], + device: str, +): image_generator = image_generator_class(device=device) # Generate images and check each of them for generated_image in image_generator.generate_images( @@ -59,10 +71,17 @@ def test_cpu_sdxl_image_generator(): _check_image_generator(StableDiffusionImageGenerator, "cpu") -@pytest.mark.skipif(not torch.cuda.is_available(), reason="Test requires GPU") +@pytest.mark.skipif( + not torch.cuda.is_available() or total_memory < 16 or total_disk_space < 16, + reason="Test requires GPU, at least 16GB of RAM and 16GB of HDD", +) def test_cuda_sdxl_turbo_image_generator(): _check_image_generator(StableDiffusionTurboImageGenerator, "cuda") +@pytest.mark.skipif( + total_memory < 16 or total_disk_space < 16, + reason="Test requires at least 16GB of RAM and 16GB of HDD", +) def test_cpu_sdxl_turbo_image_generator(): _check_image_generator(StableDiffusionTurboImageGenerator, "cpu") diff --git a/tests/unittests/test_prompt_generation.py b/tests/unittests/test_prompt_generation.py index 277ada8..78dcd72 100644 --- a/tests/unittests/test_prompt_generation.py +++ b/tests/unittests/test_prompt_generation.py @@ -8,7 +8,7 @@ # Get the total memory in GB -total_memory = psutil.virtual_memory().total / (1024 ** 3) +total_memory = psutil.virtual_memory().total / (1024**3) def test_simple_prompt_generator(): @@ -56,12 +56,17 @@ def _check_lm_prompt_generator(device: str): prompt_generator.release(empty_cuda_cache=True if device != "cpu" else False) -@pytest.mark.skipif(total_memory < 16 or not torch.cuda.is_available(), reason="Test requires at least 16GB of RAM and CUDA support") +@pytest.mark.skipif( + total_memory < 16 or not torch.cuda.is_available(), + reason="Test requires at least 16GB of RAM and CUDA support", +) def test_cuda_lm_prompt_generator(): _check_lm_prompt_generator("cuda") -@pytest.mark.skipif(total_memory < 28, reason="Test requires at least 28GB of RAM for running on CPU") +@pytest.mark.skipif( + total_memory < 28, reason="Test requires at least 28GB of RAM for running on CPU" +) def test_cpu_lm_prompt_generator(): _check_lm_prompt_generator("cpu") @@ -84,11 +89,16 @@ def _check_synonym_generator(device: str): generator.release(empty_cuda_cache=True if device != "cpu" else False) -@pytest.mark.skipif(total_memory < 16 or not torch.cuda.is_available(), reason="Test requires at least 16GB of RAM and CUDA support") +@pytest.mark.skipif( + total_memory < 16 or not torch.cuda.is_available(), + reason="Test requires at least 16GB of RAM and CUDA support", +) def test_cuda_synonym_generator(): _check_synonym_generator("cuda") -@pytest.mark.skipif(total_memory < 28, reason="Test requires at least 28GB of RAM for running on CPU") +@pytest.mark.skipif( + total_memory < 28, reason="Test requires at least 28GB of RAM for running on CPU" +) def test_cpu_synonym_generator(): _check_synonym_generator("cpu") From 41e12114a8863b57d68331c645870baefaadf5c8 Mon Sep 17 00:00:00 2001 From: Jan Cuhel Date: Sat, 13 Jan 2024 09:38:03 +0100 Subject: [PATCH 17/24] Update RAM min for SDXL --- tests/integration/test_pipeline.py | 16 ++++++++-------- tests/unittests/test_image_generation.py | 8 ++++---- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/integration/test_pipeline.py b/tests/integration/test_pipeline.py index 787c226..52f555b 100644 --- a/tests/integration/test_pipeline.py +++ b/tests/integration/test_pipeline.py @@ -221,8 +221,8 @@ def test_cuda_simple_sdxl_turbo_detection_pipeline(): @pytest.mark.skipif( - total_memory < 16 or total_disk_space < 16, - reason="Test requires at least 16GB of RAM and 16GB of HDD", + total_memory < 12 or total_disk_space < 16, + reason="Test requires at least 12GB of RAM and 16GB of HDD", ) def test_cpu_simple_sdxl_detection_pipeline(): # Define target folder @@ -243,8 +243,8 @@ def test_cpu_simple_sdxl_detection_pipeline(): @pytest.mark.skipif( - not torch.cuda.is_available() or total_memory < 16 or total_disk_space < 16, - reason="Test requires GPU, at least 16GB of RAM and 16GB of HDD", + not torch.cuda.is_available() or total_memory < 12 or total_disk_space < 16, + reason="Test requires GPU, at least 12GB of RAM and 16GB of HDD", ) def test_cuda_simple_sdxl_detection_pipeline(): # Define target folder @@ -396,8 +396,8 @@ def test_cuda_simple_sdxl_turbo_classification_pipeline(): @pytest.mark.skipif( - total_memory < 16 or total_disk_space < 16, - reason="Test requires at least 16GB of RAM and 16GB of HDD", + total_memory < 12 or total_disk_space < 16, + reason="Test requires at least 12GB of RAM and 16GB of HDD", ) def test_cpu_simple_sdxl_classification_pipeline(): # Define target folder @@ -419,8 +419,8 @@ def test_cpu_simple_sdxl_classification_pipeline(): @pytest.mark.skipif( - not torch.cuda.is_available() or total_memory < 16 or total_disk_space < 16, - reason="Test requires GPU, at least 16GB of RAM and 16GB of HDD", + not torch.cuda.is_available() or total_memory < 12 or total_disk_space < 16, + reason="Test requires GPU, at least 12GB of RAM and 16GB of HDD", ) def test_cuda_simple_sdxl_classification_pipeline(): # Define target folder diff --git a/tests/unittests/test_image_generation.py b/tests/unittests/test_image_generation.py index db11d86..ef5babc 100644 --- a/tests/unittests/test_image_generation.py +++ b/tests/unittests/test_image_generation.py @@ -72,16 +72,16 @@ def test_cpu_sdxl_image_generator(): @pytest.mark.skipif( - not torch.cuda.is_available() or total_memory < 16 or total_disk_space < 16, - reason="Test requires GPU, at least 16GB of RAM and 16GB of HDD", + not torch.cuda.is_available() or total_memory < 12 or total_disk_space < 16, + reason="Test requires GPU,at least 12GB of RAM and 16GB of HDD", ) def test_cuda_sdxl_turbo_image_generator(): _check_image_generator(StableDiffusionTurboImageGenerator, "cuda") @pytest.mark.skipif( - total_memory < 16 or total_disk_space < 16, - reason="Test requires at least 16GB of RAM and 16GB of HDD", + total_memory < 12 or total_disk_space < 16, + reason="Test requiresat least 12GB of RAM and 16GB of HDD", ) def test_cpu_sdxl_turbo_image_generator(): _check_image_generator(StableDiffusionTurboImageGenerator, "cpu") From 7327cbf0422a88071e42542cbb9da8de9e6f3622 Mon Sep 17 00:00:00 2001 From: Jan Cuhel Date: Sat, 13 Jan 2024 09:50:32 +0100 Subject: [PATCH 18/24] Finalize tests --- tests/integration/test_pipeline.py | 16 ++++++++-------- tests/unittests/test_image_generation.py | 8 ++++---- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/integration/test_pipeline.py b/tests/integration/test_pipeline.py index 52f555b..787c226 100644 --- a/tests/integration/test_pipeline.py +++ b/tests/integration/test_pipeline.py @@ -221,8 +221,8 @@ def test_cuda_simple_sdxl_turbo_detection_pipeline(): @pytest.mark.skipif( - total_memory < 12 or total_disk_space < 16, - reason="Test requires at least 12GB of RAM and 16GB of HDD", + total_memory < 16 or total_disk_space < 16, + reason="Test requires at least 16GB of RAM and 16GB of HDD", ) def test_cpu_simple_sdxl_detection_pipeline(): # Define target folder @@ -243,8 +243,8 @@ def test_cpu_simple_sdxl_detection_pipeline(): @pytest.mark.skipif( - not torch.cuda.is_available() or total_memory < 12 or total_disk_space < 16, - reason="Test requires GPU, at least 12GB of RAM and 16GB of HDD", + not torch.cuda.is_available() or total_memory < 16 or total_disk_space < 16, + reason="Test requires GPU, at least 16GB of RAM and 16GB of HDD", ) def test_cuda_simple_sdxl_detection_pipeline(): # Define target folder @@ -396,8 +396,8 @@ def test_cuda_simple_sdxl_turbo_classification_pipeline(): @pytest.mark.skipif( - total_memory < 12 or total_disk_space < 16, - reason="Test requires at least 12GB of RAM and 16GB of HDD", + total_memory < 16 or total_disk_space < 16, + reason="Test requires at least 16GB of RAM and 16GB of HDD", ) def test_cpu_simple_sdxl_classification_pipeline(): # Define target folder @@ -419,8 +419,8 @@ def test_cpu_simple_sdxl_classification_pipeline(): @pytest.mark.skipif( - not torch.cuda.is_available() or total_memory < 12 or total_disk_space < 16, - reason="Test requires GPU, at least 12GB of RAM and 16GB of HDD", + not torch.cuda.is_available() or total_memory < 16 or total_disk_space < 16, + reason="Test requires GPU, at least 16GB of RAM and 16GB of HDD", ) def test_cuda_simple_sdxl_classification_pipeline(): # Define target folder diff --git a/tests/unittests/test_image_generation.py b/tests/unittests/test_image_generation.py index ef5babc..db11d86 100644 --- a/tests/unittests/test_image_generation.py +++ b/tests/unittests/test_image_generation.py @@ -72,16 +72,16 @@ def test_cpu_sdxl_image_generator(): @pytest.mark.skipif( - not torch.cuda.is_available() or total_memory < 12 or total_disk_space < 16, - reason="Test requires GPU,at least 12GB of RAM and 16GB of HDD", + not torch.cuda.is_available() or total_memory < 16 or total_disk_space < 16, + reason="Test requires GPU, at least 16GB of RAM and 16GB of HDD", ) def test_cuda_sdxl_turbo_image_generator(): _check_image_generator(StableDiffusionTurboImageGenerator, "cuda") @pytest.mark.skipif( - total_memory < 12 or total_disk_space < 16, - reason="Test requiresat least 12GB of RAM and 16GB of HDD", + total_memory < 16 or total_disk_space < 16, + reason="Test requires at least 16GB of RAM and 16GB of HDD", ) def test_cpu_sdxl_turbo_image_generator(): _check_image_generator(StableDiffusionTurboImageGenerator, "cpu") From eafd4c3241d50ada6a0e2bd37c16a468bfb559b0 Mon Sep 17 00:00:00 2001 From: Jan Cuhel Date: Sat, 13 Jan 2024 10:02:25 +0100 Subject: [PATCH 19/24] Update pipeline tests --- tests/integration/test_pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_pipeline.py b/tests/integration/test_pipeline.py index 787c226..32d399a 100644 --- a/tests/integration/test_pipeline.py +++ b/tests/integration/test_pipeline.py @@ -42,7 +42,7 @@ def _check_wrong_value(cmd: str): try: subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: - raise ValueError(e.output.decode()) + print(f"Failed to run cmd: {cmd}\nError: {e.output.decode()}") # ========================================================= From 86c74a9336c81f23fc06a89ca44401f9dbec4086 Mon Sep 17 00:00:00 2001 From: Jan Cuhel Date: Sat, 13 Jan 2024 10:07:07 +0100 Subject: [PATCH 20/24] Format the code --- tests/integration/test_pipeline.py | 6 +++--- tests/unittests/test_annotators.py | 4 ++-- tests/unittests/test_image_generation.py | 12 ++++++------ tests/unittests/test_prompt_generation.py | 8 +++----- 4 files changed, 14 insertions(+), 16 deletions(-) diff --git a/tests/integration/test_pipeline.py b/tests/integration/test_pipeline.py index 32d399a..f73e644 100644 --- a/tests/integration/test_pipeline.py +++ b/tests/integration/test_pipeline.py @@ -1,9 +1,9 @@ -import pytest -import torch import os import subprocess -import psutil +import psutil +import pytest +import torch # Get the total memory in GB total_memory = psutil.virtual_memory().total / (1024**3) diff --git a/tests/unittests/test_annotators.py b/tests/unittests/test_annotators.py index cdbfdd1..7217236 100644 --- a/tests/unittests/test_annotators.py +++ b/tests/unittests/test_annotators.py @@ -1,8 +1,8 @@ import pytest -import torch -from PIL import Image import requests +import torch from datadreamer.dataset_annotation.owlv2_annotator import OWLv2Annotator +from PIL import Image def _check_owlv2_annotator(device: str): diff --git a/tests/unittests/test_image_generation.py b/tests/unittests/test_image_generation.py index db11d86..ba977ff 100644 --- a/tests/unittests/test_image_generation.py +++ b/tests/unittests/test_image_generation.py @@ -1,17 +1,17 @@ +from typing import Type, Union + +import psutil import pytest -import torch -from PIL import Image import requests -import psutil -from typing import Union, Type +import torch +from datadreamer.image_generation.clip_image_tester import ClipImageTester from datadreamer.image_generation.sdxl_image_generator import ( StableDiffusionImageGenerator, ) from datadreamer.image_generation.sdxl_turbo_image_generator import ( StableDiffusionTurboImageGenerator, ) -from datadreamer.image_generation.clip_image_tester import ClipImageTester - +from PIL import Image # Get the total memory in GB total_memory = psutil.virtual_memory().total / (1024**3) diff --git a/tests/unittests/test_prompt_generation.py b/tests/unittests/test_prompt_generation.py index 78dcd72..309f373 100644 --- a/tests/unittests/test_prompt_generation.py +++ b/tests/unittests/test_prompt_generation.py @@ -1,12 +1,10 @@ -from datadreamer.prompt_generation.lm_prompt_generator import LMPromptGenerator -from datadreamer.prompt_generation.simple_prompt_generator import SimplePromptGenerator +import psutil import pytest import torch -import psutil - +from datadreamer.prompt_generation.lm_prompt_generator import LMPromptGenerator +from datadreamer.prompt_generation.simple_prompt_generator import SimplePromptGenerator from datadreamer.prompt_generation.synonym_generator import SynonymGenerator - # Get the total memory in GB total_memory = psutil.virtual_memory().total / (1024**3) From e66bfd49b9b531e1fb110e406098c6a1ce611268 Mon Sep 17 00:00:00 2001 From: Jan Cuhel Date: Sat, 13 Jan 2024 13:59:55 +0100 Subject: [PATCH 21/24] Update HDD requirements --- tests/integration/test_pipeline.py | 30 ++++++++++++------------ tests/unittests/test_image_generation.py | 8 +++---- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/tests/integration/test_pipeline.py b/tests/integration/test_pipeline.py index f73e644..20cc7b4 100644 --- a/tests/integration/test_pipeline.py +++ b/tests/integration/test_pipeline.py @@ -42,7 +42,7 @@ def _check_wrong_value(cmd: str): try: subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: - print(f"Failed to run cmd: {cmd}\nError: {e.output.decode()}") + raise ValueError(e.output.decode()) from e # ========================================================= @@ -221,8 +221,8 @@ def test_cuda_simple_sdxl_turbo_detection_pipeline(): @pytest.mark.skipif( - total_memory < 16 or total_disk_space < 16, - reason="Test requires at least 16GB of RAM and 16GB of HDD", + total_memory < 16 or total_disk_space < 25, + reason="Test requires at least 16GB of RAM and 25GB of HDD", ) def test_cpu_simple_sdxl_detection_pipeline(): # Define target folder @@ -243,8 +243,8 @@ def test_cpu_simple_sdxl_detection_pipeline(): @pytest.mark.skipif( - not torch.cuda.is_available() or total_memory < 16 or total_disk_space < 16, - reason="Test requires GPU, at least 16GB of RAM and 16GB of HDD", + not torch.cuda.is_available() or total_memory < 16 or total_disk_space < 25, + reason="Test requires GPU, at least 16GB of RAM and 25GB of HDD", ) def test_cuda_simple_sdxl_detection_pipeline(): # Define target folder @@ -311,7 +311,7 @@ def test_cuda_lm_sdxl_turbo_detection_pipeline(): @pytest.mark.skipif( - total_memory < 28, reason="Test requires at least 28GB of RAM for running on CPU" + total_memory < 28 or total_disk_space < 25, reason="Test requires at least 28GB of RAM and 25GB of HDD for running on CPU" ) def test_cpu_lm_sdxl_detection_pipeline(): # Define target folder @@ -332,8 +332,8 @@ def test_cpu_lm_sdxl_detection_pipeline(): @pytest.mark.skipif( - total_memory < 16 or not torch.cuda.is_available(), - reason="Test requires at least 16GB of RAM and CUDA support", + total_memory < 16 or not torch.cuda.is_available() or total_disk_space < 25, + reason="Test requires at least 16GB of RAM, CUDA support and 25GB of HDD", ) def test_cuda_lm_sdxl_detection_pipeline(): # Define target folder @@ -396,8 +396,8 @@ def test_cuda_simple_sdxl_turbo_classification_pipeline(): @pytest.mark.skipif( - total_memory < 16 or total_disk_space < 16, - reason="Test requires at least 16GB of RAM and 16GB of HDD", + total_memory < 16 or total_disk_space < 25, + reason="Test requires at least 16GB of RAM and 25GB of HDD", ) def test_cpu_simple_sdxl_classification_pipeline(): # Define target folder @@ -419,8 +419,8 @@ def test_cpu_simple_sdxl_classification_pipeline(): @pytest.mark.skipif( - not torch.cuda.is_available() or total_memory < 16 or total_disk_space < 16, - reason="Test requires GPU, at least 16GB of RAM and 16GB of HDD", + not torch.cuda.is_available() or total_memory < 16 or total_disk_space < 25, + reason="Test requires GPU, at least 16GB of RAM and 25GB of HDD", ) def test_cuda_simple_sdxl_classification_pipeline(): # Define target folder @@ -490,7 +490,7 @@ def test_cuda_lm_sdxl_turbo_classification_pipeline(): @pytest.mark.skipif( - total_memory < 28, reason="Test requires at least 28GB of RAM for running on CPU" + total_memory < 28 or total_disk_space < 25, reason="Test requires at least 28GB of RAM and 25GB of HDD for running on CPU" ) def test_cpu_lm_sdxl_classification_pipeline(): # Define target folder @@ -512,8 +512,8 @@ def test_cpu_lm_sdxl_classification_pipeline(): @pytest.mark.skipif( - total_memory < 16 or not torch.cuda.is_available(), - reason="Test requires at least 16GB of RAM and CUDA support", + total_memory < 16 or not torch.cuda.is_available() or total_disk_space < 25, + reason="Test requires at least 16GB of RAM, CUDA support and 25GB of HDD", ) def test_cuda_lm_sdxl_classification_pipeline(): # Define target folder diff --git a/tests/unittests/test_image_generation.py b/tests/unittests/test_image_generation.py index ba977ff..636f149 100644 --- a/tests/unittests/test_image_generation.py +++ b/tests/unittests/test_image_generation.py @@ -72,16 +72,16 @@ def test_cpu_sdxl_image_generator(): @pytest.mark.skipif( - not torch.cuda.is_available() or total_memory < 16 or total_disk_space < 16, - reason="Test requires GPU, at least 16GB of RAM and 16GB of HDD", + not torch.cuda.is_available() or total_memory < 16 or total_disk_space < 25, + reason="Test requires GPU, at least 16GB of RAM and 25GB of HDD", ) def test_cuda_sdxl_turbo_image_generator(): _check_image_generator(StableDiffusionTurboImageGenerator, "cuda") @pytest.mark.skipif( - total_memory < 16 or total_disk_space < 16, - reason="Test requires at least 16GB of RAM and 16GB of HDD", + total_memory < 16 or total_disk_space < 25, + reason="Test requires at least 16GB of RAM and 25GB of HDD", ) def test_cpu_sdxl_turbo_image_generator(): _check_image_generator(StableDiffusionTurboImageGenerator, "cpu") From 58229c60cd541f3d498a35dd341d1dcdf2ce6da6 Mon Sep 17 00:00:00 2001 From: Jan Cuhel Date: Sat, 13 Jan 2024 15:09:24 +0100 Subject: [PATCH 22/24] Update test skipping if not sufficient HW --- tests/integration/test_pipeline.py | 46 ++++++++++++++++------- tests/unittests/test_image_generation.py | 9 ++++- tests/unittests/test_prompt_generation.py | 16 +++++--- 3 files changed, 50 insertions(+), 21 deletions(-) diff --git a/tests/integration/test_pipeline.py b/tests/integration/test_pipeline.py index 20cc7b4..59a6446 100644 --- a/tests/integration/test_pipeline.py +++ b/tests/integration/test_pipeline.py @@ -183,6 +183,10 @@ def test_negative_num_objects_range(): # ========================================================= # DETECTION - SIMPLE LM # ========================================================= +@pytest.mark.skipif( + total_disk_space < 25, + reason="Test requires at least 25GB of HDD", +) def test_cpu_simple_sdxl_turbo_detection_pipeline(): # Define target folder target_folder = "data/data-det-cpu-simple-sdxl-turbo/" @@ -201,7 +205,10 @@ def test_cpu_simple_sdxl_turbo_detection_pipeline(): _check_detection_pipeline(cmd, target_folder) -@pytest.mark.skipif(not torch.cuda.is_available(), reason="Test requires GPU") +@pytest.mark.skipif( + not torch.cuda.is_available() or total_disk_space < 25, + reason="Test requires GPU and 25GB of HDD", +) def test_cuda_simple_sdxl_turbo_detection_pipeline(): # Define target folder target_folder = "data/data-det-cuda-simple-sdxl-turbo/" @@ -268,7 +275,8 @@ def test_cuda_simple_sdxl_detection_pipeline(): # DETECTION - LLM # ========================================================= @pytest.mark.skipif( - total_memory < 28, reason="Test requires at least 28GB of RAM for running on CPU" + total_memory < 28 or total_disk_space < 55, + reason="Test requires at least 28GB of RAM and 55GB of HDD for running on CPU", ) def test_cpu_lm_sdxl_turbo_detection_pipeline(): # Define target folder @@ -289,8 +297,8 @@ def test_cpu_lm_sdxl_turbo_detection_pipeline(): @pytest.mark.skipif( - total_memory < 16 or not torch.cuda.is_available(), - reason="Test requires at least 16GB of RAM and CUDA support", + total_memory < 16 or not torch.cuda.is_available() or total_disk_space < 55, + reason="Test requires at least 16GB of RAM, CUDA support and 55GB of HDD", ) def test_cuda_lm_sdxl_turbo_detection_pipeline(): # Define target folder @@ -311,7 +319,8 @@ def test_cuda_lm_sdxl_turbo_detection_pipeline(): @pytest.mark.skipif( - total_memory < 28 or total_disk_space < 25, reason="Test requires at least 28GB of RAM and 25GB of HDD for running on CPU" + total_memory < 28 or total_disk_space < 55, + reason="Test requires at least 28GB of RAM and 55GB of HDD for running on CPU", ) def test_cpu_lm_sdxl_detection_pipeline(): # Define target folder @@ -332,8 +341,8 @@ def test_cpu_lm_sdxl_detection_pipeline(): @pytest.mark.skipif( - total_memory < 16 or not torch.cuda.is_available() or total_disk_space < 25, - reason="Test requires at least 16GB of RAM, CUDA support and 25GB of HDD", + total_memory < 16 or not torch.cuda.is_available() or total_disk_space < 55, + reason="Test requires at least 16GB of RAM, CUDA support and 55GB of HDD", ) def test_cuda_lm_sdxl_detection_pipeline(): # Define target folder @@ -356,6 +365,10 @@ def test_cuda_lm_sdxl_detection_pipeline(): # ========================================================= # CLASSIFICATION - SIMPLE LM # ========================================================= +@pytest.mark.skipif( + total_disk_space < 25, + reason="Test requires at least 25GB of HDD", +) def test_cpu_simple_sdxl_turbo_classification_pipeline(): # Define target folder target_folder = "data/data-cls-cpu-simple-sdxl-turbo/" @@ -375,7 +388,10 @@ def test_cpu_simple_sdxl_turbo_classification_pipeline(): _check_detection_pipeline(cmd, target_folder) -@pytest.mark.skipif(not torch.cuda.is_available(), reason="Test requires GPU") +@pytest.mark.skipif( + not torch.cuda.is_available() or total_disk_space < 25, + reason="Test requires GPU and 25GB of HDD", +) def test_cuda_simple_sdxl_turbo_classification_pipeline(): # Define target folder target_folder = "data/data-cls-cuda-simple-sdxl-turbo/" @@ -445,7 +461,8 @@ def test_cuda_simple_sdxl_classification_pipeline(): # CLASSIFICATION - LLM # ========================================================= @pytest.mark.skipif( - total_memory < 28, reason="Test requires at least 28GB of RAM for running on CPU" + total_memory < 28 or total_disk_space < 55, + reason="Test requires at least 28GB of RAM and 55GB of HDD for running on CPU", ) def test_cpu_lm_sdxl_turbo_classification_pipeline(): # Define target folder @@ -467,8 +484,8 @@ def test_cpu_lm_sdxl_turbo_classification_pipeline(): @pytest.mark.skipif( - total_memory < 16 or not torch.cuda.is_available(), - reason="Test requires at least 16GB of RAM and CUDA support", + total_memory < 16 or not torch.cuda.is_available() or total_disk_space < 55, + reason="Test requires at least 16GB of RAM, 55GB of HDD and CUDA support", ) def test_cuda_lm_sdxl_turbo_classification_pipeline(): # Define target folder @@ -490,7 +507,8 @@ def test_cuda_lm_sdxl_turbo_classification_pipeline(): @pytest.mark.skipif( - total_memory < 28 or total_disk_space < 25, reason="Test requires at least 28GB of RAM and 25GB of HDD for running on CPU" + total_memory < 28 or total_disk_space < 55, + reason="Test requires at least 28GB of RAM and 55GB of HDD for running on CPU", ) def test_cpu_lm_sdxl_classification_pipeline(): # Define target folder @@ -512,8 +530,8 @@ def test_cpu_lm_sdxl_classification_pipeline(): @pytest.mark.skipif( - total_memory < 16 or not torch.cuda.is_available() or total_disk_space < 25, - reason="Test requires at least 16GB of RAM, CUDA support and 25GB of HDD", + total_memory < 16 or not torch.cuda.is_available() or total_disk_space < 55, + reason="Test requires at least 16GB of RAM, CUDA support and 55GB of HDD", ) def test_cuda_lm_sdxl_classification_pipeline(): # Define target folder diff --git a/tests/unittests/test_image_generation.py b/tests/unittests/test_image_generation.py index 636f149..1170fd8 100644 --- a/tests/unittests/test_image_generation.py +++ b/tests/unittests/test_image_generation.py @@ -62,11 +62,18 @@ def _check_image_generator( image_generator.release(empty_cuda_cache=True if device != "cpu" else False) -@pytest.mark.skipif(not torch.cuda.is_available(), reason="Test requires GPU") +@pytest.mark.skipif( + not torch.cuda.is_available() or total_disk_space < 25, + reason="Test requires GPU and 25GB of HDD", +) def test_cuda_sdxl_image_generator(): _check_image_generator(StableDiffusionImageGenerator, "cuda") +@pytest.mark.skipif( + total_disk_space < 25, + reason="Test requires at least 25GB of HDD", +) def test_cpu_sdxl_image_generator(): _check_image_generator(StableDiffusionImageGenerator, "cpu") diff --git a/tests/unittests/test_prompt_generation.py b/tests/unittests/test_prompt_generation.py index 309f373..b672509 100644 --- a/tests/unittests/test_prompt_generation.py +++ b/tests/unittests/test_prompt_generation.py @@ -7,6 +7,8 @@ # Get the total memory in GB total_memory = psutil.virtual_memory().total / (1024**3) +# Get the total disk space in GB +total_disk_space = psutil.disk_usage("/").total / (1024**3) def test_simple_prompt_generator(): @@ -55,15 +57,16 @@ def _check_lm_prompt_generator(device: str): @pytest.mark.skipif( - total_memory < 16 or not torch.cuda.is_available(), - reason="Test requires at least 16GB of RAM and CUDA support", + total_memory < 16 or not torch.cuda.is_available() or total_disk_space < 35, + reason="Test requires at least 16GB of RAM, 35GB of HDD and CUDA support", ) def test_cuda_lm_prompt_generator(): _check_lm_prompt_generator("cuda") @pytest.mark.skipif( - total_memory < 28, reason="Test requires at least 28GB of RAM for running on CPU" + total_memory < 32 or total_disk_space < 35, + reason="Test requires at least 28GB of RAM and 35GB of HDD for running on CPU", ) def test_cpu_lm_prompt_generator(): _check_lm_prompt_generator("cpu") @@ -88,15 +91,16 @@ def _check_synonym_generator(device: str): @pytest.mark.skipif( - total_memory < 16 or not torch.cuda.is_available(), - reason="Test requires at least 16GB of RAM and CUDA support", + total_memory < 16 or not torch.cuda.is_available() or total_disk_space < 35, + reason="Test requires at least 16GB of RAM, 35GB of HDD and CUDA support", ) def test_cuda_synonym_generator(): _check_synonym_generator("cuda") @pytest.mark.skipif( - total_memory < 28, reason="Test requires at least 28GB of RAM for running on CPU" + total_memory < 32 or total_disk_space < 35, + reason="Test requires at least 28GB of RAM and 35GB of HDD for running on CPU", ) def test_cpu_synonym_generator(): _check_synonym_generator("cpu") From 453a5aa4e7a64816b8b67343b31f7d2415eff8c6 Mon Sep 17 00:00:00 2001 From: Jan Cuhel Date: Sat, 13 Jan 2024 16:18:11 +0100 Subject: [PATCH 23/24] Update --- tests/integration/test_pipeline.py | 54 +++++++++++------------- tests/unittests/test_annotators.py | 13 +++++- tests/unittests/test_image_generation.py | 17 +++++--- 3 files changed, 48 insertions(+), 36 deletions(-) diff --git a/tests/integration/test_pipeline.py b/tests/integration/test_pipeline.py index 59a6446..56f10b3 100644 --- a/tests/integration/test_pipeline.py +++ b/tests/integration/test_pipeline.py @@ -54,12 +54,6 @@ def test_invalid_task_value(): _check_wrong_argument_choice(cmd) -def test_invalid_save_dir(): - # Define the cmd - cmd = "datadreamer --save_dir 1" - _check_wrong_argument_choice(cmd) - - def test_invalid_prompts_number_type(): # Define the cmd cmd = "datadreamer --prompts_number value" @@ -184,8 +178,8 @@ def test_negative_num_objects_range(): # DETECTION - SIMPLE LM # ========================================================= @pytest.mark.skipif( - total_disk_space < 25, - reason="Test requires at least 25GB of HDD", + total_memory < 16 or total_disk_space < 35, + reason="Test requires at least 16GB of RAM and 35GB of HDD", ) def test_cpu_simple_sdxl_turbo_detection_pipeline(): # Define target folder @@ -206,8 +200,8 @@ def test_cpu_simple_sdxl_turbo_detection_pipeline(): @pytest.mark.skipif( - not torch.cuda.is_available() or total_disk_space < 25, - reason="Test requires GPU and 25GB of HDD", + not torch.cuda.is_available() or total_memory < 16 or total_disk_space < 35, + reason="Test requires GPU, at least 16GB of RAM and 35GB of HDD", ) def test_cuda_simple_sdxl_turbo_detection_pipeline(): # Define target folder @@ -228,8 +222,8 @@ def test_cuda_simple_sdxl_turbo_detection_pipeline(): @pytest.mark.skipif( - total_memory < 16 or total_disk_space < 25, - reason="Test requires at least 16GB of RAM and 25GB of HDD", + total_memory < 16 or total_disk_space < 35, + reason="Test requires at least 16GB of RAM and 35GB of HDD", ) def test_cpu_simple_sdxl_detection_pipeline(): # Define target folder @@ -250,8 +244,8 @@ def test_cpu_simple_sdxl_detection_pipeline(): @pytest.mark.skipif( - not torch.cuda.is_available() or total_memory < 16 or total_disk_space < 25, - reason="Test requires GPU, at least 16GB of RAM and 25GB of HDD", + not torch.cuda.is_available() or total_memory < 16 or total_disk_space < 35, + reason="Test requires GPU, at least 16GB of RAM and 35GB of HDD", ) def test_cuda_simple_sdxl_detection_pipeline(): # Define target folder @@ -275,8 +269,8 @@ def test_cuda_simple_sdxl_detection_pipeline(): # DETECTION - LLM # ========================================================= @pytest.mark.skipif( - total_memory < 28 or total_disk_space < 55, - reason="Test requires at least 28GB of RAM and 55GB of HDD for running on CPU", + total_memory < 32 or total_disk_space < 55, + reason="Test requires at least 32GB of RAM and 55GB of HDD for running on CPU", ) def test_cpu_lm_sdxl_turbo_detection_pipeline(): # Define target folder @@ -319,8 +313,8 @@ def test_cuda_lm_sdxl_turbo_detection_pipeline(): @pytest.mark.skipif( - total_memory < 28 or total_disk_space < 55, - reason="Test requires at least 28GB of RAM and 55GB of HDD for running on CPU", + total_memory < 32 or total_disk_space < 55, + reason="Test requires at least 32GB of RAM and 55GB of HDD for running on CPU", ) def test_cpu_lm_sdxl_detection_pipeline(): # Define target folder @@ -366,8 +360,8 @@ def test_cuda_lm_sdxl_detection_pipeline(): # CLASSIFICATION - SIMPLE LM # ========================================================= @pytest.mark.skipif( - total_disk_space < 25, - reason="Test requires at least 25GB of HDD", + total_memory < 16 or total_disk_space < 35, + reason="Test requires at least 16GB of RAM and 35GB of HDD", ) def test_cpu_simple_sdxl_turbo_classification_pipeline(): # Define target folder @@ -389,8 +383,8 @@ def test_cpu_simple_sdxl_turbo_classification_pipeline(): @pytest.mark.skipif( - not torch.cuda.is_available() or total_disk_space < 25, - reason="Test requires GPU and 25GB of HDD", + not torch.cuda.is_available() or total_memory < 16 or total_disk_space < 35, + reason="Test requires GPU, at least 16GB of RAM and 35GB of HDD", ) def test_cuda_simple_sdxl_turbo_classification_pipeline(): # Define target folder @@ -412,8 +406,8 @@ def test_cuda_simple_sdxl_turbo_classification_pipeline(): @pytest.mark.skipif( - total_memory < 16 or total_disk_space < 25, - reason="Test requires at least 16GB of RAM and 25GB of HDD", + total_memory < 16 or total_disk_space < 35, + reason="Test requires at least 16GB of RAM and 35GB of HDD", ) def test_cpu_simple_sdxl_classification_pipeline(): # Define target folder @@ -435,8 +429,8 @@ def test_cpu_simple_sdxl_classification_pipeline(): @pytest.mark.skipif( - not torch.cuda.is_available() or total_memory < 16 or total_disk_space < 25, - reason="Test requires GPU, at least 16GB of RAM and 25GB of HDD", + not torch.cuda.is_available() or total_memory < 16 or total_disk_space < 35, + reason="Test requires GPU, at least 16GB of RAM and 35GB of HDD", ) def test_cuda_simple_sdxl_classification_pipeline(): # Define target folder @@ -461,8 +455,8 @@ def test_cuda_simple_sdxl_classification_pipeline(): # CLASSIFICATION - LLM # ========================================================= @pytest.mark.skipif( - total_memory < 28 or total_disk_space < 55, - reason="Test requires at least 28GB of RAM and 55GB of HDD for running on CPU", + total_memory < 32 or total_disk_space < 55, + reason="Test requires at least 32GB of RAM and 55GB of HDD for running on CPU", ) def test_cpu_lm_sdxl_turbo_classification_pipeline(): # Define target folder @@ -507,8 +501,8 @@ def test_cuda_lm_sdxl_turbo_classification_pipeline(): @pytest.mark.skipif( - total_memory < 28 or total_disk_space < 55, - reason="Test requires at least 28GB of RAM and 55GB of HDD for running on CPU", + total_memory < 32 or total_disk_space < 55, + reason="Test requires at least 32GB of RAM and 55GB of HDD for running on CPU", ) def test_cpu_lm_sdxl_classification_pipeline(): # Define target folder diff --git a/tests/unittests/test_annotators.py b/tests/unittests/test_annotators.py index 7217236..f1f78bc 100644 --- a/tests/unittests/test_annotators.py +++ b/tests/unittests/test_annotators.py @@ -1,9 +1,13 @@ +import psutil import pytest import requests import torch from datadreamer.dataset_annotation.owlv2_annotator import OWLv2Annotator from PIL import Image +# Get the total disk space in GB +total_disk_space = psutil.disk_usage("/").total / (1024**3) + def _check_owlv2_annotator(device: str): url = "https://ultralytics.com/images/bus.jpg" @@ -28,10 +32,17 @@ def _check_owlv2_annotator(device: str): assert torch.all(final_labels >= 0) -@pytest.mark.skipif(not torch.cuda.is_available(), reason="Test requires GPU") +@pytest.mark.skipif( + not torch.cuda.is_available() or total_disk_space < 15, + reason="Test requires GPU and 15GB of HDD", +) def test_cuda_owlv2_annotator(): _check_owlv2_annotator("cuda") +@pytest.mark.skipif( + total_disk_space < 15, + reason="Test requires at least 15GB of HDD", +) def test_cou_owlv2_annotator(): _check_owlv2_annotator("cpu") diff --git a/tests/unittests/test_image_generation.py b/tests/unittests/test_image_generation.py index 1170fd8..000b48a 100644 --- a/tests/unittests/test_image_generation.py +++ b/tests/unittests/test_image_generation.py @@ -36,11 +36,18 @@ def _check_clip_image_tester(device: str): tester.release(empty_cuda_cache=True if device != "cpu" else False) -@pytest.mark.skipif(not torch.cuda.is_available(), reason="Test requires GPU") +@pytest.mark.skipif( + not torch.cuda.is_available() or total_disk_space < 15, + reason="Test requires GPU and 15GB of HDD", +) def test_cuda_clip_image_tester(): _check_clip_image_tester("cuda") +@pytest.mark.skipif( + total_disk_space < 15, + reason="Test requires at least 15GB of HDD", +) def test_cpu_clip_image_tester(): _check_clip_image_tester("cpu") @@ -63,16 +70,16 @@ def _check_image_generator( @pytest.mark.skipif( - not torch.cuda.is_available() or total_disk_space < 25, - reason="Test requires GPU and 25GB of HDD", + not torch.cuda.is_available() or total_memory < 16 or total_disk_space < 25, + reason="Test requires GPU, at least 16GB of RAM and 25GB of HDD", ) def test_cuda_sdxl_image_generator(): _check_image_generator(StableDiffusionImageGenerator, "cuda") @pytest.mark.skipif( - total_disk_space < 25, - reason="Test requires at least 25GB of HDD", + total_memory < 16 or total_disk_space < 25, + reason="Test requires at least 16GB of RAM and 25GB of HDD", ) def test_cpu_sdxl_image_generator(): _check_image_generator(StableDiffusionImageGenerator, "cpu") From 34009f03f00f9f989c533a8f220ba110ef77d9b9 Mon Sep 17 00:00:00 2001 From: GitHub Actions Date: Sat, 13 Jan 2024 15:22:27 +0000 Subject: [PATCH 24/24] [Automated] Updated coverage badge --- media/coverage_badge.svg | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/media/coverage_badge.svg b/media/coverage_badge.svg index 12876e6..636889b 100644 --- a/media/coverage_badge.svg +++ b/media/coverage_badge.svg @@ -9,13 +9,13 @@ - + coverage coverage - 78% - 78% + 44% + 44%