Merge pull request #9 from luxonis/package_changes

Style changes

.github/workflows/autogenerate_requirements.yaml

@@ -0,0 +1,43 @@
+name: Autogenerate Requirements
+
+on:
+  pull_request:
+    branches: [dev, main]
+    paths:
+      - 'pyproject.toml'
+      - 'tools/autogenerate_requirements.py'
+      - '.github/workflows/autogenerate_requirements.yaml'
+
+jobs:
+  update-requirements:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Check out repository
+      uses: actions/checkout@v3
+      with:
+        ref: ${{ github.head_ref }}
+
+    - name: Set up Python
+      uses: actions/setup-python@v3
+      with:
+        python-version: '3.10'
+
+    - name: Run autogeneration script
+      run: |
+        pip install toml
+        python tools/autogenerate_requirements.py
+
+    - name: Commit files
+      run: |
+        git config --global user.name 'GitHub Actions'
+        git config --global user.email '[email protected]'
+        git diff --quiet requirements.txt || {
+          git add requirements.txt
+          git commit -m "[Automated] Updated requirements.txt"
+        }
+
+    - name: Push changes
+      uses: ad-m/github-push-action@master
+      with:
+        branch: ${{ github.head_ref }}

.github/workflows/pre-commit.yaml

@@ -0,0 +1,15 @@
+name: pre-commit
+
+on:
+  pull_request:
+    branches: [dev, main]
+
+jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions/setup-python@v3
+      with:
+        python-version: '3.8'
+    - uses: pre-commit/[email protected]

.pre-commit-config.yaml

@@ -1,11 +1,34 @@
 repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.1.2
+    hooks:
+      - id: ruff
+        args: [--fix, --exit-non-zero-on-fix]
+        types_or: [python, pyi]
+
   - repo: https://github.com/ambv/black
     rev: 23.3.0
     hooks:
     - id: black
-      language_version: python3
+      language_version: python3.8
+
+  - repo: https://github.com/PyCQA/docformatter
+    rev: v1.7.5
+    hooks:
+      - id: docformatter
+        additional_dependencies: [tomli]
+        args: [--in-place]
+
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.4.0
     hooks:
     - id: no-commit-to-branch
-      args: ['--branch', 'main', '--branch', 'dev']
+      args: ['--branch', 'main', '--branch', 'dev']
+
+  - repo: https://github.com/executablebooks/mdformat
+    rev: 0.7.10
+    hooks:
+      - id: mdformat
+        additional_dependencies:
+          - mdformat-gfm
+          - mdformat-toc

README.md

@@ -1,8 +1,10 @@
-![DataDreamer examples](images/grid_image_3x2_generated_dataset.jpg)
+# DataDreamer
 
+[![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/luxonis/datadreamer/blob/main/examples/generate_dataset_and_train_yolo.ipynb)
 
-# DataDreamer
+![DataDreamer examples](images/grid_image_3x2_generated_dataset.jpg)
+
 `DataDreamer` is an advanced toolkit engineered to facilitate the development of edge AI models, irrespective of initial data availability. Distinctive features of DataDreamer include:
 
 - **Synthetic Data Generation**: Eliminate the dependency on extensive datasets for AI training. DataDreamer empowers users to generate synthetic datasets from the ground up, utilizing advanced AI algorithms capable of producing high-quality, diverse images.
@@ -18,11 +20,28 @@ The rationale behind the name `DataDreamer`:
 
 In essence, `DataDreamer` is designed to transform the AI development process, making it more accessible, efficient, and effective, turning visionary ideas into reality.
 
+## Table of Contents
+
+- [Features](#features)
+- [Installation](#installation)
+- [Hardware Requirements](#hardware-requirements)
+- [Usage](#usage)
+  - [Main Parameters](#main-parameters)
+  - [Additional Parameters](#additional-parameters)
+  - [Example](#example)
+  - [Output](#output)
+  - [Annotations Format](#annotations-format)
+  - [Note](#note)
+- [Limitations](#limitations)
+- [License](#license)
+- [Acknowledgements](#acknowledgements)
+
 ## Features
 
 - **Prompt Generation**: Automate the creation of image prompts using powerful language models.
 
-   *Provided class names: ["horse", "robot"]* ->  *Generated prompt: "A photo of a horse and a robot coexisting peacefully in the midst of a serene pasture."*
+  *Provided class names: \["horse", "robot"\]* ->  *Generated prompt: "A photo of a horse and a robot coexisting peacefully in the midst of a serene pasture."*
+
 - **Image Generation**: Generate synthetic datasets with state-of-the-art generative models.
 
 <img src="images/generated_image.jpg" width="512">
@@ -31,7 +50,6 @@ In essence, `DataDreamer` is designed to transform the AI development process, m
 
 <img src="images/annotated_image.jpg" width="512">
 
-
 - **Edge Model Training**: Train efficient small-scale neural networks for edge deployment. (not part of this library)
 
 [Example](https://github.com/luxonis/datadreamer/blob/main/examples/generate_dataset_and_train_yolo.ipynb)
@@ -50,24 +68,24 @@ pip install -e .
 ```
 
 ## Hardware Requirements
+
 To ensure optimal performance and compatibility with the libraries used in this project, the following hardware specifications are recommended:
 
 - `GPU`: A CUDA-compatible GPU with a minimum of 16 GB memory. This is essential for libraries like `torch`, `torchvision`, `transformers`, and `diffusers`, which leverage CUDA for accelerated computing in machine learning and image processing tasks.
 - `RAM`: At least 16 GB of system RAM, although more (32 GB or above) is beneficial for handling large datasets and intensive computations.
 
 ## Usage
 
-### Overview
 The `datadreamer/pipelines/generate_dataset_from_scratch.py` (`datadreamer` command) script is a powerful tool for generating and annotating images with specific objects. It uses advanced models to both create images and accurately annotate them with bounding boxes for designated objects.
 
-### Usage
 Run the following command in your terminal to use the script:
 
 ```bash
 datadreamer --save_dir <directory> --class_names <objects> --prompts_number <number> [additional options]
 ```
 
 ### Main Parameters
+
 - `--save_dir` (required): Path to the directory for saving generated images and annotations.
 - `--class_names` (required): Space-separated list of object names for image generation and annotation. Example: person moon robot.
 - `--prompts_number` (optional): Number of prompts to generate for each object. Defaults to 10.
@@ -88,12 +106,15 @@ datadreamer --save_dir <directory> --class_names <objects> --prompts_number <num
 - `--seed`: Set a random seed for image generation. Default is 42.
 
 ### Example
+
 ```bash
 datadreamer --save_dir path/to/save_directory --class_names person moon robot --prompts_number 20 --prompt_generator simple --num_objects_range 2 4 --image_generator sdxl-turbo
 ```
+
 This command generates images for the specified objects, saving them and their annotations in the given directory. The script allows customization of the generation process through various parameters, adapting to different needs and hardware configurations.
 
 ### Output
+
 The dataset comprises two primary components: images and their corresponding annotations, stored as JSON files.
 
 ```bash
@@ -109,10 +130,12 @@ save_dir/
 ```
 
 ### Annotations Format
+
 1. Detection Annotations (detection_annotations.json):
 
 - Each entry corresponds to an image and contains bounding boxes and labels for objects in the image.
 - Format:
+
 ```bash
 {
   "image_path": {
@@ -123,10 +146,12 @@ save_dir/
   "class_names": ["class1", "class2", ...]
 }
 ```
+
 2. Classification Annotations (classification_annotations.json):
 
 - Each entry corresponds to an image and contains labels for the image.
 - Format:
+
 ```bash
 {
   "image_path": {
@@ -137,12 +162,12 @@ save_dir/
 }
 ```
 
-
-
 ### Note
+
 Please make sure that all dependencies are correctly installed and that the datadreamer package is properly set up in your Python environment before running the script.
 
 ## Limitations
+
 While the datadreamer library leverages advanced Generative models to synthesize datasets and Foundation models for annotation, there are inherent limitations to consider:
 
 - `Incomplete Object Representation`: Occasionally, the generative models might not include all desired objects in the synthetic images. This could result from the complexity of the scene or limitations within the model's learned patterns.
@@ -152,7 +177,8 @@ While the datadreamer library leverages advanced Generative models to synthesize
 Despite these limitations, the datasets created by datadreamer provide a valuable foundation for developing and training models, especially for edge computing scenarios where data availability is often a challenge. The synthetic and annotated data should be seen as a stepping stone, granting a significant head start in the model development process.
 
 ## License
-This project is licensed under the Apache License, Version 2.0 - see the LICENSE file for details.
+
+This project is licensed under the [Apache License, Version 2.0](https://opensource.org/license/apache-2-0/) - see the [LICENSE](LICENSE) file for details.
 
 ## Acknowledgements
 

examples/image_annotation_example.py

@@ -1,8 +1,8 @@
-from PIL import Image
-from datadreamer.dataset_annotation import OWLv2Annotator, TaskList, Kosmos2Annotator
-import matplotlib.pyplot as plt
 import matplotlib.patches as patches
+import matplotlib.pyplot as plt
 import numpy as np
+from datadreamer.dataset_annotation import OWLv2Annotator
+from PIL import Image
 
 # Initialize the OWLv2Annotator
 annotator = OWLv2Annotator(

examples/image_generation_example.py

@@ -1,9 +1,8 @@
-from datadreamer.image_generation.clip_image_tester import ClipImageTester
+import matplotlib.pyplot as plt
 from datadreamer.image_generation import (
-    StableDiffusionImageGenerator,
     StableDiffusionTurboImageGenerator,
 )
-import matplotlib.pyplot as plt
+from datadreamer.image_generation.clip_image_tester import ClipImageTester
 
 # Define some prompts for image generation
 prompts = [

examples/step_by_step_dataset_generation_pipeline.ipynb

@@ -16,16 +16,14 @@
     }
    ],
    "source": [
-    "from datadreamer.prompt_generation import LMPromptGenerator\n",
-    "from datadreamer.image_generation import StableDiffusionImageGenerator, StableDiffusionTurboImageGenerator\n",
-    "from datadreamer.dataset_annotation import OWLv2Annotator\n",
-    "\n",
-    "import matplotlib.pyplot as plt\n",
-    "import torch\n",
-    "from PIL import Image\n",
-    "import matplotlib.pyplot as plt\n",
     "import matplotlib.patches as patches\n",
-    "import numpy as np\n"
+    "import matplotlib.pyplot as plt\n",
+    "import numpy as np\n",
+    "from datadreamer.dataset_annotation import OWLv2Annotator\n",
+    "from datadreamer.image_generation import (\n",
+    "    StableDiffusionTurboImageGenerator,\n",
+    ")\n",
+    "from datadreamer.prompt_generation import LMPromptGenerator\n"
    ]
   },
   {

examples/visualize_detection_dataset.py

@@ -1,7 +1,8 @@
+import argparse
 import json
 import os
+
 import cv2
-import argparse
 
 
 def draw_rounded_rectangle(img, pt1, pt2, color, thickness, r, d):

pyproject.toml

@@ -7,36 +7,66 @@ name = "datadreamer"
 version = "0.1.0"
 description = "A library for dataset generation and knowledge extraction from foundation computer vision models."
 readme = "README.md"
-authors = [{ name = "Luxonis", email = "[email protected]"}]
+requires-python = ">=3.8"
 license = { file = "LICENSE" }
+maintainers = [{ name = "Luxonis", email = "[email protected]"}]
+keywords = ["computer vision", "AI", "machine learning", "generative models"]
 classifiers = [
+    "License :: Apache License 2.0",
     "Development Status :: 3 - Alpha",
+    "Programming Language :: Python :: 3.8",
     "Intended Audience :: Developers",
-    "Programming Language :: Python :: 3",
-    "Topic :: Software Development :: Libraries"
+    "Topic :: Software Development :: Libraries",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    "Topic :: Scientific/Engineering :: Image Processing",
+    "Topic :: Scientific/Engineering :: Image Recognition",
 ]
-keywords = ["computer vision", "AI", "machine learning", "generative models"]
 dependencies = [
-    "torch",
-    "torchvision",
-    "transformers",
-    "diffusers",
-    "compel",
-    "tqdm",
-    "Pillow",
-    "numpy",
-    "matplotlib",
-    "opencv-python",
-    "accelerate",
-    "scipy"
+    "torch>=2.0.0",
+    "torchvision>=0.16.0",
+    "transformers>=4.0.0",
+    "diffusers>=0.24.0",
+    "compel>=2.0.0",
+    "tqdm>=4.0.0",
+    "Pillow>=10.0.0",
+    "numpy>=1.22.0",
+    "matplotlib>=3.6.0",
+    "opencv-python>=4.7.0",
+    "accelerate>=0.25.0",
+    "scipy>=1.10.0",
+]
+[project.optional-dependencies]
+dev = [
+    "datadreamer",
+    "pre-commit>=3.2.1",
+    "toml>=0.10.2",
 ]
-requires-python = ">=3.8"
 
 [project.urls]
 Homepage = "https://github.com/luxonis/datadreamer"
 
+[project.scripts]
+datadreamer = "datadreamer.pipelines.generate_dataset_from_scratch:main"
+
 [tool.setuptools.packages.find]
 where = ["src"]
 
-[project.scripts]
-datadreamer = "datadreamer.pipelines.generate_dataset_from_scratch:main"
+[tool.ruff]
+target-version = "py38"
+
+[tool.ruff.lint]
+ignore = ["F403", "B028", "B905", "D1"]
+select = ["E4", "E7", "E9", "F", "W", "B", "I"]
+
+[tool.ruff.pydocstyle]
+convention = "google"
+
+[tool.docformatter]
+black = true
+
+[tool.mypy]
+python_version = "3.8"
+ignore_missing_imports = true
+
+[tool.pyright]
+typeCheckingMode = "basic"