Merge pull request #8 from R3gm/dev2

version 0.5.0
R3gm · Sep 12, 2024 · 80177ae · 80177ae
2 parents 585eedc + a5fd310
commit 80177ae
Show file tree

Hide file tree

Showing 11 changed files with 462 additions and 229 deletions.
diff --git a/README.md b/README.md
@@ -6,7 +6,7 @@ The goal of this project is to make Stable Diffusion more accessible, simple and
 
 **Installation:**
 ```
-pip install stablepy==0.4.1
+pip install stablepy==0.5.0
 ```
 
 **Usage:**
@@ -80,6 +80,16 @@ images[1]
 ```
 **📖 News:**
 
+🔥 Version 0.5.0: New Update Details
+
+- Fix LoRA SDXL compatibility.
+- Latent upscaler and variants.
+- Perturbed Attention Guidance (PAG) enhances image generation quality without the need for training.
+- Multiple images for one FaceID adapter.
+- ControlNet for SDXL: MLSD, Segmentation, Normalbae.
+- ControlNet "lineart_anime" task accessible and able to load a model different from the "lineart" task.
+- ControlNet Tile and Recolor for SD1.5 and SDXL ("tile" replaces the previous task called "sdxl_tile_realistic").
+
 🔥 Version 0.4.0: New Update Details
 
 - IP Adapter with the variants FaceID and Instant-Style

diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "stablepy"
-version = "0.4.1"
+version = "0.5.0"
 description = "A tool for easy use of stable diffusion"
 authors = ["Roger Condori(R3gm) <[email protected]>"]
 readme = "README.md"
@@ -11,7 +11,7 @@ torch = {version = "*", source = "pytorch-gpu-src"}
 torchvision = {version = "*", source = "pytorch-gpu-src"}
 torchaudio = {version = "*", source = "pytorch-gpu-src"}
 omegaconf = "2.3.0"
-diffusers = "0.29.0"
+diffusers = "0.30.2"
 compel = "2.0.2"
 invisible-watermark = "^0.2.0"
 transformers = "^4.41.2"
@@ -28,6 +28,7 @@ peft = "^0.11.1"
 torchsde = "^0.2.6"
 onnxruntime = "^1.18.0"
 insightface = "^0.7.3"
+opencv-contrib-python = "^4.8.0.76"
 
 [[tool.poetry.source]]
 name = "pytorch-gpu-src"

diff --git a/stablepy/__init__.py b/stablepy/__init__.py
@@ -4,6 +4,7 @@
 from .diffusers_vanilla import utils
 from .upscalers.esrgan import UpscalerESRGAN, UpscalerLanczos, UpscalerNearest
 from .logging.logging_setup import logger
+from .diffusers_vanilla.high_resolution import LATENT_UPSCALERS
 from .diffusers_vanilla.constants import (
     CONTROLNET_MODEL_IDS,
     VALID_TASKS,

diff --git a/stablepy/__version__.py b/stablepy/__version__.py
@@ -1 +1 @@
-__version__ = "0.4.1"
+__version__ = "0.5.0"
diff --git a/stablepy/diffusers_vanilla/constants.py b/stablepy/diffusers_vanilla/constants.py
@@ -18,18 +18,68 @@
     EDMEulerScheduler,
     TCDScheduler,
 )
+from diffusers import (
+    StableDiffusionControlNetPipeline,
+    StableDiffusionControlNetInpaintPipeline,
+    StableDiffusionPipeline,
+    StableDiffusionXLInpaintPipeline,
+    StableDiffusionXLAdapterPipeline,
+    StableDiffusionXLPipeline,
+    StableDiffusionXLControlNetPipeline,
+    StableDiffusionPAGPipeline,
+    # StableDiffusionControlNetPAGInpaintPipeline,
+    StableDiffusionControlNetPAGPipeline,
+    # StableDiffusionControlNetImg2ImgPAGPipeline,
+    StableDiffusionXLPAGPipeline,
+    StableDiffusionXLPAGInpaintPipeline,
+    StableDiffusionXLControlNetPAGPipeline,
+    # StableDiffusionXLAdapterPAGPipeline,
+    # StableDiffusionXLControlNetImg2ImgPAGPipeline,
+)
+
+CLASS_DIFFUSERS_TASK = {
+    "StableDiffusionPipeline": {
+        "base": StableDiffusionPipeline,
+        "inpaint": StableDiffusionControlNetInpaintPipeline,
+        "controlnet": StableDiffusionControlNetPipeline,
+        # "controlnet_img2img": StableDiffusionControlNetImg2ImgPipeline,
+    },
+    "StableDiffusionXLPipeline": {
+        "base": StableDiffusionXLPipeline,
+        "inpaint": StableDiffusionXLInpaintPipeline,
+        "controlnet": StableDiffusionXLControlNetPipeline,
+        "adapter": StableDiffusionXLAdapterPipeline,
+        # "controlnet_img2img": StableDiffusionXLControlNetImg2ImgPipeline,
+    },
+}
+
+CLASS_PAG_DIFFUSERS_TASK = {
+    "StableDiffusionPipeline": {
+        "base": StableDiffusionPAGPipeline,
+        "inpaint": StableDiffusionControlNetInpaintPipeline,
+        "controlnet": StableDiffusionControlNetPAGPipeline,
+        # "controlnet_img2img": StableDiffusionControlNetImg2ImgPAGPipeline,
+    },
+    "StableDiffusionXLPipeline": {
+        "base": StableDiffusionXLPAGPipeline,
+        "inpaint": StableDiffusionXLPAGInpaintPipeline,
+        "controlnet": StableDiffusionXLControlNetPAGPipeline,
+        # "adapter": StableDiffusionXLAdapterPAGPipeline,
+        # "controlnet_img2img": StableDiffusionXLControlNetImg2ImgPAGPipeline,
+    },
+}
 
 CONTROLNET_MODEL_IDS = {
     "openpose": ["lllyasviel/control_v11p_sd15_openpose", "r3gm/controlnet-openpose-sdxl-1.0-fp16"],
     "canny": ["lllyasviel/control_v11p_sd15_canny", "r3gm/controlnet-canny-scribble-integrated-sdxl-v2-fp16"],
-    "mlsd": "lllyasviel/control_v11p_sd15_mlsd",
+    "mlsd": ["lllyasviel/control_v11p_sd15_mlsd", "r3gm/controlnet-union-sdxl-1.0-fp16"],
     "scribble": ["lllyasviel/control_v11p_sd15_scribble", "r3gm/controlnet-canny-scribble-integrated-sdxl-v2-fp16"],
-    "softedge": ["lllyasviel/control_v11p_sd15_softedge", "r3gm/controlnet-canny-scribble-integrated-sdxl-v2-fp16"],
-    "segmentation": "lllyasviel/control_v11p_sd15_seg",
-    "depth": ["lllyasviel/control_v11f1p_sd15_depth", "diffusers/controlnet-depth-sdxl-1.0-mid"],
-    "normalbae": "lllyasviel/control_v11p_sd15_normalbae",
-    "lineart": ["lllyasviel/control_v11p_sd15_lineart", "r3gm/controlnet-lineart-anime-sdxl-fp16"],
-    "lineart_anime": "lllyasviel/control_v11p_sd15s2_lineart_anime",
+    "softedge": ["lllyasviel/control_v11p_sd15_softedge", "r3gm/controlnet-union-sdxl-1.0-fp16"],
+    "segmentation": ["lllyasviel/control_v11p_sd15_seg", "r3gm/controlnet-union-sdxl-1.0-fp16"],
+    "depth": ["lllyasviel/control_v11f1p_sd15_depth", "r3gm/controlnet-union-sdxl-1.0-fp16"],
+    "normalbae": ["lllyasviel/control_v11p_sd15_normalbae", "r3gm/controlnet-union-sdxl-1.0-fp16"],
+    "lineart": ["lllyasviel/control_v11p_sd15_lineart", "r3gm/controlnet-union-sdxl-1.0-fp16"],
+    "lineart_anime": ["lllyasviel/control_v11p_sd15s2_lineart_anime", "r3gm/controlnet-lineart-anime-sdxl-fp16"],
     "shuffle": "lllyasviel/control_v11e_sd15_shuffle",
     "ip2p": "lllyasviel/control_v11e_sd15_ip2p",
     "inpaint": "lllyasviel/control_v11p_sd15_inpaint",
@@ -41,7 +91,8 @@
     "sdxl_openpose_t2i": "TencentARC/t2i-adapter-openpose-sdxl-1.0",
     "img2img": "Nothinghere",
     "pattern": ["monster-labs/control_v1p_sd15_qrcode_monster", "r3gm/control_v1p_sdxl_qrcode_monster_fp16"],
-    "sdxl_tile_realistic": "Yakonrus/SDXL_Controlnet_Tile_Realistic_v2",
+    "tile": ["lllyasviel/control_v11f1e_sd15_tile", "r3gm/controlnet-tile-sdxl-1.0-fp16"],  # "sdxl_tile_realistic": "Yakonrus/SDXL_Controlnet_Tile_Realistic_v2",
+    "recolor": ["latentcat/control_v1p_sd15_brightness", "r3gm/controlnet-recolor-sdxl-fp16"],
     # "sdxl_depth-zoe_t2i": "TencentARC/t2i-adapter-depth-zoe-sdxl-1.0",
     # "sdxl_recolor_t2i": "TencentARC/t2i-adapter-recolor-sdxl-1.0",
 }
@@ -89,16 +140,17 @@
 OLD_PROMPT_WEIGHT_OPTIONS = ALL_PROMPT_WEIGHT_OPTIONS[0:2]
 
 SCHEDULER_CONFIG_MAP = {
-    "DPM++ 2M": (DPMSolverMultistepScheduler, {"use_karras_sigmas": False}),
-    "DPM++ 2M Karras": (DPMSolverMultistepScheduler, {"use_karras_sigmas": True}),
+    "DPM++ 2M": (DPMSolverMultistepScheduler, {"algorithm_type": "dpmsolver++", "use_karras_sigmas": False}),
+    "DPM++ 2M Karras": (DPMSolverMultistepScheduler, {"algorithm_type": "dpmsolver++", "use_karras_sigmas": True}),
     "DPM++ 2M SDE": (DPMSolverMultistepScheduler, {"use_karras_sigmas": False, "algorithm_type": "sde-dpmsolver++"}),
     "DPM++ 2M SDE Karras": (DPMSolverMultistepScheduler, {"use_karras_sigmas": True, "algorithm_type": "sde-dpmsolver++"}),
-    "DPM++ 2S": (DPMSolverSinglestepScheduler, {"use_karras_sigmas": False}),
-    "DPM++ 2S Karras": (DPMSolverSinglestepScheduler, {"use_karras_sigmas": True}),
-    "DPM++ 1S": (DPMSolverMultistepScheduler, {"solver_order": 1}),
-    "DPM++ 1S Karras": (DPMSolverMultistepScheduler, {"solver_order": 1, "use_karras_sigmas": True}),
-    "DPM++ 3M": (DPMSolverMultistepScheduler, {"solver_order": 3}),
-    "DPM++ 3M Karras": (DPMSolverMultistepScheduler, {"solver_order": 3, "use_karras_sigmas": True}),
+    "DPM++ 2S": (DPMSolverSinglestepScheduler, {"algorithm_type": "dpmsolver++", "use_karras_sigmas": False}),
+    "DPM++ 2S Karras": (DPMSolverSinglestepScheduler, {"algorithm_type": "dpmsolver++", "use_karras_sigmas": True}),
+    "DPM++ 1S": (DPMSolverMultistepScheduler, {"algorithm_type": "dpmsolver++", "solver_order": 1}),
+    "DPM++ 1S Karras": (DPMSolverMultistepScheduler, {"algorithm_type": "dpmsolver++", "solver_order": 1, "use_karras_sigmas": True}),
+    "DPM++ 3M": (DPMSolverMultistepScheduler, {"algorithm_type": "dpmsolver++", "solver_order": 3}),
+    "DPM++ 3M Karras": (DPMSolverMultistepScheduler, {"algorithm_type": "dpmsolver++", "solver_order": 3, "use_karras_sigmas": True}),
+    "DPM 3M": (DPMSolverMultistepScheduler, {"algorithm_type": "dpmsolver", "final_sigmas_type": "sigma_min", "solver_order": 3}),
     "DPM++ SDE": (DPMSolverSDEScheduler, {"use_karras_sigmas": False}),
     "DPM++ SDE Karras": (DPMSolverSDEScheduler, {"use_karras_sigmas": True}),
     "DPM2": (KDPM2DiscreteScheduler, {}),
@@ -125,8 +177,8 @@
     "DPM++ 2M EDM Karras": (EDMDPMSolverMultistepScheduler, {"use_karras_sigmas": True, "solver_order": 2, "solver_type": "midpoint", "final_sigmas_type": "zero", "algorithm_type": "dpmsolver++"}),
     "DDPM": (DDPMScheduler, {}),
 
-    "DPM++ 2M Lu": (DPMSolverMultistepScheduler, {"use_lu_lambdas": True}),
-    "DPM++ 2M Ef": (DPMSolverMultistepScheduler, {"euler_at_final": True}),
+    "DPM++ 2M Lu": (DPMSolverMultistepScheduler, {"algorithm_type": "dpmsolver++", "use_lu_lambdas": True}),
+    "DPM++ 2M Ef": (DPMSolverMultistepScheduler, {"algorithm_type": "dpmsolver++", "euler_at_final": True}),
     "DPM++ 2M SDE Lu": (DPMSolverMultistepScheduler, {"use_lu_lambdas": True, "algorithm_type": "sde-dpmsolver++"}),
     "DPM++ 2M SDE Ef": (DPMSolverMultistepScheduler, {"algorithm_type": "sde-dpmsolver++", "euler_at_final": True}),
 
@@ -153,9 +205,9 @@
         "base_light_v2": ["h94/IP-Adapter", "models", "ip-adapter_sd15_light_v11.bin", "H"],
         "faceid_plus": ["h94/IP-Adapter-FaceID", "", "ip-adapter-faceid-plus_sd15.bin", "H"],
         "faceid_plus_v2": ["h94/IP-Adapter-FaceID", "", "ip-adapter-faceid-plusv2_sd15.bin", "H"],
-        "faceid": ["h94/IP-Adapter-FaceID", "", "ip-adapter-faceid_sd15.bin", None],
-        "faceid_portrait_v2": ["h94/IP-Adapter-FaceID", "", "ip-adapter-faceid-portrait-v11_sd15.bin", None],  # last portrait
-        "faceid_portrait": ["h94/IP-Adapter-FaceID", "", "ip-adapter-faceid-portrait_sd15.bin", None],
+        "faceid": ["h94/IP-Adapter-FaceID", "", "ip-adapter-faceid_sd15.bin", "H"],  # None
+        "faceid_portrait_v2": ["h94/IP-Adapter-FaceID", "", "ip-adapter-faceid-portrait-v11_sd15.bin", "H"],  # None
+        "faceid_portrait": ["h94/IP-Adapter-FaceID", "", "ip-adapter-faceid-portrait_sd15.bin", "H"],  # None
         "composition_plus": ["ostris/ip-composition-adapter", "", "ip_plus_composition_sd15.safetensors", "H"]
     },
     "StableDiffusionXLPipeline": {
@@ -165,9 +217,9 @@
         "base_vit_G": ["h94/IP-Adapter", "sdxl_models", "ip-adapter_sdxl.safetensors", "G"],
         "base": ["h94/IP-Adapter", "sdxl_models", "ip-adapter_sdxl_vit-h.safetensors", "H"],
         "faceid_plus_v2": ["h94/IP-Adapter-FaceID", "", "ip-adapter-faceid-plusv2_sdxl.bin", "H"],
-        "faceid": ["h94/IP-Adapter-FaceID", "", "ip-adapter-faceid_sdxl.bin", None],
-        "faceid_portrait": ["h94/IP-Adapter-FaceID", "", "ip-adapter-faceid-portrait_sdxl.bin", None],
-        "faceid_portrait_v2": ["h94/IP-Adapter-FaceID", "", "ip-adapter-faceid-portrait_sdxl_unnorm.bin", None],
+        "faceid": ["h94/IP-Adapter-FaceID", "", "ip-adapter-faceid_sdxl.bin", "H"],  # None
+        "faceid_portrait": ["h94/IP-Adapter-FaceID", "", "ip-adapter-faceid-portrait_sdxl.bin", "H"],  # None
+        "faceid_portrait_v2": ["h94/IP-Adapter-FaceID", "", "ip-adapter-faceid-portrait_sdxl_unnorm.bin", "H"],  # None
         "composition_plus": ["ostris/ip-composition-adapter", "", "ip_plus_composition_sdxl.safetensors", "H"]
     }
 }  # no suffix lora

diff --git a/stablepy/diffusers_vanilla/extra_model_loaders.py b/stablepy/diffusers_vanilla/extra_model_loaders.py
@@ -64,7 +64,7 @@ def custom_task_model_loader(
 
     elif model_category in ["hires", "detailfix_img2img"]:
         # Pipe hires detailfix_pipe img2img
-        if task_name != "txt2img":
+        if task_name != "txt2img" or hasattr(pipe, "set_pag_applied_layers"):
             if not hasattr(pipe, "text_encoder_2"):
                 hires_pipe = StableDiffusionPipeline(
                     vae=pipe.vae,
@@ -91,9 +91,9 @@ def custom_task_model_loader(
                     image_encoder=pipe.image_encoder,
                 )
 
-            hires_pipe = AutoPipelineForImage2Image.from_pipe(hires_pipe)
+            hires_pipe = AutoPipelineForImage2Image.from_pipe(hires_pipe, enable_pag=False)
         else:
-            hires_pipe = AutoPipelineForImage2Image.from_pipe(pipe)
+            hires_pipe = AutoPipelineForImage2Image.from_pipe(pipe, enable_pag=False)
 
         if hasattr(hires_pipe, "text_encoder_2"):
             hires_pipe.enable_vae_slicing()