ControlNet_with_GPT_4.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "V100",
      "authorship_tag": "ABX9TyNdnNfFdzFXPeU3tHWq1xyj",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "bd3ba58ef21f4125b702ed9b01e524b0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_a993e2667bae40febbad769e54489ef3",
              "IPY_MODEL_98628783a7e14033ace2af4fbb3e8107",
              "IPY_MODEL_e6ca7a8529384052a33ad90e5d114ec4"
            ],
            "layout": "IPY_MODEL_fa3d627474f342aaa8e3da841d65c903"
          }
        },
        "a993e2667bae40febbad769e54489ef3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4c08a47828c148c6bfbb5477432269af",
            "placeholder": "​",
            "style": "IPY_MODEL_9c47510793ee49bea9590dc23f153703",
            "value": "Loading pipeline components...: 100%"
          }
        },
        "98628783a7e14033ace2af4fbb3e8107": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b8612aa5533a4c6eb29185903c132055",
            "max": 6,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_27e2fb8a96054127bc0a3e666c4d2821",
            "value": 6
          }
        },
        "e6ca7a8529384052a33ad90e5d114ec4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_282db119ce57419887295e59e78d695c",
            "placeholder": "​",
            "style": "IPY_MODEL_7715573a22454048b1c2d40049f67ccf",
            "value": " 6/6 [00:20&lt;00:00,  2.43s/it]"
          }
        },
        "fa3d627474f342aaa8e3da841d65c903": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4c08a47828c148c6bfbb5477432269af": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9c47510793ee49bea9590dc23f153703": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b8612aa5533a4c6eb29185903c132055": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "27e2fb8a96054127bc0a3e666c4d2821": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "282db119ce57419887295e59e78d695c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7715573a22454048b1c2d40049f67ccf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/KevinWang676/ControlNet-with-GPT-4/blob/main/ControlNet_with_GPT_4.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!git clone https://github.com/KevinWang676/ControlNet-with-GPT-4.git"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "V_xn3m7arRfn",
        "outputId": "c6788369-5ff9-4eb7-dc6a-70afb418c516"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "fatal: destination path 'ControlNet-with-GPT-4' already exists and is not an empty directory.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "cd ControlNet-with-GPT-4"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "woMlZSRJ0ldd",
        "outputId": "001e83bf-8822-41e3-84de-c375f13efcc4"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/ControlNet-with-GPT-4\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pip install -r requirements.txt"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FT-71Css0n0J",
        "outputId": "ede00008-6249-4707-b82c-661ce839c6d6"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting git+https://github.com/huggingface/diffusers.git (from -r requirements.txt (line 1))\n",
            "  Cloning https://github.com/huggingface/diffusers.git to /tmp/pip-req-build-7ltfv4m_\n",
            "  Running command git clone --filter=blob:none --quiet https://github.com/huggingface/diffusers.git /tmp/pip-req-build-7ltfv4m_\n",
            "  Resolved https://github.com/huggingface/diffusers.git to commit bc7a4d4917456afd70913be85bd25c556c25862c\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting git+https://github.com/huggingface/accelerate (from -r requirements.txt (line 3))\n",
            "  Cloning https://github.com/huggingface/accelerate to /tmp/pip-req-build-7lcwcgy4\n",
            "  Running command git clone --filter=blob:none --quiet https://github.com/huggingface/accelerate /tmp/pip-req-build-7lcwcgy4\n",
            "  Resolved https://github.com/huggingface/accelerate to commit c7c99a30ea88f00c854017d362e72b9cf60c7db6\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: transformers in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 2)) (4.34.1)\n",
            "Requirement already satisfied: opencv-contrib-python in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 4)) (4.8.0.76)\n",
            "Requirement already satisfied: torch in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 5)) (1.13.1)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 6)) (1.23.5)\n",
            "Requirement already satisfied: xformers==0.0.16 in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 7)) (0.0.16)\n",
            "Requirement already satisfied: openai in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 8)) (0.28.1)\n",
            "Requirement already satisfied: cairosvg in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 9)) (2.7.1)\n",
            "Requirement already satisfied: gradio==3.33.0 in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 10)) (3.33.0)\n",
            "Requirement already satisfied: pyre-extensions==0.0.23 in /usr/local/lib/python3.10/dist-packages (from xformers==0.0.16->-r requirements.txt (line 7)) (0.0.23)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from torch->-r requirements.txt (line 5)) (4.8.0)\n",
            "Requirement already satisfied: nvidia-cuda-runtime-cu11==11.7.99 in /usr/local/lib/python3.10/dist-packages (from torch->-r requirements.txt (line 5)) (11.7.99)\n",
            "Requirement already satisfied: nvidia-cudnn-cu11==8.5.0.96 in /usr/local/lib/python3.10/dist-packages (from torch->-r requirements.txt (line 5)) (8.5.0.96)\n",
            "Requirement already satisfied: nvidia-cublas-cu11==11.10.3.66 in /usr/local/lib/python3.10/dist-packages (from torch->-r requirements.txt (line 5)) (11.10.3.66)\n",
            "Requirement already satisfied: nvidia-cuda-nvrtc-cu11==11.7.99 in /usr/local/lib/python3.10/dist-packages (from torch->-r requirements.txt (line 5)) (11.7.99)\n",
            "Requirement already satisfied: aiofiles in /usr/local/lib/python3.10/dist-packages (from gradio==3.33.0->-r requirements.txt (line 10)) (23.2.1)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from gradio==3.33.0->-r requirements.txt (line 10)) (3.8.6)\n",
            "Requirement already satisfied: altair>=4.2.0 in /usr/local/lib/python3.10/dist-packages (from gradio==3.33.0->-r requirements.txt (line 10)) (4.2.2)\n",
            "Requirement already satisfied: fastapi in /usr/local/lib/python3.10/dist-packages (from gradio==3.33.0->-r requirements.txt (line 10)) (0.104.0)\n",
            "Requirement already satisfied: ffmpy in /usr/local/lib/python3.10/dist-packages (from gradio==3.33.0->-r requirements.txt (line 10)) (0.3.1)\n",
            "Requirement already satisfied: gradio-client>=0.2.4 in /usr/local/lib/python3.10/dist-packages (from gradio==3.33.0->-r requirements.txt (line 10)) (0.6.1)\n",
            "Requirement already satisfied: httpx in /usr/local/lib/python3.10/dist-packages (from gradio==3.33.0->-r requirements.txt (line 10)) (0.25.0)\n",
            "Requirement already satisfied: huggingface-hub>=0.14.0 in /usr/local/lib/python3.10/dist-packages (from gradio==3.33.0->-r requirements.txt (line 10)) (0.17.3)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from gradio==3.33.0->-r requirements.txt (line 10)) (3.1.2)\n",
            "Requirement already satisfied: markdown-it-py[linkify]>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from gradio==3.33.0->-r requirements.txt (line 10)) (2.2.0)\n",
            "Requirement already satisfied: markupsafe in /usr/local/lib/python3.10/dist-packages (from gradio==3.33.0->-r requirements.txt (line 10)) (2.1.3)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.10/dist-packages (from gradio==3.33.0->-r requirements.txt (line 10)) (3.7.1)\n",
            "Requirement already satisfied: mdit-py-plugins<=0.3.3 in /usr/local/lib/python3.10/dist-packages (from gradio==3.33.0->-r requirements.txt (line 10)) (0.3.3)\n",
            "Requirement already satisfied: orjson in /usr/local/lib/python3.10/dist-packages (from gradio==3.33.0->-r requirements.txt (line 10)) (3.9.9)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from gradio==3.33.0->-r requirements.txt (line 10)) (1.5.3)\n",
            "Requirement already satisfied: pillow in /usr/local/lib/python3.10/dist-packages (from gradio==3.33.0->-r requirements.txt (line 10)) (9.4.0)\n",
            "Requirement already satisfied: pydantic in /usr/local/lib/python3.10/dist-packages (from gradio==3.33.0->-r requirements.txt (line 10)) (1.10.13)\n",
            "Requirement already satisfied: pydub in /usr/local/lib/python3.10/dist-packages (from gradio==3.33.0->-r requirements.txt (line 10)) (0.25.1)\n",
            "Requirement already satisfied: pygments>=2.12.0 in /usr/local/lib/python3.10/dist-packages (from gradio==3.33.0->-r requirements.txt (line 10)) (2.16.1)\n",
            "Requirement already satisfied: python-multipart in /usr/local/lib/python3.10/dist-packages (from gradio==3.33.0->-r requirements.txt (line 10)) (0.0.6)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.10/dist-packages (from gradio==3.33.0->-r requirements.txt (line 10)) (6.0.1)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from gradio==3.33.0->-r requirements.txt (line 10)) (2.31.0)\n",
            "Requirement already satisfied: semantic-version in /usr/local/lib/python3.10/dist-packages (from gradio==3.33.0->-r requirements.txt (line 10)) (2.10.0)\n",
            "Requirement already satisfied: uvicorn>=0.14.0 in /usr/local/lib/python3.10/dist-packages (from gradio==3.33.0->-r requirements.txt (line 10)) (0.23.2)\n",
            "Requirement already satisfied: websockets>=10.0 in /usr/local/lib/python3.10/dist-packages (from gradio==3.33.0->-r requirements.txt (line 10)) (11.0.3)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from nvidia-cublas-cu11==11.10.3.66->torch->-r requirements.txt (line 5)) (67.7.2)\n",
            "Requirement already satisfied: wheel in /usr/local/lib/python3.10/dist-packages (from nvidia-cublas-cu11==11.10.3.66->torch->-r requirements.txt (line 5)) (0.41.2)\n",
            "Requirement already satisfied: typing-inspect in /usr/local/lib/python3.10/dist-packages (from pyre-extensions==0.0.23->xformers==0.0.16->-r requirements.txt (line 7)) (0.9.0)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.10/dist-packages (from diffusers==0.22.0.dev0->-r requirements.txt (line 1)) (6.8.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from diffusers==0.22.0.dev0->-r requirements.txt (line 1)) (3.12.4)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from diffusers==0.22.0.dev0->-r requirements.txt (line 1)) (2023.6.3)\n",
            "Requirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from diffusers==0.22.0.dev0->-r requirements.txt (line 1)) (0.4.0)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers->-r requirements.txt (line 2)) (23.2)\n",
            "Requirement already satisfied: tokenizers<0.15,>=0.14 in /usr/local/lib/python3.10/dist-packages (from transformers->-r requirements.txt (line 2)) (0.14.1)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers->-r requirements.txt (line 2)) (4.66.1)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from accelerate==0.24.0.dev0->-r requirements.txt (line 3)) (5.9.5)\n",
            "Requirement already satisfied: cairocffi in /usr/local/lib/python3.10/dist-packages (from cairosvg->-r requirements.txt (line 9)) (1.6.1)\n",
            "Requirement already satisfied: cssselect2 in /usr/local/lib/python3.10/dist-packages (from cairosvg->-r requirements.txt (line 9)) (0.7.0)\n",
            "Requirement already satisfied: defusedxml in /usr/local/lib/python3.10/dist-packages (from cairosvg->-r requirements.txt (line 9)) (0.7.1)\n",
            "Requirement already satisfied: tinycss2 in /usr/local/lib/python3.10/dist-packages (from cairosvg->-r requirements.txt (line 9)) (1.2.1)\n",
            "Requirement already satisfied: entrypoints in /usr/local/lib/python3.10/dist-packages (from altair>=4.2.0->gradio==3.33.0->-r requirements.txt (line 10)) (0.4)\n",
            "Requirement already satisfied: jsonschema>=3.0 in /usr/local/lib/python3.10/dist-packages (from altair>=4.2.0->gradio==3.33.0->-r requirements.txt (line 10)) (4.19.1)\n",
            "Requirement already satisfied: toolz in /usr/local/lib/python3.10/dist-packages (from altair>=4.2.0->gradio==3.33.0->-r requirements.txt (line 10)) (0.12.0)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from gradio-client>=0.2.4->gradio==3.33.0->-r requirements.txt (line 10)) (2023.6.0)\n",
            "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.10/dist-packages (from markdown-it-py[linkify]>=2.0.0->gradio==3.33.0->-r requirements.txt (line 10)) (0.1.2)\n",
            "Requirement already satisfied: linkify-it-py<3,>=1 in /usr/local/lib/python3.10/dist-packages (from markdown-it-py[linkify]>=2.0.0->gradio==3.33.0->-r requirements.txt (line 10)) (2.0.2)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->gradio==3.33.0->-r requirements.txt (line 10)) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->gradio==3.33.0->-r requirements.txt (line 10)) (2023.3.post1)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->gradio==3.33.0->-r requirements.txt (line 10)) (3.3.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->gradio==3.33.0->-r requirements.txt (line 10)) (3.4)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->gradio==3.33.0->-r requirements.txt (line 10)) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->gradio==3.33.0->-r requirements.txt (line 10)) (2023.7.22)\n",
            "Requirement already satisfied: click>=7.0 in /usr/local/lib/python3.10/dist-packages (from uvicorn>=0.14.0->gradio==3.33.0->-r requirements.txt (line 10)) (8.1.7)\n",
            "Requirement already satisfied: h11>=0.8 in /usr/local/lib/python3.10/dist-packages (from uvicorn>=0.14.0->gradio==3.33.0->-r requirements.txt (line 10)) (0.14.0)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->gradio==3.33.0->-r requirements.txt (line 10)) (23.1.0)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->gradio==3.33.0->-r requirements.txt (line 10)) (6.0.4)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.10/dist-packages (from aiohttp->gradio==3.33.0->-r requirements.txt (line 10)) (4.0.3)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->gradio==3.33.0->-r requirements.txt (line 10)) (1.9.2)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->gradio==3.33.0->-r requirements.txt (line 10)) (1.4.0)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->gradio==3.33.0->-r requirements.txt (line 10)) (1.3.1)\n",
            "Requirement already satisfied: cffi>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from cairocffi->cairosvg->-r requirements.txt (line 9)) (1.16.0)\n",
            "Requirement already satisfied: webencodings in /usr/local/lib/python3.10/dist-packages (from cssselect2->cairosvg->-r requirements.txt (line 9)) (0.5.1)\n",
            "Requirement already satisfied: anyio<4.0.0,>=3.7.1 in /usr/local/lib/python3.10/dist-packages (from fastapi->gradio==3.33.0->-r requirements.txt (line 10)) (3.7.1)\n",
            "Requirement already satisfied: starlette<0.28.0,>=0.27.0 in /usr/local/lib/python3.10/dist-packages (from fastapi->gradio==3.33.0->-r requirements.txt (line 10)) (0.27.0)\n",
            "Requirement already satisfied: httpcore<0.19.0,>=0.18.0 in /usr/local/lib/python3.10/dist-packages (from httpx->gradio==3.33.0->-r requirements.txt (line 10)) (0.18.0)\n",
            "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from httpx->gradio==3.33.0->-r requirements.txt (line 10)) (1.3.0)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.10/dist-packages (from importlib-metadata->diffusers==0.22.0.dev0->-r requirements.txt (line 1)) (3.17.0)\n",
            "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->gradio==3.33.0->-r requirements.txt (line 10)) (1.1.1)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib->gradio==3.33.0->-r requirements.txt (line 10)) (0.12.1)\n",
            "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib->gradio==3.33.0->-r requirements.txt (line 10)) (4.43.1)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->gradio==3.33.0->-r requirements.txt (line 10)) (1.4.5)\n",
            "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->gradio==3.33.0->-r requirements.txt (line 10)) (3.1.1)\n",
            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<4.0.0,>=3.7.1->fastapi->gradio==3.33.0->-r requirements.txt (line 10)) (1.1.3)\n",
            "Requirement already satisfied: pycparser in /usr/local/lib/python3.10/dist-packages (from cffi>=1.1.0->cairocffi->cairosvg->-r requirements.txt (line 9)) (2.21)\n",
            "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=3.0->altair>=4.2.0->gradio==3.33.0->-r requirements.txt (line 10)) (2023.7.1)\n",
            "Requirement already satisfied: referencing>=0.28.4 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=3.0->altair>=4.2.0->gradio==3.33.0->-r requirements.txt (line 10)) (0.30.2)\n",
            "Requirement already satisfied: rpds-py>=0.7.1 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=3.0->altair>=4.2.0->gradio==3.33.0->-r requirements.txt (line 10)) (0.10.6)\n",
            "Requirement already satisfied: uc-micro-py in /usr/local/lib/python3.10/dist-packages (from linkify-it-py<3,>=1->markdown-it-py[linkify]>=2.0.0->gradio==3.33.0->-r requirements.txt (line 10)) (1.0.2)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas->gradio==3.33.0->-r requirements.txt (line 10)) (1.16.0)\n",
            "Requirement already satisfied: mypy-extensions>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from typing-inspect->pyre-extensions==0.0.23->xformers==0.0.16->-r requirements.txt (line 7)) (1.0.0)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from diffusers import StableDiffusionControlNetPipeline, ControlNetModel\n",
        "from diffusers import UniPCMultistepScheduler\n",
        "import cv2\n",
        "import gradio as gr\n",
        "import numpy as np\n",
        "import torch\n",
        "from PIL import Image\n",
        "import re\n",
        "import openai\n",
        "from cairosvg import svg2png\n",
        "\n",
        "# Constants\n",
        "low_threshold = 100\n",
        "high_threshold = 200\n",
        "\n",
        "# Models\n",
        "controlnet = ControlNetModel.from_pretrained(\"lllyasviel/sd-controlnet-canny\", torch_dtype=torch.float16)\n",
        "pipe = StableDiffusionControlNetPipeline.from_pretrained(\n",
        "    \"runwayml/stable-diffusion-v1-5\", controlnet=controlnet, safety_checker=None, torch_dtype=torch.float16\n",
        ")\n",
        "pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config)\n",
        "\n",
        "# This command loads the individual model components on GPU on-demand. So, we don't\n",
        "# need to explicitly call pipe.to(\"cuda\").\n",
        "pipe.enable_model_cpu_offload()\n",
        "\n",
        "pipe.enable_xformers_memory_efficient_attention()\n",
        "\n",
        "# Generator seed,\n",
        "generator = torch.manual_seed(0)\n",
        "\n",
        "def get_canny_filter(image):\n",
        "    if not isinstance(image, np.ndarray):\n",
        "        image = np.array(image)\n",
        "\n",
        "    image = cv2.Canny(image, low_threshold, high_threshold)\n",
        "    image = image[:, :, None]\n",
        "    image = np.concatenate([image, image, image], axis=2)\n",
        "    canny_image = Image.fromarray(image)\n",
        "    return canny_image\n",
        "\n",
        "\n",
        "def generate_images(image, prompt):\n",
        "    canny_image = get_canny_filter(image)\n",
        "    output = pipe(\n",
        "        prompt,\n",
        "        canny_image,\n",
        "        generator=generator,\n",
        "        num_images_per_prompt=3,\n",
        "        num_inference_steps=20,\n",
        "    )\n",
        "    all_outputs = []\n",
        "    all_outputs.append(canny_image)\n",
        "    for image in output.images:\n",
        "        all_outputs.append(image)\n",
        "    return all_outputs\n",
        "\n",
        "# GPT-4 control\n",
        "\n",
        "def gpt_control(apikey, prompt):\n",
        "\n",
        "    openai.api_key = apikey\n",
        "\n",
        "    # gpt\n",
        "    messages = [{\"role\": \"system\", \"content\": \"You are an SVG expert with years of experience and multiple contributions to the SVG project. Based on the prompt and the description, please generate the corresponding SVG code.\"},\n",
        "     {\"role\": \"user\", \"content\": f\"\"\"Provide only the shell command without any explanations.\n",
        "The current objective is below. Reply with the SVG code only:\n",
        "OBJECTIVE: {prompt}\n",
        "YOUR SVG CODE:\n",
        "\"\"\"}]\n",
        "\n",
        "    completion = openai.ChatCompletion.create(\n",
        "      model = \"gpt-4\",\n",
        "      messages = messages\n",
        "    )\n",
        "\n",
        "    chat_response = completion.choices[0].message.content\n",
        "\n",
        "    code = re.findall('<.*>', chat_response)\n",
        "    code_new = '\\n'.join(code)\n",
        "\n",
        "    svg_code = f\"\"\"\n",
        "    {code_new}\n",
        "    \"\"\"\n",
        "    svg2png(bytestring=svg_code,write_to='output.png')\n",
        "\n",
        "    return 'output.png'\n",
        "\n",
        "with gr.Blocks(theme=\"JohnSmith9982/small_and_pretty\") as app:\n",
        "  gr.HTML(\"<center>\"\n",
        "          \"<h1>🌁🪄🌃 - ControlNet with GPT-4</h1>\"\n",
        "          \"</center>\")\n",
        "\n",
        "  gr.Markdown(\"## <center>🌟 Born to Create: Controllable Text-to-Image Generation with GPT-4</center>\")\n",
        "\n",
        "  with gr.Row():\n",
        "    with gr.Column():\n",
        "      inp1 = gr.Textbox(label=\"OpenAI API Key\", type=\"password\")\n",
        "      inp2 = gr.Textbox(label=\"Position Prompt (as simple as possible)\")\n",
        "      inp3 = gr.Textbox(label=\"Image Prompt (make it shine)\")\n",
        "      btn1 = gr.Button(\"GPT-4 Control\", variant=\"primary\")\n",
        "      btn2 = gr.Button(\"Generate\", variant=\"primary\")\n",
        "    with gr.Column():\n",
        "      out1 = gr.Image(label=\"Output Image\", type=\"pil\")\n",
        "      out2 = gr.Gallery(label=\"Generated Images\").style(grid=[2], height=\"auto\")\n",
        "  btn1.click(gpt_control, [inp1, inp2], [out1])\n",
        "  btn2.click(generate_images, [out1, inp3], [out2])\n",
        "\n",
        "app.launch(show_error=True, share=True)"
      ],
      "metadata": {
        "id": "a3pnNrqyMqrb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 697,
          "referenced_widgets": [
            "bd3ba58ef21f4125b702ed9b01e524b0",
            "a993e2667bae40febbad769e54489ef3",
            "98628783a7e14033ace2af4fbb3e8107",
            "e6ca7a8529384052a33ad90e5d114ec4",
            "fa3d627474f342aaa8e3da841d65c903",
            "4c08a47828c148c6bfbb5477432269af",
            "9c47510793ee49bea9590dc23f153703",
            "b8612aa5533a4c6eb29185903c132055",
            "27e2fb8a96054127bc0a3e666c4d2821",
            "282db119ce57419887295e59e78d695c",
            "7715573a22454048b1c2d40049f67ccf"
          ]
        },
        "outputId": "fa548beb-7995-42ff-af13-eabaaca16227"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Loading pipeline components...:   0%|          | 0/6 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "bd3ba58ef21f4125b702ed9b01e524b0"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "You have disabled the safety checker for <class 'diffusers.pipelines.controlnet.pipeline_controlnet.StableDiffusionControlNetPipeline'> by passing `safety_checker=None`. Ensure that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered results in services or applications open to the public. Both the diffusers team and Hugging Face strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling it only for use-cases that involve analyzing network behavior or auditing its results. For more information, please have a look at https://github.com/huggingface/diffusers/pull/254 .\n",
            "/usr/local/lib/python3.10/dist-packages/gradio/components.py:4505: UserWarning: The 'grid' parameter will be deprecated. Please use 'columns' instead.\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Colab notebook detected. To show errors in colab notebook, set debug=True in launch()\n",
            "Running on public URL: https://b80aaad075cccf8f43.gradio.live\n",
            "\n",
            "This share link expires in 72 hours. For free permanent hosting and GPU upgrades, run `gradio deploy` from Terminal to deploy to Spaces (https://huggingface.co/spaces)\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "<div><iframe src=\"https://b80aaad075cccf8f43.gradio.live\" width=\"100%\" height=\"500\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": []
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "V-VROejfAkXX"
      },
      "execution_count": 4,
      "outputs": []
    }
  ]
}