Configure test endpoint so it calls tests within docker container

Dockerfile

@@ -10,6 +10,26 @@ RUN apt-get update && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
+# Install required packages for adding Docker repository
+RUN apt-get update && \
+    apt-get install -y \
+    ca-certificates \
+    curl \
+    gnupg \
+    lsb-release \
+    apt-transport-https \
+    software-properties-common
+
+# Add Docker’s official GPG key
+RUN curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
+
+# Set up the Docker repository
+RUN echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" > /etc/apt/sources.list.d/docker.list
+
+# Install Docker CLI
+RUN apt-get update && \
+    apt-get install -y docker-ce-cli
+
 RUN pip install virtualenv
 
 # Copy the requirements file into the container
@@ -18,12 +38,6 @@ COPY requirements.txt ./
 # Install any needed packages specified in requirements.txt
 RUN pip install --trusted-host pypi.python.org -r requirements.txt
 
-# Create a virtual environment for the target repo and install its requirements if available
-# This must be defined in a file llm_target_repo_requirements.txt
-RUN virtualenv /venv
-COPY ./llm_target_repo_requirements.txt /llm_target_repo_requirements.txt
-RUN if [ -f /llm_target_repo_requirements.txt ]; then /venv/bin/pip install -r /llm_target_repo_requirements.txt; fi
-
 # Copy the rest of the application code
 COPY . .
 

README.md

@@ -42,14 +42,17 @@ To get started with LLM Repo Assistant, follow these steps:
 2. Clone the LLM Repo Assistant repo
 3. Add an `.llmignore` file to the root directory of your target repo - this tells
 LLM Repo Assistant about files in your target repo that you want ignored by the LLM.
+3. Build and run the docker container for your code repo, and add the name of 
+the container to `TARGET_REPO_DOCKER_IMAGE_NAME` in `.env`
+in the LLM Repo Assistant root directory. (i.e. `TARGET_REPO_DOCKER_IMAGE_NAME=insert-your-container-name`)
 4. If you want the API to be able to run tests on your target repo,
 then edit the requirements file to the root directory of the LLM Repo Assistant repo
 called `llm_target_repo_requirements.txt`. This file should define the python environment
 your tests should run in and should include `pytest`.
 5. Navigate to the `llm_repo_assistant` cloned repository and build the
 Docker image to run LLM Repo Assistant with `docker build -t llm_repo_assistant .`
 6. Run LLM Repo Assistant locally in a docker container with the following command:
-`docker run --rm -v "/path/to/cloned/repo/llm_repo_assistant:/app" -v "/path/to/your/code/repo:/repo" -p 8000:8000 --name llm_repo_assistant llm_repo_assistant`
+`docker run --rm -v "/var/run/docker.sock:/var/run/docker.sock" -v "/path/to/cloned/repo/llm_repo_assistant:/app" -v "/path/to/your/code/repo:/repo" -p 8000:8000 --name llm_repo_assistant llm_repo_assistant`
 7. View and test the API endpoints by visiting `localhost:8000/docs`
 
 ### Adding ChatGPT plugin

llm_target_repo_requirements.txt

@@ -1,36 +1,2 @@
-anyio==3.6.2
-astunparse==1.6.3
-certifi==2022.12.7
-click==8.1.3
-dnspython==2.3.0
-email-validator==2.0.0.post1
-exceptiongroup==1.1.1
-fastapi==0.95.1
-h11==0.14.0
-httpcore==0.17.0
-httptools==0.5.0
-httpx==0.24.0
-idna==3.4
-iniconfig==2.0.0
-itsdangerous==2.1.2
-Jinja2==3.1.2
-MarkupSafe==2.1.2
-orjson==3.8.10
-packaging==23.1
-pluggy==1.0.0
-pydantic==1.10.7
 pytest==7.3.1
-python-dotenv==1.0.0
-python-multipart==0.0.6
-PyYAML==6.0
-six==1.16.0
-sniffio==1.3.0
-starlette==0.26.1
-tomli==2.0.1
-typing-extensions==4.5.0
-ujson==5.7.0
-uvicorn==0.21.1
-uvloop==0.17.0
-watchfiles==0.19.0
-websockets==11.0.2
 emoji

requirements.txt

@@ -1,8 +1,10 @@
 anyio==3.6.2
 astunparse==1.6.3
 certifi==2022.12.7
+charset-normalizer==3.1.0
 click==8.1.3
 dnspython==2.3.0
+docker==6.0.1
 email-validator==2.0.0.post1
 exceptiongroup==1.1.1
 fastapi==0.95.1
@@ -23,13 +25,16 @@ pytest==7.3.1
 python-dotenv==1.0.0
 python-multipart==0.0.6
 PyYAML==6.0
+requests==2.29.0
 six==1.16.0
 sniffio==1.3.0
 starlette==0.26.1
 tomli==2.0.1
 typing-extensions==4.5.0
 ujson==5.7.0
+urllib3==1.26.15
 uvicorn==0.21.1
 uvloop==0.17.0
 watchfiles==0.19.0
+websocket-client==1.5.1
 websockets==11.0.2

src/api/endpoints/command.py

@@ -2,13 +2,14 @@
 import sys
 from enum import Enum
 import subprocess
-from typing import Optional, List
+from typing import Optional, List, Tuple
 from fastapi import HTTPException, APIRouter
 from contextlib import contextmanager
+import docker
 
 from src.schemas import TestRunRequest
 from src.core.config import settings
-from src.utils import get_filesystem_path
+from src.utils import get_filesystem_path, run_command_in_image
 
 
 router = APIRouter()
@@ -18,8 +19,9 @@ class TestFramework(str, Enum):
     pytest = "pytest"
 
 def run_tests(test_command: List[str], 
+              docker_image_name: str,
               test_file_path: Optional[str] = None,
-              test_function_name: Optional[str] = None) -> str:
+              test_function_name: Optional[str] = None) -> Tuple[int, str]:
     if test_file_path and not test_function_name:
         test_command.extend([test_file_path])
     if test_function_name and not test_file_path:
@@ -31,47 +33,37 @@ def run_tests(test_command: List[str],
     os.chdir(settings.REPO_ROOT)
 
     try:
-        result = subprocess.run(test_command, check=True, text=True, capture_output=True)
-        return result.stdout
+        #result = subprocess.run(test_command, check=True, text=True, capture_output=True)
+        #return result.stdout
+        exit_code, output_str = run_command_in_image(docker_image_name, test_command)
+        return exit_code, output_str
     except subprocess.CalledProcessError as e:
         return e.stdout + e.stderr
     finally:
         os.chdir(original_cwd)
 
-@contextmanager
-def activate_virtualenv(venv_path):
-    """Temporarily add the virtual environment to sys.path."""
-    original_sys_path = sys.path.copy()
-    sys.path.insert(0, venv_path + "/lib/python3.9/site-packages")
-    try:
-        yield
-    finally:
-        sys.path = original_sys_path
-
 @router.post("/run_tests/{test_framework}")
 async def run_tests_endpoint(test_framework: TestFramework, test_run_request: TestRunRequest):
     """
     Runs tests for repo. Endpoint executes tests with optional specification of test file and test function.
     If no test file or function is specified, all available tests are run. Currently only `pytest` is supported.
     """
 
-    venv_path = "/venv"
-
     test_file_path = ""
     if test_run_request.test_file_path:
         test_file_path = get_filesystem_path(test_run_request.test_file_path)
 
     if test_framework.lower() == "pytest":
-        if not os.path.exists(venv_path + "/lib/python3.9/site-packages"):
-            error_message = """
-            Virtual environment for target repo not found. Provide a 'llm_target_repo_requirements.txt' file in your
-            target repo to use the run_tests endpoint and re-build the docker container. 
-            """
-            raise HTTPException(status_code=400, detail=error_message)
-        test_command = [os.path.join(venv_path, "bin", "python"), "-m", "pytest"]
+        test_command = ["python", "-m", "pytest"]
     else:
         raise HTTPException(status_code=400, detail="Unsupported test framework")
 
-    test_output = run_tests(test_command, test_file_path, test_run_request.test_function_name)
+    try:
+        test_output = run_tests(test_command,
+                                settings.TARGET_REPO_DOCKER_IMAGE_NAME,
+                                test_file_path,
+                                test_run_request.test_function_name)
+    except docker.errors.ImageNotFound:
+        raise HTTPException(status_code=400, detail=f"Target repo docker image with name '{settings.TARGET_REPO_DOCKER_IMAGE_NAME}' not found.")
     return {"status": "success", "test_output": test_output}
 

src/core/config.py

@@ -29,6 +29,10 @@ def assemble_cors_origins(cls, v: Union[str, List[str]]) -> Union[List[str], str
 
     LLMIGNORE_PATH: Optional[FilePath] = None
 
+    TARGET_REPO_DOCKER_IMAGE_NAME: str
+
+    TARGET_REPO_PATH: str
+
     class Config:
         env_file = ".env"
         case_sensitive = True

src/utils.py

@@ -6,6 +6,8 @@
 from typing import List, Dict, Any
 from fastapi import HTTPException
 import fnmatch
+import docker
+from typing import Tuple
 
 from src.core.config import settings
 
@@ -157,3 +159,30 @@ def get_git_diff(directory: str) -> str:
     )
     return result.stdout
 
+
+def run_command_in_image(image_name: str, command: List[str]) -> Tuple[int, str]:
+    client = docker.from_env()
+
+    # Run the command in a new container
+    container = client.containers.run(image=image_name, 
+                                      command=command, 
+                                      detach=True,
+                                      working_dir=settings.REPO_ROOT,
+                                      volumes={settings.TARGET_REPO_PATH: {"bind": settings.REPO_ROOT, "mode": "rw"}})
+
+    # Wait for the container to finish executing
+    result = container.wait()
+
+    # Fetch the logs (standard output and standard error)
+    output = container.logs()
+
+    # Clean up the container
+    container.remove()
+
+    # Decode the output to a string
+    output_str = output.decode("utf-8")
+
+    # The result contains a dictionary with status and StatusCode (exit code)
+    exit_code = result['StatusCode']
+
+    return exit_code, output_str