Debugpy support for unit tests (#715)

* debug tests!

* Move debugpy logic into makefile

.env.template

@@ -42,4 +42,4 @@ RAY_DASHBOARD_PORT=${RAY_DASHBOARD_PORT:-8265}
 RAY_WORKER_GPUS=${RAY_WORKER_GPUS:-0}
 RAY_WORKER_GPUS_FRACTION=${RAY_WORKER_GPUS_FRACTION:-0}
 NVIDIA_VISIBLE_DEVICES=${NVIDIA_VISIBLE_DEVICES:-all}
-GPU_COUNT=${GPU_COUNT:-0}
+GPU_COUNT=${GPU_COUNT:-0}

.vscode/launch.json

@@ -19,6 +19,16 @@
                     "remoteRoot": "/mzai"
                 }
             ]
+        },
+        {
+            "name": "Attach to Tests",
+            "type": "debugpy",
+            "request": "attach",
+            "justMyCode":  false,
+            "connect": {
+                "host": "localhost",
+                "port": 5679
+            }
         }
     ]
 }

Makefile

@@ -15,6 +15,11 @@ RAY_WORKER_GPUS ?= 0
 RAY_WORKER_GPUS_FRACTION ?= 0.0
 GPU_COMPOSE :=
 
+DEBUGPY_ARGS :=
+ifneq ($(shell echo $(DEBUGPY) | grep -i '^true$$'),)
+    DEBUGPY_ARGS := -m debugpy --listen 5679 --wait-for-client
+endif
+
 $(info RAY_WORKER_GPUS = $(RAY_WORKER_GPUS))
 
 ifeq ($(ARCH), arm64)
@@ -132,11 +137,11 @@ clean-all: clean-docker-buildcache clean-docker-containers
 # start them if they are not present or use the currently running ones.
 test-sdk-unit:
 	cd lumigator/python/mzai/sdk/tests; \
-	uv run pytest -o python_files="unit/*/test_*.py unit/test_*.py"
+	uv run $(DEBUGPY_ARGS) -m pytest -o python_files="unit/*/test_*.py unit/test_*.py"
 
 test-sdk-integration:
 	cd lumigator/python/mzai/sdk/tests; \
-	uv run pytest -s -o python_files="integration/test_*.py integration/*/test_*.py"
+	uv run $(DEBUGPY_ARGS) -m pytest -s -o python_files="integration/test_*.py integration/*/test_*.py"
 
 test-sdk-integration-containers:
 ifeq ($(CONTAINERS_RUNNING),)
@@ -160,7 +165,7 @@ test-backend-unit:
 	RAY_DASHBOARD_PORT=8265 \
 	SQLALCHEMY_DATABASE_URL=sqlite:////tmp/local.db \
 	PYTHONPATH=../jobs:$$PYTHONPATH \
-	uv run pytest -s -o python_files="backend/tests/unit/*/test_*.py backend/tests/unit/test_*.py"
+	uv run $(DEBUGPY_ARGS) -m pytest -s -o python_files="backend/tests/unit/*/test_*.py backend/tests/unit/test_*.py"
 
 test-backend-integration:
 	cd lumigator/python/mzai/backend/; \
@@ -178,7 +183,7 @@ test-backend-integration:
 	EVALUATOR_LITE_PIP_REQS=../jobs/evaluator_lite/requirements.txt \
 	EVALUATOR_LITE_WORK_DIR=../jobs/evaluator_lite \
 	PYTHONPATH=../jobs:$$PYTHONPATH \
-	uv run pytest -s -o python_files="backend/tests/integration/*/test_*.py"
+	uv run $(DEBUGPY_ARGS) -m pytest -s -o python_files="backend/tests/integration/*/test_*.py"
 
 test-backend-integration-containers:
 ifeq ($(CONTAINERS_RUNNING),)
@@ -196,11 +201,11 @@ test-backend: test-backend-unit test-backend-integration-containers
 # with all the deps specified in their respective `requirements.txt` files.
 test-jobs-evaluation-unit:
 	cd lumigator/python/mzai/jobs/evaluator_lite; \
-	uv run --with pytest --with-requirements requirements.txt --isolated pytest
+	uv run $(DEBUGPY_ARGS) --with pytest --with-requirements requirements.txt --isolated pytest
 
 test-jobs-inference-unit:
 	cd lumigator/python/mzai/jobs/inference; \
-	uv run --with pytest --with-requirements requirements.txt --isolated pytest
+	uv run $(DEBUGPY_ARGS) --with pytest --with-requirements requirements.txt --isolated pytest
 
 test-jobs-unit: test-jobs-evaluation-unit test-jobs-inference-unit
 

docs/source/operations-guide/dev.md

@@ -146,3 +146,10 @@ To fix linting issues automatically:
 ```console
 user@host:~/lumigator/lumigator/frontend$ npm run lint:fix
 ```
+
+
+### Debugging with VSCode and DebugPy
+
+The backend and unit tests can be debugged with VsCode using Debugpy. If you're not familiar with how debugging python in vscode works, we recommend you read VSCode's documentation [here](https://code.visualstudio.com/docs/python/debugging). When running the command `make local-up`, the backend automatically creates the debugpy listener, which you can attach to using the `Attach to Backend` Debug configuration present in the `.vscode/launch.json` file of this repo.
+
+Similarly, the unit tests can be debugged by adding the parameter `DEBUGPY=true` when running a make test command. For example, `make test-sdk-unit DEBUGPY=true` will run the test with the debugger and you can attach using the `Attach to Tests` Debug configuration. Note that with this enabled, the unit tests wait for you to attach the debugger before running any tests.

lumigator/python/mzai/sdk/pyproject.toml

@@ -14,6 +14,7 @@ dependencies = [
 dev-dependencies = [
     "pytest>=8.3.3",
     "requests-mock>=1.12.1",
+    "debugpy>=1.8.12",
 ]
 
 [tool.uv.sources]

pyproject.toml

@@ -18,3 +18,11 @@ docs = [
     "pydantic>=2.8.0",
     "loguru==0.7.2",
 ]
+
+[dependency-groups]
+dev = [
+    "alembic>=1.14.1",
+    "debugpy>=1.8.12",
+    "fastapi>=0.115.6",
+    "loguru>=0.7.2",
+]