diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
index 57caba1c61..d0c537a947 100644
--- a/.github/workflows/nightly.yml
+++ b/.github/workflows/nightly.yml
@@ -8,6 +8,8 @@ on:
 env:
   RUN_SLOW: "yes"
   IS_GITHUB_CI: "1"
+  # To be able to run tests on CUDA 12.2
+  NVIDIA_DISABLE_REQUIRE: "1"
   SLACK_API_TOKEN: ${{ secrets.SLACK_API_TOKEN }}
 
 
diff --git a/docker/peft-gpu/Dockerfile b/docker/peft-gpu/Dockerfile
index 375e9a65e3..9b538d218d 100644
--- a/docker/peft-gpu/Dockerfile
+++ b/docker/peft-gpu/Dockerfile
@@ -40,7 +40,7 @@ RUN source activate peft && \
     peft[test]@git+https://github.com/huggingface/peft
 
 # Stage 2
-FROM nvidia/cuda:11.8.0-devel-ubuntu22.04 AS build-image
+FROM nvidia/cuda:12.2.2-devel-ubuntu22.04 AS build-image
 COPY --from=compile-image /opt/conda /opt/conda
 ENV PATH /opt/conda/bin:$PATH