diff --git a/.github/workflows/hpu-gaudi2.yml b/.github/workflows/hpu-gaudi2.yml
index a64a337d50af2..bccdb735724c3 100644
--- a/.github/workflows/hpu-gaudi2.yml
+++ b/.github/workflows/hpu-gaudi2.yml
@@ -26,6 +26,14 @@ jobs:
         - 80
       options: --runtime=habana -e HABANA_VISIBLE_DEVICES=all -e OMPI_MCA_btl_vader_single_copy_mechanism=none --cap-add=sys_nice
 
+    env:
+      PT_HPU_LAZY_MODE: 0
+      TEST_LIST: |
+        test_accelerator.py
+        test_autotuning.py
+        test_compression.py
+        test_dist.py
+
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
@@ -38,6 +46,13 @@ jobs:
           python -c "import torch; print('torch:', torch.__version__, torch)"
           python -c "import torch; print('CUDA available:', torch.cuda.is_available())"
 
+      - name: Install transformers
+        run: |
+          git clone https://github.com/huggingface/transformers
+          cd transformers
+          git rev-parse --short HEAD
+          pip install .
+
       - name: Install deepspeed
         run: |
           pip install .[dev]
@@ -46,3 +61,25 @@ jobs:
       - name: Python environment
         run: |
           pip list
+
+      - name: Set up test list
+        run: |
+          # Convert multiline TEST_LIST to an array
+          IFS=$'\n' read -r -d '' -a tests <<< "$TEST_LIST"
+
+          # Construct the test list with 'or' operator
+          for test in "${tests[@]}"; do
+            TEST_STRING+="$test or "
+          done;
+
+          # Remove the trailing 'or' and leading/trailing whitespace
+          TEST_LIST=${TEST_STRING% or }
+          TEST_LIST=${TEST_LIST# }
+
+      - name: Unit tests
+        run: |
+          unset TORCH_CUDA_ARCH_LIST # only jit compile for current arch
+          cd tests
+          env
+          echo ${TEST_LIST}
+          echo ${PT_HPU_LAZY_MODE}