diff --git a/.github/workflows/build_docker_images.yml b/.github/workflows/build_docker_images.yml
index 5b93f80dcd..ce5577309b 100644
--- a/.github/workflows/build_docker_images.yml
+++ b/.github/workflows/build_docker_images.yml
@@ -10,6 +10,8 @@ concurrency:
   group: docker-image-builds
   cancel-in-progress: false
 
+permissions: {}
+
 env:
   CI_SLACK_CHANNEL: ${{ secrets.CI_DOCKER_CHANNEL }}
 
diff --git a/.github/workflows/build_documentation.yml b/.github/workflows/build_documentation.yml
index 1ff01d1a5e..42e7972bc2 100644
--- a/.github/workflows/build_documentation.yml
+++ b/.github/workflows/build_documentation.yml
@@ -7,6 +7,8 @@ on:
       - doc-builder*
       - v*-release
 
+permissions: {}
+
 jobs:
    build:
     uses: huggingface/doc-builder/.github/workflows/build_main_documentation.yml@main
diff --git a/.github/workflows/build_pr_documentation.yml b/.github/workflows/build_pr_documentation.yml
index 35ceab6e60..3fe27e8a04 100644
--- a/.github/workflows/build_pr_documentation.yml
+++ b/.github/workflows/build_pr_documentation.yml
@@ -7,6 +7,8 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
 
+permissions: {}
+
 jobs:
   build:
     uses: huggingface/doc-builder/.github/workflows/build_pr_documentation.yml@main
diff --git a/.github/workflows/integrations_tests.yml b/.github/workflows/integrations_tests.yml
index 38ab96246e..3d61c8d915 100644
--- a/.github/workflows/integrations_tests.yml
+++ b/.github/workflows/integrations_tests.yml
@@ -7,6 +7,8 @@ on:
         description: 'Branch to test on'
         required: true
 
+permissions: {}
+
 jobs:
   run_transformers_integration_tests:
     strategy:
diff --git a/.github/workflows/nightly-bnb.yml b/.github/workflows/nightly-bnb.yml
index 0fba12dfb9..bc68af80c8 100644
--- a/.github/workflows/nightly-bnb.yml
+++ b/.github/workflows/nightly-bnb.yml
@@ -12,6 +12,7 @@ env:
   NVIDIA_DISABLE_REQUIRE: "1"
   SLACK_API_TOKEN: ${{ secrets.SLACK_API_TOKEN }}
 
+permissions: {}
 
 jobs:
   run_all_tests_single_gpu:
diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
index 7e6635b392..d578900489 100644
--- a/.github/workflows/nightly.yml
+++ b/.github/workflows/nightly.yml
@@ -12,6 +12,7 @@ env:
   NVIDIA_DISABLE_REQUIRE: "1"
   SLACK_API_TOKEN: ${{ secrets.SLACK_API_TOKEN }}
 
+permissions: {}
 
 jobs:
   run_all_tests_single_gpu:
diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml
index fc65794663..054c4b53c4 100644
--- a/.github/workflows/stale.yml
+++ b/.github/workflows/stale.yml
@@ -4,6 +4,8 @@ on:
   schedule:
     - cron: "0 15 * * *"
 
+permissions: {}
+
 jobs:
   close_stale_issues:
     name: Close Stale Issues
diff --git a/.github/workflows/test-docker-build.yml b/.github/workflows/test-docker-build.yml
index 33a177bba2..558c5f74de 100644
--- a/.github/workflows/test-docker-build.yml
+++ b/.github/workflows/test-docker-build.yml
@@ -5,6 +5,9 @@ on:
     paths:
       # Run only when DockerFile files are modified
       - "docker/*/Dockerfile"
+
+permissions: {}
+
 jobs:
   get_changed_files:
     name: "Build all modified docker images"
diff --git a/.github/workflows/tests-main.yml b/.github/workflows/tests-main.yml
index 1b06083e73..d614d547b7 100644
--- a/.github/workflows/tests-main.yml
+++ b/.github/workflows/tests-main.yml
@@ -6,6 +6,8 @@ on:
     paths-ignore:
         - 'docs/**'
 
+permissions: {}
+
 jobs:
   tests:
     runs-on: ubuntu-latest
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index d96f1f6f31..f89e9d7243 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -9,6 +9,8 @@ on:
     paths-ignore:
       - 'docs/**'
 
+permissions: {}
+
 jobs:
   check_code_quality:
     runs-on: ubuntu-latest
diff --git a/.github/workflows/torch_compile_tests.yml b/.github/workflows/torch_compile_tests.yml
index f93d3760d6..02243de643 100644
--- a/.github/workflows/torch_compile_tests.yml
+++ b/.github/workflows/torch_compile_tests.yml
@@ -17,6 +17,8 @@ env:
   # To be able to run tests on CUDA 12.2
   NVIDIA_DISABLE_REQUIRE: "1"
 
+permissions: {}
+
 jobs:
   run_tests_with_compile:
     runs-on:
diff --git a/.github/workflows/trufflehog.yml b/.github/workflows/trufflehog.yml
index 9a613bb5b7..bdcdac7561 100644
--- a/.github/workflows/trufflehog.yml
+++ b/.github/workflows/trufflehog.yml
@@ -3,6 +3,8 @@ on:
 
 name: Secret Leaks
 
+permissions: {}
+
 jobs:
   trufflehog:
     runs-on: ubuntu-latest
diff --git a/.github/workflows/upload_pr_documentation.yml b/.github/workflows/upload_pr_documentation.yml
index 380f67550d..7659af7e5c 100644
--- a/.github/workflows/upload_pr_documentation.yml
+++ b/.github/workflows/upload_pr_documentation.yml
@@ -6,6 +6,8 @@ on:
     types:
       - completed
 
+permissions: {}
+
 jobs:
   build:
     uses: huggingface/doc-builder/.github/workflows/upload_pr_documentation.yml@main
diff --git a/.github/workflows/zizmor.yaml b/.github/workflows/zizmor.yaml
index c51ef5abc4..502c4f8d62 100644
--- a/.github/workflows/zizmor.yaml
+++ b/.github/workflows/zizmor.yaml
@@ -8,6 +8,8 @@ on:
     paths:
       - '.github/**'
 
+permissions: {}
+
 jobs:
   zizmor:
     name: zizmor latest via Cargo
diff --git a/.github/zizmor.yml b/.github/zizmor.yml
index 8e0fba3157..1746cfe25f 100644
--- a/.github/zizmor.yml
+++ b/.github/zizmor.yml
@@ -8,8 +8,8 @@ rules:
       # the docker buildx binary is cached and zizmor warns about a cache poisoning attack.
       # OTOH this cache would make us more resilient against an intrusion on docker-buildx' side.
       # There is no obvious benefit so we leave it as it is.
-      - build_docker_images.yml:35:9
-      - build_docker_images.yml:68:9
-      - build_docker_images.yml:101:9
-      - build_docker_images.yml:134:9
-      - build_docker_images.yml:167:9
+      - build_docker_images.yml:37:9
+      - build_docker_images.yml:70:9
+      - build_docker_images.yml:103:9
+      - build_docker_images.yml:136:9
+      - build_docker_images.yml:169:9