diff --git a/resources/flakiness_report.py b/resources/flakiness_report.py
index dbadd654..708cbe80 100644
--- a/resources/flakiness_report.py
+++ b/resources/flakiness_report.py
@@ -64,19 +64,25 @@ def parse_options() -> argparse.Namespace:
 
 if __name__ == "__main__":
 
-    args = parse_options()
+    try:
+        args = parse_options()
 
-    archive = FlakyTestsArchive(
-        args.junit_archive,
-        args.window_size,
-        args.delete_old_files
-    )
+        archive = FlakyTestsArchive(
+            args.junit_archive,
+            args.window_size,
+            args.delete_old_files
+        )
+
+        if args.markdown_file:
+            FlakyTestsMdPublisher.publish(archive, args.markdown_file)
 
-    if args.markdown_file:
-        FlakyTestsMdPublisher.publish(archive, args.markdown_file)
+        if args.json_file:
+            FlakyTestsJSONPublisher.publish(archive, args.json_file)
 
-    if args.json_file:
-        FlakyTestsJSONPublisher.publish(archive, args.json_file)
+        ret = 0 if archive.flaky_test_count == 0 else 1
+        exit(ret)
 
-    ret = 0 if archive.flaky_test_count == 0 else 1
-    exit(ret)
+    except Exception as e:
+        # Exit with 255 error so that the action can fail
+        print(e)
+        exit(255)
diff --git a/ubuntu/flakiness_report/action.yml b/ubuntu/flakiness_report/action.yml
index a6223e5d..39d2aad1 100644
--- a/ubuntu/flakiness_report/action.yml
+++ b/ubuntu/flakiness_report/action.yml
@@ -87,7 +87,8 @@ runs:
 
         echo "Flakiness report generated"
 
-        if [[ "${{ inputs.fail_on_flaky_tests }}" != "True" ]]
+        # A 255 exit code indicates that an exception occurred in the script, we want to fail in that case
+        if [ "${{ inputs.fail_on_flaky_tests }}" != "True" ] && [ "$EXIT_CODE" != "255" ]
         then
           echo "Ignoring flaky tests failures"
           EXIT_CODE=0
diff --git a/windows/flakiness_report/action.yml b/windows/flakiness_report/action.yml
index 15085739..02eaab50 100644
--- a/windows/flakiness_report/action.yml
+++ b/windows/flakiness_report/action.yml
@@ -74,7 +74,9 @@ runs:
 
         $EXIT_CODE = $LASTEXITCODE
 
-        if ("${{ inputs.fail_on_flaky_tests }}" -ne "True") {
+        # A 255 exit code indicates that an exception occurred in the script, we want to fail in that case
+        if ("${{ inputs.fail_on_flaky_tests }}" -ne "True" -and $EXIT_CODE -ne 255) {
+          Write-Output "Ignoring flaky tests failures"
           $EXIT_CODE = 0
         }