diff --git a/src/ansible_sign/checksum/base.py b/src/ansible_sign/checksum/base.py
index 7c14a9d..48b1edd 100644
--- a/src/ansible_sign/checksum/base.py
+++ b/src/ansible_sign/checksum/base.py
@@ -11,7 +11,33 @@ class NoDifferException(Exception):
 
 
 class ChecksumMismatch(Exception):
-    pass
+    def __init__(self, msg, differences={}):
+        super().__init__(msg)
+        self.msg = msg
+        self.differences = differences
+
+    def __str__(self):
+        """
+        This *must* be something human-readable, it is currently used in the AWX
+        action plugin on project sync when validation fails.
+        """
+        out = self.msg
+        extra_info = ""
+        if "changes" in self.differences and self.differences["changes"]:
+            extra_info += f"Files changed: {', '.join(self.differences['changes'])}"
+
+        if "added" in self.differences and self.differences["added"]:
+            sep = "; " if extra_info else ""
+            extra_info += f"{sep}Files added: {', '.join(self.differences['added'])}"
+
+        if "removed" in self.differences and self.differences["removed"]:
+            sep = "; " if extra_info else ""
+            extra_info += f"{sep}Files removed: {', '.join(self.differences['removed'])}"
+
+        if extra_info:
+            return f"{out}. {extra_info}"
+
+        return out
 
 
 class ChecksumFile:
@@ -166,7 +192,7 @@ def verify(self, parsed_manifest_dct, diff=True):
             # If there are any differences in existing paths, fail the check...
             differences = self.diff(parsed_manifest_dct.keys())
             if differences["added"] or differences["removed"]:
-                raise ChecksumMismatch(differences)
+                raise ChecksumMismatch("Files were added or removed", differences)
 
         recalculated = self.calculate_checksums_from_root(verifying=True)
         mismatches = set()
@@ -174,6 +200,7 @@ def verify(self, parsed_manifest_dct, diff=True):
             if recalculated[parsed_path] != parsed_checksum:
                 mismatches.add(parsed_path)
         if mismatches:
-            raise ChecksumMismatch(f"Checksum mismatch: {', '.join(mismatches)}")
+            differences = {"changes": list(mismatches)}
+            raise ChecksumMismatch("Checksum mismatch", differences)
 
         return True
diff --git a/src/ansible_sign/cli.py b/src/ansible_sign/cli.py
index c4f6a79..c2d58e2 100644
--- a/src/ansible_sign/cli.py
+++ b/src/ansible_sign/cli.py
@@ -109,6 +109,14 @@ def parse_args(self, args):
             dest="gnupg_home",
             default=None,
         )
+        cmd_gpg_verify.add_argument(
+            "--no-truncate",
+            help="Disable truncation of file listings when enumerating file differences",
+            required=False,
+            dest="no_truncate",
+            default=False,
+            action="store_true",
+        )
         cmd_gpg_verify.add_argument(
             "project_root",
             help="The directory containing the files being validated and verified",
@@ -227,7 +235,26 @@ def validate_checksum(self):
             checksum.verify(manifest, diff=True)
         except ChecksumMismatch as e:
             self._error("Checksum validation failed.")
-            self._error(str(e))
+            if e.differences:
+                differences = (
+                    ("added", "added"),
+                    ("removed", "removed"),
+                    ("changes", "changed"),
+                )
+                for key, verb in differences:
+                    if key in e.differences and e.differences[key]:
+                        self._note(f"Files {verb}:")
+                        num_changes = len(e.differences[key])
+                        if self.args.no_truncate:
+                            truncate_at = num_changes
+                            truncated = False
+                        else:
+                            truncate_at = 6
+                            truncated = num_changes > truncate_at
+                        for path in e.differences[key][0:(truncate_at)]:
+                            self._note(f"  - {path}")
+                        if truncated:
+                            self._note(f"  [{num_changes - truncate_at} lines omitted, use --no-truncate to see all...]")
             return 2
         except FileNotFoundError as e:
             if os.path.islink(e.filename):
diff --git a/tests/test_checksum.py b/tests/test_checksum.py
index ccf9120..aebe1fd 100644
--- a/tests/test_checksum.py
+++ b/tests/test_checksum.py
@@ -116,19 +116,19 @@ def test_parse_manifest_empty():
     [
         (
             "files-added",
-            "{'added': ['hello2', 'hello3'], 'removed': []}",
+            "Files were added or removed. Files added: hello2, hello3",
         ),
         (
             "files-added-removed",
-            "{'added': ['hello2', 'hello3'], 'removed': ['hello1']}",
+            "Files were added or removed. Files added: hello2, hello3; Files removed: hello1",
         ),
         (
             "files-removed",
-            "{'added': [], 'removed': ['hello1']}",
+            "Files were added or removed. Files removed: hello1",
         ),
         (
             "files-changed",
-            "Checksum mismatch: hello1",
+            "Checksum mismatch. Files changed: hello1",
         ),
         (
             "success",
@@ -136,7 +136,7 @@ def test_parse_manifest_empty():
         ),
     ],
 )
-def test_directory_diff(
+def test_directory_diff_human_readable_exc(
     fixture,
     diff_output,
 ):