From 77b2966094580c36df6ae03bec834b529271468b Mon Sep 17 00:00:00 2001
From: pkoprda <47797196+pkoprda@users.noreply.github.com>
Date: Wed, 5 Feb 2025 02:03:31 +0100
Subject: [PATCH] fix(client): Support playbooks with unicode for Python 3.12+
 (#4342)

* Card ID: CCT-644

Some Unicode characters, like the zero-width joiner in emojis, were not
processed correctly. This update improves Unicode support, mainly aimed
for Python 3.12+.

Signed-off-by: pkoprda <pkoprda@redhat.com>
(cherry picked from commit c4467937f01d3db6a856055a084d5cd29efa0673)
---
 .../ansible/playbook_verifier/serializer.py   |   3 +
 .../client/apps/playbooks/unicode.digest.bin  | Bin 32 -> 32 bytes
 .../apps/playbooks/unicode.serialized.bin     |   2 +-
 .../tests/client/apps/playbooks/unicode.yml   |  54 +++++++++++-------
 .../client/apps/test_playbook_verifier.py     |  21 +++++--
 5 files changed, 55 insertions(+), 25 deletions(-)

diff --git a/insights/client/apps/ansible/playbook_verifier/serializer.py b/insights/client/apps/ansible/playbook_verifier/serializer.py
index 12daafe5a0..9bf8751c71 100644
--- a/insights/client/apps/ansible/playbook_verifier/serializer.py
+++ b/insights/client/apps/ansible/playbook_verifier/serializer.py
@@ -57,6 +57,9 @@ def _str(cls, value):
             "\\": "\\\\",
             "\n": "\\n",
             "\t": "\\t",
+            "\u200b": "\\u200b",  # Zero-width space
+            "\u200c": "\\u200c",  # Zero-width non-joiner
+            "\u200d": "\\u200d",  # Zero-width joiner
         }
         escaped_string = ""
         for char in value:
diff --git a/insights/tests/client/apps/playbooks/unicode.digest.bin b/insights/tests/client/apps/playbooks/unicode.digest.bin
index 74969d2570cabb0eb04603dd626b997a0227a4f0..dff74670da20133597198f39a3530419f8e408c1 100644
GIT binary patch
literal 32
ocmWG;7W$&g^QTTj`kc&;OIz6Q-ea&h5}a!Ei)mG()_P`B0P5on+W-In

literal 32
ocmXp#(rh#@lS4;(!rn&v<DX3*arR}#@XJs5F<o<^o3nKs0O2|fcmMzZ

diff --git a/insights/tests/client/apps/playbooks/unicode.serialized.bin b/insights/tests/client/apps/playbooks/unicode.serialized.bin
index f6cd5e1842..c3b971759f 100644
--- a/insights/tests/client/apps/playbooks/unicode.serialized.bin
+++ b/insights/tests/client/apps/playbooks/unicode.serialized.bin
@@ -1 +1 @@
-ordereddict([('name', 'The legend says one day Unicode will just work'), ('become', 'yes'), ('vars', ordereddict([('insights_signature_exclude', '/hosts,/vars/insights_signature')])), ('tasks', [ordereddict([('name', 'Not all languages are as boring as English /s'), ('ansible.builtin.find', ordereddict([('paths', ['/tříštivá/hrušeň', '/ご飯が熱い。/彼は変だ。', '/电脑/汉堡包', '/אני פה/הוא אכל את העוגה/', '/تَكَاتَبْنَا/كيف حالك؟/'])]))]), ordereddict([('name', 'Linux supports emojis in paths. Now you know.'), ('ansible.builtin.find', ordereddict([('paths', ['/🍏/👨🏼\u200d🚀/', '/usr/bin/🙀', '/var/lib/ඞ/'])]))])])])
\ No newline at end of file
+ordereddict([('name', 'Playbook with various Unicode characters'), ('become', 'yes'), ('vars', ordereddict([('insights_signature_exclude', '/hosts,/vars/insights_signature')])), ('tasks', [ordereddict([('name', 'Not all languages are as boring as English /s'), ('ansible.builtin.find', ordereddict([('paths', ['/tříštivá/hrušeň', '/ご飯が熱い。/彼は変だ。', '/电脑/汉堡包', '/אני פה/הוא אכל את העוגה/', '/تَكَاتَبْنَا/كيف حالك؟/', '/რამდენიმე/ქართული/', '/κάποιο/ελληνικό/'])]))]), ordereddict([('name', 'Linux supports emojis in paths. Now you know.'), ('ansible.builtin.find', ordereddict([('paths', ['/🍏/👨🏼\u200d🚀/', '/usr/bin/🙀', '/var/lib/ඞ/'])]))]), ordereddict([('name', 'Various special characters'), ('ansible.builtin.find', ordereddict([('paths', ['/👨\u200d👩\u200d👦/👨\u200d🌾/👨\u200d🦰/', '/ണ്\u200d/ශ්\u200dර/क्\u200dष/', '/text\u200cjoined\u200cby\u200czero\u200cwidth\u200cnon\u200cjoiner/', '/text\u200bjoined\u200bby\u200bzero\u200bwidth\u200bspace/'])]))])])])
\ No newline at end of file
diff --git a/insights/tests/client/apps/playbooks/unicode.yml b/insights/tests/client/apps/playbooks/unicode.yml
index ccb67aaef5..f483b410f9 100644
--- a/insights/tests/client/apps/playbooks/unicode.yml
+++ b/insights/tests/client/apps/playbooks/unicode.yml
@@ -1,29 +1,28 @@
----
-- name: The legend says one day Unicode will just work
+- name: Playbook with various Unicode characters
   hosts: localhost
   become: yes
   vars:
     insights_signature_exclude: /hosts,/vars/insights_signature
     insights_signature: !!binary |
       TFMwdExTMUNSVWRKVGlCUVIxQWdVMGxIVGtGVVZWSkZMUzB0TFMwS1ZtVnljMmx2YmpvZ1IyNTFV
-      RWNnZGpFS0NtbFJTV05DUVVGQ1EwRkJSMEpSU20xdGJpODFRVUZ2U2tWTmRuYzFPRVFyYWpWd1Rr
-      bFVkMUF2TTFkdWFGWjNPVmRoZDFoSVMwTXhTR0pEVkhSRGNsQUtPR1JPY2xveFZuUmlla2xXUkd3
-      NFQwTkZZVVJQYzNOSU4xbDFUMDVxU1haMGVGUlNOVXBxTHpKYVkyZFVhMnRRYzJwRkwwYzRaRlZq
-      Tm1kUE1GbDViUXBpUTJreFprSm9XVkZIVW1sdVZUbHpiME5yWlU5UlVYQnFZVVZSUVRoME1XZHli
-      VEI1TVRKTGVXcERkVGQ2WmxSR1EyMXFiSEpDVEVjM2RsbEdkMlZRQ2xneFp6VkpPSHB1TkN0Mk0w
-      WTNhbXhFWVZseVdTOWhhM1EzTTBVeVNEVk1aVllyTWxoblQzTk5WblZMUXpGd1JGVm9NV3RqYkRk
-      V00yVjNRV3RtWlhZS1ZWbG5Ta3BHYzJ0MmFVNXlSVGRrV0hSRFRYY3liRkpwYkVselNIa3hNRFpP
-      S3l0VWJYQTROMVpTTTNWU09XWlBNRFpRU1ZsRFRUVnhORXhHYzFCVFZBcDZRMFJESzFST1NESXJV
-      akF6WjJ0WGVFaFRUMlZtVjBscVRqUnNWakJzUkd0dFNEaEdaM1ZGZFdSbFYzWkthMVpaZG5OMVJH
-      NUtVbE5sVDBWYVUzSkxDa3hrYUVJMlFWYzFlbHBKV25Wb1FXeDJWRTE2YjFoMlJ6TXhRbHBpTkho
-      VlVqRjFZWEZaVjNkaU4yVkhkaXRzTlhoSmFVUnNORzV5ZHpWUlR6SjZlbGdLVmpKVmRXdE1RM1Z5
-      TUd4UVptSTVWblFyWjI1VmJYbzBhbVpCYVU1UVlUQm9SRzAxWldWU1luWlJhSGs0V0hsSFZEQXJR
-      a0phYldoelVXSTJSRkZNUXdwRWJGZHpXbFZJU3pRdkwzTlRVMjFCVEM5V1FuQnBTR0ZpU1VkRlVV
-      dFVOR2RMWlhSaWJYcFFhRXB0TjNWbE1uWXJORlJqVlRCMmIzQmFMMUZOU25wM0NuaFFaVzlGVWpC
-      c0x6UklaVWhUU0VjcmRtMXJkakZUU0ZWdmFXcEVWUzlETjNGUGN6bGtNVXM0V1Zsc2RsUkJjbXhV
-      U1ZseE4yMUZUMkphTHpsSlpFNEthREExYzBOWk5sRnRSM05yZEV0TVNrTTJWVGhFWm05R1VHeFJi
-      MGhIUWpOWE9ITXhVVVZqV0RObmVITmhiVWgxYW5sRlZUbG1kbUpLVURKR1oyOXhWQXBIVW5GMGNs
-      aHZZblJIYm5aU1dtUTNVbXB1VXdvOVpYVXhZZ290TFMwdExVVk9SQ0JRUjFBZ1UwbEhUa0ZVVlZK
+      RWNnZGpFS0NtbFJTV05DUVVGQ1EwRkJSMEpSU201dFFYcDNRVUZ2U2tWTmRuYzFPRVFyYWpWd1Rt
+      NXNkMUZCU2tWRFR5OTRkelJ3ZVZvMVpVdGpRVVUzWkdSNWJGY0taME5STDJwdmVWaEZibVZXVUVV
+      NVUxaElWbXRhUVVWc1NqUTViVEp1Tm5aQkwyNVVjM2R4YW5kVU5TODJLMFp2VFdSQlZDdEhkVGhy
+      TDBzeVFYUnhhQXBJY0Vod1Z5dDRNM1E0YVc1alZWVjViR1YzSzA5TFpDOU1TbWxqZDI5aWNUaHVS
+      RXh2YWpWNk1XNVFSWGhGT1VkM1FWUkZTVE5SWlc5T2FucFpNMUZDQ2sxM1kzSk5hMFZEU21reU1I
+      RkRWMHc0YlVoVFVHRlhSSFpSVTNKeWFtd3pkR0ZUTHpaSVEyaHFSM3AxY0dkMVIyMWphbGR0VjNS
+      ck1IbDFSakkwVjBjS1YzSkNOalJ3Y1RGaFExUXhURk15VVZSd2QxUjFkbTUyVERJM1NWbFZURlJ2
+      ZFdsbGNFWjVhM1JXWTJob2R5OVdXR1l4TmtKMFQybHlTbWRZUmlzellncE9jVXRpV0RsM0wySlJi
+      RlJSUlZnM2VFRkhNRUUzV0RaV05uZE5VbnBhY2pCQ1pGTnBhVEE1VWtsa2NGbE1XVmRqTVV0TFVU
+      bGhZMWs1Y1N0dWRqZEdDbFF4VGpkRmNrSkNTbFE0YkhWM01IQkRPVnB5Y1VneFVuQTVhVTlKUjB4
+      WlVHdFVOVWRRV1dkYVVHUkpRekI1Tm1sTWNrRjZNMEpsU21aaFZrRnFhR2NLU1VSc09EZE1TVFEy
+      Y2tzMGJETnlMMVJVUW5WU1lXZGxSVmhuZVU1a2JuVmxhRkpLZVZOUGVXTmhiV1ZCWjFNck5uQlVa
+      RVpNVEVKaE16TkZWVFp0ZGdwUlNURTRiSEp2UVdaSlRIUlNTRFF4YzB4NlpHUTFZMDgxU2tWV1Qx
+      Wm5RV0Z4UVVkTFVrUlFjbko2TjJvd01rcDNNU3RQZEdseGNtWjBiWGg2UkVVckNrWkxPWEZST0Zs
+      VVJubDZabGN4VDFsMFZIbzRVMDFJV1ZOdGJubFRPRXBOVlVZMlZTdHhSR1ZWYlZsQlFUaFpaRXQz
+      ZVRaS2MwdE1SR3B1VTFOR09Wb0tSMnd3TXpWYWRURkxka1JCZERKM05IQk5Wa1JDY2tWalQwOVFl
+      VmR6YVdac1Jpc3ZLekF5SzB0WVEyUjVZemRKZUZsbVZYUmhXRE5uV1hCUEswMTZhUXBUTm0wNFFr
+      eFJWMFpxTmxKaVYyeHlUMDF5VHdvOU9HSkRUQW90TFMwdExVVk9SQ0JRUjFBZ1UwbEhUa0ZVVlZK
       RkxTMHRMUzBL
   tasks:
     - name: Not all languages are as boring as English /s
@@ -35,6 +34,8 @@
           - /电脑/汉堡包
           - /אני פה/הוא אכל את העוגה/
           - /تَكَاتَبْنَا/كيف حالك؟/
+          - /რამდენიმე/ქართული/
+          - /κάποιο/ελληνικό/
 
     - name: Linux supports emojis in paths. Now you know.
       ansible.builtin.find:
@@ -42,3 +43,16 @@
           - /🍏/👨🏼‍🚀/
           - /usr/bin/🙀
           - /var/lib/ඞ/
+
+    - name: Various special characters
+      ansible.builtin.find:
+        paths:
+          # Zero-width joiner
+          - /👨‍👩‍👦/👨‍🌾/👨‍🦰/
+          - /ണ്‍/ශ්‍ර/क्‍ष/
+
+          # Zero-width non-joiner
+          - /text‌joined‌by‌zero‌width‌non‌joiner/
+
+          # Zero-width space
+          - /text​joined​by​zero​width​space/
diff --git a/insights/tests/client/apps/test_playbook_verifier.py b/insights/tests/client/apps/test_playbook_verifier.py
index 2a30883772..a7037fd732 100644
--- a/insights/tests/client/apps/test_playbook_verifier.py
+++ b/insights/tests/client/apps/test_playbook_verifier.py
@@ -379,6 +379,21 @@ def test_strings(self, source, expected):
         result = playbook_verifier.PlaybookSerializer.serialize(source)
         assert result == expected
 
+    @pytest.mark.parametrize(
+        "source,expected",
+        [
+            pytest.param("zw​space", "'zw\\u200bspace'", id="zero-width space"),
+            pytest.param("zw‌nonjoiner", "'zw\\u200cnonjoiner'", id="zero-width non-joiner"),
+            pytest.param("👨🏼‍🚀", "'👨🏼\\u200d🚀'", id="zero-width joiner"),
+        ],
+    )
+    def test_strings_unicode(self, source, expected):
+        if sys.version_info < (3, 0):
+            raise pytest.skip("Unicode characters are not supported on Python 2 systems")
+
+        result = playbook_verifier.PlaybookSerializer.serialize(source)
+        assert result == expected
+
 
 class TestSerializePlaybookSnippet:
     def test_serialize_dictionary(self):
@@ -427,10 +442,8 @@ def test_small(self):
 
     @pytest.mark.parametrize("filename", ("insights_setup", "insights_remove", "document-from-hell", "unicode"))
     def test_real(self, filename):
-        if filename == "unicode" and sys.version_info < (3, 0):
-            raise pytest.skip("Playbooks containing unicode are not supported in Python 2 systems")
-        if filename == "unicode" and sys.version_info >= (3, 12):
-            raise pytest.xfail("Known RFE in Unicode serialization.")
+        if filename == "unicode" and sys.version_info < (3, 7):
+            raise pytest.skip("Python 3.7 or later is required to test playbooks containing unicode")
 
         parent = os.path.dirname(__file__)  # type: str
         with open("{parent}/playbooks/{filename}.yml".format(parent=parent, filename=filename), "r") as f: