⏪ Revert removal of DISABLE_SENDING_HIDDEN_FIELDS feature flag

Reverts #4759.

DH is facing challenges with legacy forms while upgrading to 3.0 on
a strict timeline. Because the forms haven't been migrated to
variables mapping yet, this has a huge impact on downstream
systems processing the submissions.

(cherry picked from commit b53634b)

src/openforms/conf/base.py

@@ -628,6 +628,7 @@
 )  # 1mb in bytes
 # Perform HTML escaping on user's data-input
 ESCAPE_REGISTRATION_OUTPUT = config("ESCAPE_REGISTRATION_OUTPUT", default=False)
+DISABLE_SENDING_HIDDEN_FIELDS = config("DISABLE_SENDING_HIDDEN_FIELDS", default=False)
 
 # TODO: convert to feature flags so that newly deployed instances get the new behaviour
 # while staying backwards compatible for existing instances

src/openforms/formio/rendering/default.py

@@ -1,6 +1,7 @@
 from dataclasses import dataclass
 from typing import Callable, Iterator
 
+from django.conf import settings
 from django.urls import reverse
 from django.utils.html import format_html_join
 from django.utils.safestring import SafeString, mark_safe
@@ -35,7 +36,10 @@ def is_visible(self) -> bool:
         # class.
         # In registration mode, we need to treat layout/container nodes as visible so
         # that their children are emitted too.
-        if self.mode in {RenderModes.export, RenderModes.registration}:
+        visible_modes = {RenderModes.export, RenderModes.registration}
+        if settings.DISABLE_SENDING_HIDDEN_FIELDS:
+            visible_modes.remove(RenderModes.registration)
+        if self.mode in visible_modes:
             return True
 
         # We only pass the step data, since frontend logic only has access to the current step data.

src/openforms/formio/rendering/nodes.py

@@ -2,6 +2,8 @@
 from dataclasses import dataclass
 from typing import TYPE_CHECKING, Any, Callable, Iterator, Literal
 
+from django.conf import settings
+
 from glom import Path, assign, glom
 
 from openforms.submissions.models import SubmissionStep
@@ -117,7 +119,10 @@ def is_visible(self) -> bool:
 
         # everything is emitted in export mode to get consistent columns
         # the same happens with the registration in order to include hidden fields as well
-        if self.mode in {RenderModes.export, RenderModes.registration}:
+        visible_modes = {RenderModes.export, RenderModes.registration}
+        if settings.DISABLE_SENDING_HIDDEN_FIELDS:
+            visible_modes.remove(RenderModes.registration)
+        if self.mode in visible_modes:
             return True
 
         # explicitly hidden components never show up. Note that this property can be set

src/openforms/registrations/contrib/objects_api/tests/test_template.py

@@ -439,3 +439,112 @@ def test_object_nulls_regression(self, m):
                 },
             },
         )
+
+    @tag("dh-673", "gh-4140")
+    @override_settings(DISABLE_SENDING_HIDDEN_FIELDS=True)
+    @requests_mock.Mocker()
+    def test_opt_out_of_sending_hidden_fields(self, m):
+        submission = SubmissionFactory.from_components(
+            components_list=[
+                {
+                    "type": "radio",
+                    "key": "radio",
+                    "label": "Radio",
+                    "values": [
+                        {"label": "1", "value": "1"},
+                        {"label": "2", "value": "2"},
+                    ],
+                    "defaultValue": None,
+                    "validate": {"required": True},
+                    "openForms": {"dataSrc": "manual"},
+                },
+                {
+                    "type": "textfield",
+                    "key": "tekstveld",
+                    "label": "Tekstveld",
+                    "hidden": True,
+                    "validate": {"required": True},
+                    "conditional": {"eq": "1", "show": True, "when": "radio"},
+                    "defaultValue": None,
+                    "clearOnHide": True,
+                },
+                {
+                    "type": "currency",
+                    "currency": "EUR",
+                    "key": "bedrag",
+                    "label": "Bedrag",
+                    "hidden": True,
+                    "validate": {"required": True},
+                    "conditional": {"eq": "1", "show": True, "when": "radio"},
+                    "defaultValue": None,
+                    "clearOnHide": True,
+                },
+                {
+                    "type": "fieldset",
+                    "key": "fieldsetNoVisibleChildren",
+                    "label": "A container without visible children",
+                    "hidden": True,
+                    "components": [
+                        {
+                            "type": "textfield",
+                            "key": "input7",
+                            "label": "Input 7",
+                            "hidden": True,
+                        }
+                    ],
+                },
+            ],
+            with_report=True,
+            submitted_data={"radio": "2"},
+            form_definition_kwargs={"slug": "stepwithnulls"},
+        )
+        config = ObjectsAPIGroupConfigFactory.create(
+            objecttypes_service__api_root="https://objecttypen.nl/api/v1/",
+        )
+        plugin = ObjectsAPIRegistration(PLUGIN_IDENTIFIER)
+
+        m.get(
+            "https://objecttypen.nl/api/v1/objecttypes/f3f1b370-97ed-4730-bc7e-ebb20c230377",
+            json={
+                "url": "https://objecttypen.nl/api/v1/objecttypes/f3f1b370-97ed-4730-bc7e-ebb20c230377"
+            },
+            status_code=200,
+        )
+
+        with (
+            patch(
+                "openforms.registrations.contrib.objects_api.plugin.get_objects_client"
+            ) as mock_objects_client,
+        ):
+            _objects_client = mock_objects_client.return_value.__enter__.return_value
+            _objects_client.create_object.return_value = {"dummy": "response"}
+
+            plugin.register_submission(
+                submission,
+                {
+                    "objects_api_group": config,
+                    "version": 1,
+                    "objecttype": UUID("f3f1b370-97ed-4730-bc7e-ebb20c230377"),
+                    "objecttype_version": 300,
+                    # skip document uploads
+                    "informatieobjecttype_submission_report": "",
+                    "upload_submission_csv": False,
+                    "update_existing_object": False,
+                    "informatieobjecttype_attachment": "",
+                    "content_json": "{% json_summary %}",
+                },
+            )
+
+        _objects_client.create_object.mock_assert_called_once()
+        record_data = _objects_client.create_object.call_args[1]["record_data"]["data"]
+        # for missing values, the empty value (depending on component type) must be used
+        # Note that the input data was validated against the hidden/visible and
+        # clearOnHide state - absence of the data implies that the component was not
+        # visible and its data was cleared (otherwise the value *would* have been sent
+        # along and be present).
+        self.assertEqual(
+            record_data,
+            {
+                "stepwithnulls": {"radio": "2"},
+            },
+        )