Collect all product definitions in a single transaction

Before this commit, the product definitions are collected in 3 different
transactions. The creation of update streams and modules are done in
different transactions, but the linkage between both is important and
if the user performs some action that requires it, like updating a
tracker, while this linkage is not done, it can lead to issues like
trackers unlinking from the flaw.

To avoid this, with this commit all product definitions are collected in
a single transaction. This ensures that we have all modules, update
streams and the links between them at all times.

Closes OSIDB-3590.

collectors/product_definitions/core.py

@@ -71,7 +71,6 @@ def sanitize_product_definitions(data: dict) -> Tuple[dict, dict, dict, dict]:
     )
 
 
-@transaction.atomic
 def sync_ps_contacts(data: dict):
     """
     clean and re-create PS contacts from given data
@@ -89,7 +88,6 @@ def sync_ps_contacts(data: dict):
         PsContact.objects.create(username=contact_username, **filtered_contact_data)
 
 
-@transaction.atomic
 def sync_ps_update_streams(data: dict):
     """
     clean and re-create PS update streams from given data
@@ -107,7 +105,6 @@ def sync_ps_update_streams(data: dict):
         PsUpdateStream.objects.create(name=stream_name, **filtered_stream_data)
 
 
-@transaction.atomic
 def sync_ps_products_modules(ps_products_data: dict, ps_modules_data: dict):
     """
     clean and re-create PS products and PS module from given data
@@ -176,3 +173,16 @@ def sync_ps_products_modules(ps_products_data: dict, ps_modules_data: dict):
                     ).first()
                     if unacked_ps_update_stream:
                         ps_module.ps_update_streams.add(unacked_ps_update_stream)
+
+
+@transaction.atomic
+def sync_product_definitions(
+    ps_products: dict, ps_modules: dict, ps_update_streams: dict, ps_contacts: dict
+):
+    sync_ps_contacts(ps_contacts)
+    sync_ps_update_streams(ps_update_streams)
+    # PS Products and Modules need to be synced together
+    # because every Product holds information about related
+    # Modules, but from Module there is no way of telling
+    # to which Product it relates to
+    sync_ps_products_modules(ps_products, ps_modules)

collectors/product_definitions/tasks.py

@@ -12,9 +12,7 @@
     PRODUCT_DEFINITIONS_URL,
     fetch_product_definitions,
     sanitize_product_definitions,
-    sync_ps_contacts,
-    sync_ps_products_modules,
-    sync_ps_update_streams,
+    sync_product_definitions,
 )
 
 logger = get_task_logger(__name__)
@@ -55,13 +53,8 @@ def product_definitions_collector(collector_obj) -> None:
         )
     )
 
-    sync_ps_contacts(ps_contacts)
-    sync_ps_update_streams(ps_update_streams)
-    # PS Products and Modules need to be synced together
-    # because every Product holds information about related
-    # Modules, but from Module there is no way of telling
-    # to which Product it relates to
-    sync_ps_products_modules(ps_products, ps_modules)
+    # Sync all product definitions in a single transaction
+    sync_product_definitions(ps_products, ps_modules, ps_update_streams, ps_contacts)
 
     collector_obj.store(updated_until_dt=timezone.now())
     logger.info("Product Definitions sync was successful.")

docs/CHANGELOG.md

@@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Changed
 - Add upstream references to Jira trackers on creation (OSIDB-3148)
 - Change ACL mixin serializer to support internal ACLs (OSIDB-3578)
+- Make product definitions collector atomic (OSIDB-3590)
 
 ## [4.5.2] - 2024-10-24
 ### Changed

osidb/management/commands/sync_product_definitions.py

@@ -5,9 +5,7 @@
 from collectors.product_definitions.core import (
     fetch_product_definitions,
     sanitize_product_definitions,
-    sync_ps_contacts,
-    sync_ps_products_modules,
-    sync_ps_update_streams,
+    sync_product_definitions,
 )
 
 
@@ -25,9 +23,9 @@ def handle(self, *args, **options):
             ps_contacts,
         ) = sanitize_product_definitions(raw_data)
 
-        sync_ps_contacts(ps_contacts)
-        sync_ps_update_streams(ps_update_streams)
-        sync_ps_products_modules(ps_products, ps_modules)
+        sync_product_definitions(
+            ps_products, ps_modules, ps_update_streams, ps_contacts
+        )
 
         cm = CollectorMetadata.objects.get(
             name="collectors.product_definitions.tasks.product_definitions_collector"