Component ordering based on dependencies (#250)

* Component ordering

* Added component ordering

* Documentation of the component sorting code

* tests

* Better sorting algorithm and tests

* version bump

oarepo_runtime/services/components.py

@@ -1,17 +1,29 @@
+from __future__ import annotations
+
 import inspect
 from collections import defaultdict
+from dataclasses import dataclass, field
 from typing import Type
 
 from flask import current_app
 from invenio_accounts.models import User
+from invenio_drafts_resources.services.records.config import (
+    RecordServiceConfig as DraftsRecordServiceConfig,
+)
+from invenio_rdm_records.services.config import RDMRecordServiceConfig
 from invenio_records import Record
+from invenio_records_resources.services import FileServiceConfig
+from invenio_records_resources.services.records.config import (
+    RecordServiceConfig as RecordsRecordServiceConfig,
+)
 
-from oarepo_runtime.services.custom_fields import CustomFieldsMixin, CustomFields, InlinedCustomFields
+from oarepo_runtime.services.custom_fields import (
+    CustomFields,
+    CustomFieldsMixin,
+    InlinedCustomFields,
+)
 from oarepo_runtime.services.generators import RecordOwners
-from invenio_rdm_records.services.config import RDMRecordServiceConfig
-from invenio_drafts_resources.services.records.config import RecordServiceConfig as DraftsRecordServiceConfig
-from invenio_records_resources.services.records.config import RecordServiceConfig as RecordsRecordServiceConfig
-from invenio_records_resources.services import  FileServiceConfig
+
 try:
     from invenio_drafts_resources.services.records.uow import ParentRecordCommitOp
 except ImportError:
@@ -62,18 +74,23 @@ def publish(self, identity, data=None, record=None, errors=None, **kwargs):
         if "dateIssued" not in record["metadata"]:
             record["metadata"]["dateIssued"] = datetime.today().strftime("%Y-%m-%d")
 
+
 class CFRegistry:
     def __init__(self):
         self.custom_field_names = defaultdict(list)
 
     def lookup(self, record_type: Type[Record]):
         if record_type not in self.custom_field_names:
-            for fld in inspect.getmembers(record_type, lambda x: isinstance(x, CustomFieldsMixin)):
+            for fld in inspect.getmembers(
+                record_type, lambda x: isinstance(x, CustomFieldsMixin)
+            ):
                 self.custom_field_names[record_type].append(fld[1])
         return self.custom_field_names[record_type]
 
+
 cf_registry = CFRegistry()
 
+
 class CustomFieldsComponent(ServiceComponent):
     def create(self, identity, data=None, record=None, **kwargs):
         """Create a new record."""
@@ -90,16 +107,16 @@ def _set_cf_to_record(self, record, data):
             elif isinstance(cf, InlinedCustomFields):
                 config = current_app.config.get(cf.config_key, {})
                 for c in config:
-                    record[c.name] =data.get(c.name)
+                    record[c.name] = data.get(c.name)
 
-def process_service_configs(service_config):
 
+def process_service_configs(service_config, *additional_components):
     processed_components = []
     target_classes = {
         RDMRecordServiceConfig,
         DraftsRecordServiceConfig,
         RecordsRecordServiceConfig,
-        FileServiceConfig
+        FileServiceConfig,
     }
 
     for end_index, cls in enumerate(type(service_config).mro()):
@@ -110,19 +127,150 @@ def process_service_configs(service_config):
     # there are two service_config instances in the MRO (Method Resolution Order) output.
     start_index = 2 if hasattr(service_config, "build") else 1
 
-    service_configs = type(service_config).mro()[start_index:end_index + 1]
+    service_configs = type(service_config).mro()[start_index : end_index + 1]
     for config in service_configs:
-
         if hasattr(config, "build"):
             config = config.build(current_app)
 
-        if hasattr(config, 'components'):
+        if hasattr(config, "components"):
             component_property = config.components
             if isinstance(component_property, list):
                 processed_components.extend(component_property)
             elif isinstance(component_property, tuple):
-                processed_components.extend(list (component_property))
+                processed_components.extend(list(component_property))
             else:
                 raise ValueError(f"{config} component's definition is not supported")
 
-    return processed_components
+    processed_components.extend(additional_components)
+    processed_components = _sort_components(processed_components)
+    return processed_components
+
+
+@dataclass
+class ComponentPlacement:
+    """Component placement in the list of components.
+
+    This is a helper class used in the component ordering algorithm.
+    """
+
+    component: Type[ServiceComponent]
+    """Component to be ordered."""
+
+    depends_on: list[ComponentPlacement] = field(default_factory=list)
+    """List of components this one depends on.
+    
+    The components must be classes of ServiceComponent or '*' to denote
+    that this component depends on all other components and should be placed last.
+    """
+
+    affects: list[ComponentPlacement] = field(default_factory=list)
+    """List of components that depend on this one.
+    
+    This is a temporary list used for evaluation of '*' dependencies
+    but does not take part in the sorting algorithm."""
+
+    def __hash__(self) -> int:
+        return id(self.component)
+
+    def __eq__(self, other: ComponentPlacement) -> bool:
+        return self.component is other.component
+
+
+def _sort_components(components):
+    """Sort components based on their dependencies while trying to
+    keep the initial order as far as possible."""
+
+    placements: list[ComponentPlacement] = _prepare_component_placement(components)
+    placements = _propagate_dependencies(placements)
+
+    ret = []
+    while placements:
+        without_dependencies = [p for p in placements if not p.depends_on]
+        if not without_dependencies:
+            raise ValueError("Circular dependency detected in components.")
+        for p in without_dependencies:
+            ret.append(p.component)
+            placements.remove(p)
+            for p2 in placements:
+                if p in p2.depends_on:
+                    p2.depends_on.remove(p)
+    return ret
+
+
+def _matching_placements(placements, dep_class_or_factory):
+    for pl in placements:
+        pl_component = pl.component
+        if not inspect.isclass(pl_component):
+            pl_component = type(pl_component(service=object()))
+        if issubclass(pl_component, dep_class_or_factory):
+            yield pl
+
+
+def _prepare_component_placement(components) -> list[ComponentPlacement]:
+    """Convert components to ComponentPlacement instances and resolve dependencies."""
+    placements = []
+    for idx, c in enumerate(components):
+        placement = ComponentPlacement(component=c)
+        placements.append(placement)
+
+    # direct dependencies
+    for idx, placement in enumerate(placements):
+        placements_without_this = placements[:idx] + placements[idx + 1 :]
+        for dep in getattr(placement.component, "depends_on", []):
+            if dep == "*":
+                continue
+            for pl in _matching_placements(placements_without_this, dep):
+                placement.depends_on.append(pl)
+                pl.affects.append(placement)
+
+        for dep in getattr(placement.component, "affects", []):
+            if dep == "*":
+                continue
+            for pl in _matching_placements(placements_without_this, dep):
+                placement.affects.append(pl)
+                pl.depends_on.append(placement)
+
+    # star dependencies
+    for idx, placement in enumerate(placements):
+        placements_without_this = placements[:idx] + placements[idx + 1 :]
+        if "*" in getattr(placement.component, "depends_on", []):
+            for pl in placements_without_this:
+                # if this placement is not in placements that pl depends on
+                # (added via direct dependencies above), add it
+                if placement not in pl.depends_on:
+                    placement.depends_on.append(pl)
+                    pl.affects.append(placement)
+
+        if "*" in getattr(placement.component, "affects", []):
+            for pl in placements_without_this:
+                # if this placement is not in placements that pl affects
+                # (added via direct dependencies above), add it
+                if placement not in pl.affects:
+                    placement.affects.append(pl)
+                    pl.depends_on.append(placement)
+    return placements
+
+
+def _propagate_dependencies(
+    placements: list[ComponentPlacement],
+) -> list[ComponentPlacement]:
+    # now propagate dependencies
+    dependency_propagated = True
+    while dependency_propagated:
+        dependency_propagated = False
+        for placement in placements:
+            for dep in placement.depends_on:
+                for dep_of_dep in dep.depends_on:
+                    if dep_of_dep not in placement.depends_on:
+                        placement.depends_on.append(dep_of_dep)
+                        dep_of_dep.affects.append(placement)
+                        dependency_propagated = True
+
+            for dep in placement.affects:
+                for dep_of_dep in dep.affects:
+                    if dep_of_dep not in placement.affects:
+                        placement.affects.append(dep_of_dep)
+                        dep_of_dep.depends_on.append(placement)
+                        dependency_propagated = True
+
+    return placements

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = oarepo-runtime
-version = 1.5.89
+version = 1.5.90
 description = A set of runtime extensions of Invenio repository
 authors = Alzbeta Pokorna
 readme = README.md

tests/test_component_ordering.py

@@ -0,0 +1,89 @@
+from oarepo_runtime.services.components import _sort_components
+
+
+class FakeComponent:
+    pass
+
+
+def make_component(name, depends_on=None, affects=None):
+    fields = {}
+    if depends_on:
+        fields["depends_on"] = depends_on
+    if affects:
+        fields["affects"] = affects
+    return type(name, (FakeComponent,), fields)
+
+
+def test_component_no_dependencies():
+    components = [
+        make_component("A"),
+        make_component("B"),
+        make_component("C"),
+        make_component("D"),
+    ]
+    assert _sort_components(components) == components
+
+
+def test_component_after_all():
+    components = [
+        make_component("A", depends_on=["*"]),
+        make_component("B"),
+        make_component("C"),
+        make_component("D"),
+    ]
+    assert [x.__name__ for x in _sort_components(components)] == ["B", "C", "D", "A"]
+
+
+def test_component_after_all_2():
+    components = [
+        make_component("A", depends_on="*"),
+        make_component("B"),
+        make_component("C"),
+        make_component("D"),
+    ]
+    assert [x.__name__ for x in _sort_components(components)] == ["B", "C", "D", "A"]
+
+
+def test_component_after_all_direct():
+    components = [
+        a := make_component("A", depends_on="*"),
+        make_component("B", depends_on=[a]),
+        make_component("C"),
+        make_component("D"),
+    ]
+    assert [x.__name__ for x in _sort_components(components)] == ["C", "D", "A", "B"]
+
+
+def test_component_before_all():
+    components = [
+        make_component("A"),
+        make_component("B"),
+        make_component("C"),
+        make_component("D", affects=["*"]),
+    ]
+    assert [x.__name__ for x in _sort_components(components)] == ["D", "A", "B", "C"]
+
+
+def test_component_before_all_direct():
+    components = [
+        make_component("A"),
+        make_component("B"),
+        make_component("C"),
+        d := make_component("D", affects=["*"]),
+        make_component("E", affects=[d]),
+    ]
+    assert [x.__name__ for x in _sort_components(components)] == [
+        "E",
+        "D",
+        "A",
+        "B",
+        "C",
+    ]
+
+
+def test_component_in_between():
+    a = make_component("A")
+    c = make_component("C")
+    b = make_component("B", affects=[a], depends_on=[c])
+    components = [a, b, c]
+    assert [x.__name__ for x in _sort_components(components)] == ["C", "B", "A"]