Corrected handling of namespaced jinjax macros, enhanced macro resolu…

…tion (both components.Field and Field can be used), working on code coverage

oarepo_ui/ext.py

@@ -1,93 +1,16 @@
 import functools
-import json
-import os
-import re
-from importlib import import_module
-from pathlib import Path
-from typing import Dict
-
 from flask import Response, current_app
-from frozendict import frozendict
-from importlib_metadata import entry_points
 
 import oarepo_ui.cli  # noqa
 from oarepo_ui.resources.catalog import OarepoCatalog as Catalog
 from oarepo_ui.resources.templating import TemplateRegistry
 
 
-def crop_component_path(path):
-    parent_dir = os.path.dirname(path)
-
-    return parent_dir
-
-
-def crop_root_path(path, app_theme):
-    if app_theme:
-        for theme in app_theme:
-            if theme in path:
-                folder_index = path.index(theme)
-                cropped_path = path[: folder_index + len(theme)]
-
-                return cropped_path
-
-    return crop_component_path(path)
-
-
-def extract_priority(path):
-    match, _s = prefix_match(path)
-    if match:
-        path_parts = path.split("/")
-        file_parts = path_parts[-1].split("-")
-        return int(file_parts[0])
-    return 0
-
-
-def prefix_match(path):
-    match = re.match(r"^(?:.*[\\\/])?\d{3}-.*$", path)
-    if match:
-        return True, match
-    return False, None
-
-
-def list_templates(env):
-    searchpath = []
-
-    path_dict = {}
-    for path in env.loader.list_templates():
-        try:
-            if path.endswith("jinja") or path.endswith("jinja2"):
-                priority = extract_priority(path)
-                file_name = path.split("/")[-1]
-                match, _s = prefix_match(file_name)
-                if match:
-                    _, _, file_name = file_name.partition("-")
-                if file_name not in path_dict or priority > path_dict[file_name][0]:
-                    path_dict[file_name] = (priority, path)
-        except Exception as e:
-            print(e)
-    jinja_templates = [
-        env.loader.load(env, path) for priority, path in path_dict.values()
-    ]
-
-    for temp in jinja_templates:
-        app_theme = current_app.config.get("APP_THEME", None)
-        searchpath.append(
-            {
-                "root_path": crop_root_path(temp.filename, app_theme),
-                "component_path": crop_component_path(temp.filename),
-                "component_file": temp.filename,
-            }
-        )
-
-    return searchpath
-
-
 class OARepoUIState:
     def __init__(self, app):
         self.app = app
         self.templates = TemplateRegistry(app, self)
         self._resources = []
-        self.layouts = self._load_layouts()
         self.init_builder_plugin()
         self._catalog = None
 
@@ -112,31 +35,17 @@ def _catalog_config(self, catalog, env):
         catalog.jinja_env.filters.update(env.filters)
         catalog.jinja_env.policies.update(env.policies)
 
-        env.loader.searchpath = list_templates(catalog.jinja_env)
+        # env.loader.searchpath = list_templates(catalog.jinja_env)
         catalog.prefixes[""] = catalog.jinja_env.loader
 
         return catalog
 
-    def get_template(self, layout: str, blocks: Dict[str, str]):
-        return self.templates.get_template(layout, frozendict(blocks))
-
     def register_resource(self, ui_resource):
         self._resources.append(ui_resource)
 
     def get_resources(self):
         return self._resources
 
-    def get_layout(self, layout_name):
-        return self.layouts.get(layout_name, {})
-
-    def _load_layouts(self):
-        layouts = {}
-        for ep in entry_points(group="oarepo.ui"):
-            m = import_module(ep.module)
-            path = Path(m.__file__).parent / ep.attr
-            layouts[ep.name] = json.loads(path.read_text())
-        return layouts
-
     def init_builder_plugin(self):
         if self.app.config["OAREPO_UI_DEVELOPMENT_MODE"]:
             self.app.after_request(self.development_after_request)

oarepo_ui/resources/catalog.py

@@ -1,10 +1,12 @@
 import os
 import re
+from collections import namedtuple
 from functools import cached_property
 from pathlib import Path
 from typing import Dict, Tuple
 
 import jinja2
+from flask import current_app
 from jinjax import Catalog
 from jinjax.component import Component
 from jinjax.exceptions import ComponentNotFound
@@ -19,6 +21,8 @@
 PROP_CONTENT = "content"
 
 
+SearchPathItem = namedtuple("SearchPath", ["template_name", "absolute_path", "relative_path", "priority"])
+
 class OarepoCatalog(Catalog):
     singleton_check = None
 
@@ -35,34 +39,39 @@ def get_source(self, cname: str, file_ext: "TFileExt" = "") -> str:
     def component_paths(self) -> Dict[str, Tuple[Path, Path]]:
         """
         Returns a cache of component-name => (root_path, component_path).
+        The component name is either the filename without the '.jinja' suffix
+        (such as "DetailPage"), or it is a namespaced name (such as
+        "oarepo_vocabularies.DetailPage").
+
+        Note: current theme (such as semantic-ui) is stripped from the namespace.
+
         To invalidate the cache, call `del self.component_paths`.
+
+        Example keys:
+            * "DetailPage" -> DetailPage.jinja
+            * "oarepo_vocabularies.DetailPage" -> oarepo_vocabularies/DetailPage.jinja
+            * "oarepo_vocabularies.DetailPage" -> semantic-ui/oarepo_vocabularies/DetailPage.jinja
+
+        The method also adds partial keys to the cache with lower priority (-10 for each omitted segment),
+        so that the following are also added:
+
+            * "DetailPage" -> oarepo_vocabularies/DetailPage.jinja (priority -10)
         """
         paths: Dict[str, Tuple[Path, Path, int]] = {}
 
-        # paths known by the current jinja environment
-        search_paths = {
-            Path(searchpath["component_file"])
-            for searchpath in self.jinja_env.loader.searchpath
-        }
+        for template_name, absolute_template_path, relative_template_path, priority in self.list_templates():
+            split_template_name = template_name.split(DELIMITER)
 
-        for root_path, namespace, template_path in self._get_all_template_files():
-            # if the file is known to the current jinja environment,
-            # get the priority and add it to known components
-            if template_path in search_paths:
-                template_filename = os.path.basename(template_path)
-                template_filename, priority = self._extract_priority(template_filename)
-
-                if namespace:
-                    relative_filepath = f"{namespace}/{template_filename}"
-                else:
-                    relative_filepath = template_filename
+            for idx in range(0, len(split_template_name)):
+                partial_template_name = DELIMITER.join(split_template_name[idx:])
+                partial_priority = priority - idx * 10
 
                 # if the priority is greater, replace the path
                 if (
-                    relative_filepath not in paths
-                    or priority > paths[relative_filepath][2]
+                    partial_template_name not in paths
+                    or partial_priority > paths[template_name][2]
                 ):
-                    paths[relative_filepath] = (root_path, template_path, priority)
+                    paths[partial_template_name] = (absolute_template_path, relative_template_path, partial_priority)
 
         return {k: (v[0], v[1]) for k, v in paths.items()}
 
@@ -76,32 +85,11 @@ def _extract_priority(self, filename):
             filename = filename[4:]
         return filename, priority
 
-    def _get_all_template_files(self):
-        for prefix in self.prefixes:
-            root_paths = self.prefixes[prefix].searchpath
-
-            for root_path_rec in root_paths:
-                component_path = root_path_rec["component_path"]
-                root_path = Path(root_path_rec["root_path"])
-
-                for file_absolute_folder, _folders, files in os.walk(
-                    component_path, topdown=False, followlinks=True
-                ):
-                    namespace = os.path.relpath(
-                        file_absolute_folder, component_path
-                    ).strip(".")
-                    for filename in files:
-                        yield root_path, namespace, Path(
-                            file_absolute_folder
-                        ) / filename
 
     def _get_component_path(
         self, prefix: str, name: str, file_ext: "TFileExt" = ""
     ) -> "tuple[Path, Path]":
-        file_ext = file_ext or self.file_ext
-        if not file_ext.startswith("."):
-            file_ext = "." + file_ext
-        name = name.replace(SLASH, DELIMITER) + file_ext
+        name = name.replace(SLASH, DELIMITER)
 
         paths = self.component_paths
         if name in paths:
@@ -120,6 +108,9 @@ def _get_component_path(
     def _get_from_file(
         self, *, prefix: str, name: str, url_prefix: str, file_ext: str
     ) -> "Component":
+        return super()._get_from_file(
+            prefix=prefix, name=name, url_prefix=url_prefix, file_ext=file_ext
+        )
         root_path, path = self._get_component_path(prefix, name, file_ext=file_ext)
         component = Component(
             name=name,
@@ -131,6 +122,32 @@ def _get_from_file(
         component.tmpl = self.jinja_env.get_template(tmpl_name)
         return component
 
+    def list_templates(self):
+        searchpath = []
+
+        app_theme = current_app.config.get("APP_THEME", None)
+
+        for path in self.jinja_env.loader.list_templates():
+            if not path.endswith(DEFAULT_EXTENSION):
+                continue
+            jinja_template = self.jinja_env.loader.load(self.jinja_env, path)
+            absolute_path = Path(jinja_template.filename)
+            template_name, stripped = strip_app_theme(jinja_template.name, app_theme)
+            template_name = template_name.rstrip(DEFAULT_EXTENSION)
+            template_name = template_name.replace(SLASH, DELIMITER)
+
+            # extract priority
+            split_name = list(template_name.rsplit(DELIMITER, 1))
+            split_name[-1], priority = self._extract_priority(split_name[-1])
+            template_name = DELIMITER.join(split_name)
+
+            if stripped:
+                priority += 10
+
+            searchpath.append(SearchPathItem(template_name, absolute_path, path, priority))
+
+        return searchpath
+
 
 def lazy_string_encoder(obj):
     if isinstance(obj, list):
@@ -139,3 +156,12 @@ def lazy_string_encoder(obj):
         return {key: lazy_string_encoder(value) for key, value in obj.items()}
     else:
         return str(obj)
+
+
+
+def strip_app_theme(template_name, app_theme):
+    if app_theme:
+        for theme in app_theme:
+            if template_name.startswith(f"{theme}/"):
+                return template_name[len(theme) + 1:], True
+    return template_name, False

oarepo_ui/resources/templating.py

@@ -2,33 +2,6 @@
 from jinja2.loaders import BaseLoader
 
 
-class RegistryLoader(BaseLoader):
-    def __init__(self, parent_loader) -> None:
-        super().__init__()
-        self.parent_loader = parent_loader
-
-    def get_source(self, environment: "Environment", template: str):
-        return self.parent_loader.get_source(environment=environment, template=template)
-
-    def list_templates(self):
-        return self.parent_loader.list_templates()
-
-    def load(
-        self,
-        environment: "Environment",
-        name: str,
-        globals=None,
-    ):
-        return self.parent_loader.load(
-            environment=environment, name=name, globals=globals
-        )
-
-
-def to_dict(value=None):
-    if value:
-        return value
-
-
 class TemplateRegistry:
     def __init__(self, app, ui_state) -> None:
         self.app = app
@@ -45,7 +18,7 @@ def jinja_env(self):
             return self._cached_jinja_env
 
         self._cached_jinja_env = self.app.jinja_env.overlay(
-            loader=RegistryLoader(self.app.jinja_env.loader),
+            loader=self.app.jinja_env.loader,
             extensions=[],
         )
         self._cached_jinja_env.filters["id"] = id_filter
@@ -55,3 +28,9 @@ def jinja_env(self):
 
 def id_filter(x):
     return id(x)
+
+
+# TODO: do we still need this ?
+def to_dict(value=None):
+    if value:
+        return value

oarepo_ui/utils.py

@@ -2,47 +2,6 @@
 from marshmallow.schema import SchemaMeta
 from marshmallow_utils.fields import NestedAttribute
 
-num2words = {
-    1: "one",
-    2: "two",
-    3: "three",
-    4: "four",
-    5: "five",
-    6: "six",
-    7: "seven",
-    8: "eight",
-    9: "nine",
-    10: "ten",
-    11: "eleven",
-    12: "twelve",
-    13: "thirteen",
-    14: "fourteen",
-    15: "fifteen",
-    16: "sixteen",
-    17: "seventeen",
-    18: "eighteen",
-    19: "nineteen",
-    20: "twenty",
-    30: "thirty",
-    40: "forty",
-    50: "fifty",
-    60: "sixty",
-    70: "seventy",
-    80: "eighty",
-    90: "ninety",
-    0: "zero",
-}
-
-
-def n2w(n):
-    try:
-        return num2words[n]
-    except KeyError:
-        try:
-            return num2words[n - n % 10] + num2words[n % 10].lower()
-        except KeyError:
-            raise AttributeError("Number out of range")
-
 
 def dump_empty(schema_or_field):
     """Return a full json-compatible dict of schema representation with empty values."""