Verification

.gitignore

@@ -201,6 +201,8 @@ cython_debug/
 *.txt
 !requirements.txt
 !dev-requirements.txt
+!pynxtools/dataconverter/units/default_en.txt
+!pynxtools/dataconverter/units/constants_en.txt
 !mkdocs-requirements.txt
 !pynxtools/nexus-version.txt
 build/

MANIFEST.in

@@ -2,5 +2,6 @@ recursive-include pynxtools/definitions/base_classes/ *.xml
 recursive-include pynxtools/definitions/applications/ *.xml
 recursive-include pynxtools/definitions/contributed_definitions/ *.xml
 include pynxtools/definitions/*.xsd
+include pynxtools/dataconverter/units *.txt
 include pynxtools/nexus-version.txt
-include pynxtools/definitions/NXDL_VERSION
+include pynxtools/definitions/NXDL_VERSION

pynxtools/dataconverter/helpers.py

@@ -16,21 +16,27 @@
 # limitations under the License.
 #
 """Helper functions commonly used by the convert routine."""
-
 import json
 import logging
 import re
+import sys
 import xml.etree.ElementTree as ET
 from datetime import datetime, timezone
+from functools import lru_cache
 from typing import Any, Callable, List, Optional, Tuple, Union
 
 import h5py
 import numpy as np
 from ase.data import chemical_symbols
 
 from pynxtools import get_nexus_version, get_nexus_version_hash
+from pynxtools.dataconverter.units import ureg
 from pynxtools.nexus import nexus
-from pynxtools.nexus.nexus import NxdlAttributeError
+from pynxtools.nexus.nexus import NxdlAttributeError, get_inherited_nodes
+
+logger = logging.getLogger(__name__)  # pylint: disable=C0103
+logger.setLevel(logging.INFO)
+logger.addHandler(logging.StreamHandler(sys.stdout))
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
@@ -386,6 +392,35 @@ def is_valid_data_field(value, nxdl_type, path):
     return value
 
 
+def is_valid_unit(unit: str, nx_category: str) -> bool:
+    """
+    The provided unit belongs to the provided nexus unit category.
+
+    Args:
+        unit (str): The unit to check. Should be according to pint.
+        nx_category (str): A nexus unit category, e.g. `NX_LENGTH`,
+            or derived unit category, e.g., `NX_LENGTH ** 2`.
+
+    Returns:
+        bool: The unit belongs to the provided category
+    """
+    if nx_category in ("NX_ANY"):
+        ureg(unit)  # Check if unit is generally valid
+        return True
+    nx_category = re.sub(r"(NX_[A-Z]+)", r"[\1]", nx_category)
+    if nx_category == "[NX_TRANSFORMATION]":
+        # NX_TRANSFORMATIONS is a pseudo unit
+        # and can be either an angle, a length or unitless
+        return True
+        # Currently disabled for the mpes tests
+        # return (
+        #     ureg(unit).check("[NX_ANGLE]")
+        #     or ureg(unit).check("[NX_LENGTH]")
+        #     or ureg(unit).check("[NX_UNITLESS]")
+        # )
+    return ureg(unit).check(f"{nx_category}")
+
+
 def path_in_data_dict(nxdl_path: str, hdf_path: str, data: dict) -> Tuple[bool, str]:
     """Checks if there is an accepted variation of path in the dictionary & returns the path."""
     accepted_unfilled_key = None
@@ -577,9 +612,9 @@ def validate_data_dict(template, data, nxdl_root: ET.Element):
     """Checks whether all the required paths from the template are returned in data dict."""
     assert nxdl_root is not None, "The NXDL file hasn't been loaded."
 
-    # nxdl_path_set helps to skip validation check on the same type of nxdl signiture
-    # This reduces huge amount of runing time
-    nxdl_path_to_elm: dict = {}
+    @lru_cache
+    def get_xml_node(nxdl_path: str) -> ET.Element:
+        return nexus.get_node_at_nxdl_path(nxdl_path=nxdl_path, elem=nxdl_root)
 
     # Make sure all required fields exist.
     ensure_all_required_fields_exist(template, data, nxdl_root)
@@ -590,18 +625,42 @@ def validate_data_dict(template, data, nxdl_root: ET.Element):
             entry_name = get_name_from_data_dict_entry(path[path.rindex("/") + 1 :])
             nxdl_path = convert_data_converter_dict_to_nxdl_path(path)
 
-            if entry_name == "@units":
-                continue
-
             if entry_name[0] == "@" and "@" in nxdl_path:
                 index_of_at = nxdl_path.rindex("@")
                 nxdl_path = nxdl_path[0:index_of_at] + nxdl_path[index_of_at + 1 :]
 
-            if nxdl_path in nxdl_path_to_elm:
-                elem = nxdl_path_to_elm[nxdl_path]
-            else:
-                elem = nexus.get_node_at_nxdl_path(nxdl_path=nxdl_path, elem=nxdl_root)
-                nxdl_path_to_elm[nxdl_path] = elem
+            if entry_name == "@units":
+                elempath = get_inherited_nodes(nxdl_path, None, nxdl_root)[1]
+                elem = elempath[-2]
+                field_path = path.rsplit("/", 1)[0]
+                if (
+                    field_path not in data.get_documented()
+                    and "units" not in elem.attrib
+                ):
+                    logger.warning(
+                        "The unit, %s = %s, is being written but has no documentation.",
+                        path,
+                        data[path],
+                    )
+                    continue
+
+                field = nexus.get_node_at_nxdl_path(
+                    nxdl_path=convert_data_converter_dict_to_nxdl_path(
+                        # The part below is the backwards compatible version of
+                        # nxdl_path.removesuffix("/units")
+                        nxdl_path[:-6] if nxdl_path.endswith("/units") else nxdl_path
+                    ),
+                    elem=nxdl_root,
+                )
+                nxdl_unit = field.attrib.get("units", "")
+                if not is_valid_unit(data[path], nxdl_unit):
+                    raise ValueError(
+                        f"Invalid unit in {path}. {data[path]} "
+                        f"is not in unit category {nxdl_unit}"
+                    )
+                continue
+
+            elem = get_xml_node(nxdl_path)
 
             # Only check for validation in the NXDL if we did find the entry
             # otherwise we just pass it along

pynxtools/dataconverter/readers/ellips/reader.py

@@ -447,6 +447,7 @@ def read(
                 # MK:: Carola, Ron, Flo, Tamas, Sandor refactor the following line
                 template[f"/ENTRY[entry]/plot/DATA[{key}_errors]/@units"] = "degree"
 
+        template["/ENTRY[entry]/data_collection/measured_data/@units"] = ""
         # Define default plot showing Psi and Delta at all angles:
         template["/@default"] = "entry"
         template["/ENTRY[entry]/@default"] = "plot"

pynxtools/dataconverter/units/__init__.py

@@ -0,0 +1,23 @@
+#
+# Copyright The NOMAD Authors.
+#
+# This file is part of NOMAD. See https://nomad-lab.eu for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""A unit registry for nexus units"""
+
+import os
+from pint import UnitRegistry
+
+ureg = UnitRegistry(os.path.join(os.path.dirname(__file__), "default_en.txt"))

pynxtools/dataconverter/units/constants_en.txt

@@ -0,0 +1,73 @@
+# Default Pint constants definition file
+# Based on the International System of Units
+# Language: english
+# Source: https://physics.nist.gov/cuu/Constants/
+#         https://physics.nist.gov/PhysRefData/XrayTrans/Html/search.html
+# :copyright: 2013,2019 by Pint Authors, see AUTHORS for more details.
+
+#### MATHEMATICAL CONSTANTS ####
+# As computed by Maxima with fpprec:50
+
+pi     = 3.1415926535897932384626433832795028841971693993751 = π  # pi
+tansec = 4.8481368111333441675396429478852851658848753880815e-6   # tangent of 1 arc-second ~ arc_second/radian
+ln10   = 2.3025850929940456840179914546843642076011014886288      # natural logarithm of 10
+wien_x = 4.9651142317442763036987591313228939440555849867973      # solution to (x-5)*exp(x)+5 = 0 => x = W(5/exp(5))+5
+wien_u = 2.8214393721220788934031913302944851953458817440731      # solution to (u-3)*exp(u)+3 = 0 => u = W(3/exp(3))+3
+
+#### DEFINED EXACT CONSTANTS ####
+
+speed_of_light = 299792458 m/s = c = c_0                      # since 1983
+planck_constant = 6.62607015e-34 J s = h                      # since May 2019
+elementary_charge = 1.602176634e-19 C = e                     # since May 2019
+avogadro_number = 6.02214076e23                               # since May 2019
+boltzmann_constant = 1.380649e-23 J K^-1 = k = k_B            # since May 2019
+standard_gravity = 9.80665 m/s^2 = g_0 = g0 = g_n = gravity   # since 1901
+standard_atmosphere = 1.01325e5 Pa = atm = atmosphere         # since 1954
+conventional_josephson_constant = 4.835979e14 Hz / V = K_J90  # since Jan 1990
+conventional_von_klitzing_constant = 2.5812807e4 ohm = R_K90  # since Jan 1990
+
+#### DERIVED EXACT CONSTANTS ####
+# Floating-point conversion may introduce inaccuracies
+
+zeta = c / (cm/s) = ζ
+dirac_constant = h / (2 * π) = ħ = h_bar = atomic_unit_of_action = a_u_action
+avogadro_constant = avogadro_number * mol^-1 = N_A
+molar_gas_constant = k * N_A = R
+faraday_constant = e * N_A
+conductance_quantum = 2 * e ** 2 / h = G_0
+magnetic_flux_quantum = h / (2 * e) = Φ_0 = Phi_0
+josephson_constant = 2 * e / h = K_J
+von_klitzing_constant = h / e ** 2 = R_K
+stefan_boltzmann_constant = 2 / 15 * π ** 5 * k ** 4 / (h ** 3 * c ** 2) = σ = sigma
+first_radiation_constant = 2 * π * h * c ** 2 = c_1
+second_radiation_constant = h * c / k = c_2
+wien_wavelength_displacement_law_constant = h * c / (k * wien_x)
+wien_frequency_displacement_law_constant = wien_u * k / h
+
+#### MEASURED CONSTANTS ####
+# Recommended CODATA-2018 values
+# To some extent, what is measured and what is derived is a bit arbitrary.
+# The choice of measured constants is based on convenience and on available uncertainty.
+# The uncertainty in the last significant digits is given in parentheses as a comment.
+
+newtonian_constant_of_gravitation = 6.67430e-11 m^3/(kg s^2) = _ = gravitational_constant  # (15)
+rydberg_constant = 1.0973731568160e7 * m^-1 = R_∞ = R_inf                                  # (21)
+electron_g_factor = -2.00231930436256 = g_e                                                # (35)
+atomic_mass_constant = 1.66053906660e-27 kg = m_u                                          # (50)
+electron_mass = 9.1093837015e-31 kg = m_e = atomic_unit_of_mass = a_u_mass                 # (28)
+proton_mass = 1.67262192369e-27 kg = m_p                                                   # (51)
+neutron_mass = 1.67492749804e-27 kg = m_n                                                  # (95)
+K_alpha_Cu_d_220 = 0.80232719                                                              # (22)
+K_alpha_Mo_d_220 = 0.36940604                                                              # (19)
+K_alpha_W_d_220 = 0.108852175                                                              # (98)
+
+#### DERIVED CONSTANTS ####
+
+fine_structure_constant = (2 * h * R_inf / (m_e * c)) ** 0.5 = α = alpha
+vacuum_permeability = 2 * α * h / (e ** 2 * c) = µ_0 = mu_0 = mu0 = magnetic_constant
+vacuum_permittivity = e ** 2 / (2 * α * h * c) = ε_0 = epsilon_0 = eps_0 = eps0 = electric_constant
+impedance_of_free_space = 2 * α * h / e ** 2 = Z_0 = characteristic_impedance_of_vacuum
+coulomb_constant = α * h_bar * c / e ** 2 = k_C
+classical_electron_radius = α * h_bar / (m_e * c) = r_e
+thomson_cross_section = 8 / 3 * π * r_e ** 2 = σ_e = sigma_e
+