Merge pull request #152 from khaeru/enh/reader-xml-2023-W52

Add metadata structures and sets to IM and SDMX-ML reader

.gitignore

@@ -10,6 +10,7 @@ __pycache__
 .coverage*
 .mypy_cache
 .pytest_cache
+.ruff_cache
 build
 coverage.xml
 dist

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
 - repo: https://github.com/pre-commit/mirrors-mypy
-  rev: v1.6.1
+  rev: v1.8.0
   hooks:
   - id: mypy
     additional_dependencies:
@@ -15,7 +15,7 @@ repos:
     - types-requests
     args: []
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.1.2
+  rev: v0.1.9
   hooks:
   - id: ruff
   - id: ruff-format

doc/api/model-common-list.rst

@@ -1,5 +1,6 @@
 .. This file is auto-generated by doc/conf.py.
 
+:obj:`~.common.ActionType`
 :obj:`~.common.Agency`
 :obj:`~.common.AgencyScheme`
 :obj:`~.common.AnnotableArtefact`
@@ -12,12 +13,14 @@
 :obj:`~.common.CategoryScheme`
 :obj:`~.common.Code`
 :obj:`~.common.Codelist`
+:obj:`~.common.CodingFormat`
 :obj:`~.common.Component`
 :obj:`~.common.ComponentList`
 :obj:`~.common.Concept`
 :obj:`~.common.ConceptScheme`
 :obj:`~.common.ConstrainableArtefact`
 :obj:`~.common.ConstraintRole`
+:obj:`~.common.ConstraintRoleType`
 :obj:`~.common.Contact`
 :obj:`~.common.CubeRegion`
 :obj:`~.common.CustomType`
@@ -34,19 +37,23 @@
 :obj:`~.common.DimensionDescriptor`
 :obj:`~.common.DimensionRelationship`
 :obj:`~.common.EndPeriod`
+:obj:`~.common.ExtendedFacetValueType`
 :obj:`~.common.Facet`
 :obj:`~.common.FacetType`
+:obj:`~.common.FacetValueType`
 :obj:`~.common.FromVTLSpaceKey`
 :obj:`~.common.GroupDimensionDescriptor`
 :obj:`~.common.GroupKey`
 :obj:`~.common.GroupRelationship`
+:obj:`~.common.HierarchicalCode`
 :obj:`~.common.ISOConceptReference`
 :obj:`~.common.IdentifiableArtefact`
 :obj:`~.common.InternationalString`
 :obj:`~.common.Item`
 :obj:`~.common.ItemScheme`
 :obj:`~.common.Key`
 :obj:`~.common.KeyValue`
+:obj:`~.common.Level`
 :obj:`~.common.MaintainableArtefact`
 :obj:`~.common.MetadataTargetRegion`
 :obj:`~.common.NamePersonalisation`

doc/api/model-v21-list.rst

@@ -2,27 +2,55 @@
 
 :obj:`~.v21.AfterPeriod`
 :obj:`~.v21.BeforePeriod`
+:obj:`~.v21.CodeMap`
+:obj:`~.v21.CodelistMap`
 :obj:`~.v21.Constraint`
 :obj:`~.v21.ContentConstraint`
 :obj:`~.v21.DataKey`
 :obj:`~.v21.DataKeySet`
+:obj:`~.v21.DataSetTarget`
 :obj:`~.v21.DataStructureDefinition`
 :obj:`~.v21.DataflowDefinition`
+:obj:`~.v21.DimensionDescriptorValuesTarget`
+:obj:`~.v21.EnumeratedAttributeValue`
 :obj:`~.v21.GenericDataSet`
 :obj:`~.v21.GenericTimeSeriesDataSet`
+:obj:`~.v21.HierarchicalCodelist`
+:obj:`~.v21.Hierarchy`
+:obj:`~.v21.IdentifiableObjectTarget`
+:obj:`~.v21.ItemAssociation`
+:obj:`~.v21.ItemSchemeMap`
 :obj:`~.v21.MeasureDescriptor`
 :obj:`~.v21.MeasureDimension`
 :obj:`~.v21.MemberSelection`
 :obj:`~.v21.MemberValue`
+:obj:`~.v21.MetadataReport`
+:obj:`~.v21.MetadataSet`
 :obj:`~.v21.MetadataStructureDefinition`
+:obj:`~.v21.MetadataTarget`
 :obj:`~.v21.MetadataflowDefinition`
 :obj:`~.v21.NoSpecifiedRelationship`
+:obj:`~.v21.NonEnumeratedAttributeValue`
 :obj:`~.v21.Observation`
+:obj:`~.v21.OtherNonEnumeratedAttributeValue`
 :obj:`~.v21.PrimaryMeasure`
 :obj:`~.v21.PrimaryMeasureRelationship`
 :obj:`~.v21.RangePeriod`
+:obj:`~.v21.ReportPeriodTarget`
+:obj:`~.v21.ReportStructure`
+:obj:`~.v21.ReportedAttribute`
+:obj:`~.v21.ReportingCategory`
+:obj:`~.v21.ReportingTaxonomy`
 :obj:`~.v21.ReportingYearStartDay`
 :obj:`~.v21.SelectionValue`
+:obj:`~.v21.StructureSet`
 :obj:`~.v21.StructureSpecificDataSet`
 :obj:`~.v21.StructureSpecificTimeSeriesDataSet`
+:obj:`~.v21.TargetIdentifiableObject`
+:obj:`~.v21.TargetObject`
+:obj:`~.v21.TargetObjectKey`
+:obj:`~.v21.TargetObjectValue`
+:obj:`~.v21.TargetReportPeriod`
+:obj:`~.v21.TextAttributeValue`
 :obj:`~.v21.TimeRangeValue`
+:obj:`~.v21.XHTMLAttributeValue`

doc/api/model-v30-list.rst

@@ -2,8 +2,8 @@
 
 :obj:`~.v30.AfterPeriod`
 :obj:`~.v30.BeforePeriod`
+:obj:`~.v30.CodedMetadataAttributeValue`
 :obj:`~.v30.CodelistExtension`
-:obj:`~.v30.CodingFormat`
 :obj:`~.v30.Constraint`
 :obj:`~.v30.DataConstraint`
 :obj:`~.v30.DataKey`
@@ -17,25 +17,32 @@
 :obj:`~.v30.GeoGridCodelist`
 :obj:`~.v30.GeoRefCode`
 :obj:`~.v30.GeographicCodelist`
-:obj:`~.v30.HierarchicalCode`
 :obj:`~.v30.Hierarchy`
 :obj:`~.v30.HierarchyAssociation`
-:obj:`~.v30.Level`
+:obj:`~.v30.IdentifiableObjectSelection`
 :obj:`~.v30.Measure`
 :obj:`~.v30.MeasureDescriptor`
 :obj:`~.v30.MeasureRelationship`
 :obj:`~.v30.MemberSelection`
 :obj:`~.v30.MemberValue`
+:obj:`~.v30.MetadataAttributeDescriptor`
+:obj:`~.v30.MetadataAttributeValue`
 :obj:`~.v30.MetadataConstraint`
 :obj:`~.v30.MetadataProvider`
 :obj:`~.v30.MetadataProviderScheme`
+:obj:`~.v30.MetadataSet`
 :obj:`~.v30.MetadataStructureDefinition`
 :obj:`~.v30.Metadataflow`
 :obj:`~.v30.Observation`
 :obj:`~.v30.ObservationRelationship`
+:obj:`~.v30.OtherUncodedAttributeValue`
 :obj:`~.v30.RangePeriod`
 :obj:`~.v30.SelectionValue`
 :obj:`~.v30.StructureSpecificDataSet`
+:obj:`~.v30.TargetIdentifiableObject`
+:obj:`~.v30.TextAttributeValue`
 :obj:`~.v30.TimeRangeValue`
+:obj:`~.v30.UncodedMetadataAttributeValue`
 :obj:`~.v30.ValueItem`
 :obj:`~.v30.ValueList`
+:obj:`~.v30.XHTMLAttributeValue`

doc/conf.py

@@ -87,13 +87,11 @@ def linkcode_resolve(domain, info):
 # If True, todo and todolist produce output, else they produce nothing
 todo_include_todos = True
 
-
 # -- Options for IPython.sphinxext.ipython_directive -----------------------------------
 
 # Specify if the embedded Sphinx shell should import Matplotlib and set the backend
 ipython_mplbackend = ""
 
-
 # -- Dynamic configuration
 
 

doc/dev.rst

@@ -137,6 +137,11 @@ Address any failures before releasing.
 Internal code reference
 =======================
 
+.. automodule:: sdmx.dictlike
+   :noindex:
+   :undoc-members:
+   :show-inheritance:
+
 ``testing``: Testing utilities
 ------------------------------
 
@@ -147,13 +152,12 @@ Internal code reference
 
 ``util``: Utilities
 -------------------
+
 .. automodule:: sdmx.util
    :noindex:
-   :members: summarize_dictlike
    :undoc-members:
    :show-inheritance:
 
-
 Inline TODOs
 ============
 

doc/whatsnew.rst

@@ -3,8 +3,69 @@
 What's new?
 ***********
 
-.. Next release
-.. ============
+Next release
+============
+
+- Expand :mod:`.model` and :mod:`.reader.xml` support for metadata structures and metadata sets (§7 of the Information Model in both SDMX 2.1 and 3.0) (:issue:`73`, :pull:`152`).
+  This includes the additional classes:
+
+  - :mod:`.model.common`:
+    :class:`.CodingFormat`
+    :class:`.ExtendedFacetValueType`
+    :class:`.HierarchicalCode`
+    :class:`.Level`.
+  - :mod:`.model.v21`:
+    :class:`.CodelistMap`
+    :class:`.CodeMap`
+    :class:`.DataSetTarget`
+    :class:`.DimensionDescriptorValuesTarget`
+    :class:`.EnumeratedAttributeValue`
+    :class:`.IdentifiableObjectTarget`
+    :class:`.ItemAssociation`
+    :class:`.ItemSchemeMap`
+    :class:`.MetadataReport`
+    :class:`.MetadataSet`
+    :class:`.MetadataTarget`
+    :class:`.NonEnumeratedAttributeValue`
+    :class:`.OtherNonEnumeratedAttributeValue`
+    :class:`.ReportedAttribute`
+    :class:`.ReportingCategory`
+    :class:`.ReportingTaxonomy`
+    :class:`.ReportPeriodTarget`
+    :class:`.ReportStructure`
+    :class:`.StructureSet`
+    :class:`.TargetIdentifiableObject`
+    :class:`.TargetObject`
+    :class:`.TargetObjectKey`
+    :class:`.TargetObjectValue`
+    :class:`.TargetReportPeriod`
+    :class:`.TextAttributeValue`
+    :class:`.XHTMLAttributeValue`.
+  - :mod:`.model.v30`:
+    :class:`.CodedMetadataAttributeValue`
+    :class:`.IdentifiableObjectSelection`
+    :class:`.MetadataAttributeDescriptor`
+    :class:`.MetadataAttributeValue`
+    :class:`.Metadataflow`
+    :class:`.MetadataSet`
+    :class:`.MetadataStructureDefinition`
+    :class:`.OtherUncodedAttributeValue`
+    :class:`.TargetIdentifiableObject`
+    :class:`.TextAttributeValue`
+    :class:`.UncodedMetadataAttributeValue`
+    :class:`.XHTMLAttributeValue`.
+- New collections on StructureMessage:
+  :attr:`.hierarchical_codelist`,
+  :attr:`.hierarchy`,
+  :attr:`.metadatastructure`.
+- New class :class:`.MetadataMessage`.
+- Improve :class:`.Structure`:
+
+  - New attribute :attr:`~.Structure.grouping` per the information model.
+  - New convenience method :meth:`~.Structure.replace_grouping`.
+- :mod:`.reader.xml` parses messages available from 'actualconstraint', 'allowedconstraint', 'contentconstraint', 'hierarchicalcodelist', 'metadatstructure', 'structure', and 'structureset' SDMX 2.1 REST API endpoints for all known data sources that support these.
+
+  - Expand explicit marking of particular data sources that do not support the above endpoints.
 
 v2.12.1 (2023-12-20)
 ====================

pyproject.toml

@@ -63,6 +63,8 @@ omit = [
 exclude_also = [
   # Don't complain about abstract methods, they aren't run
   "@(abc\\.)?abstractmethod",
+  # Imports only used by type checkers
+  "if TYPE_CHECKING:",
 ]
 
 [tool.mypy]

sdmx/format/xml/common.py

@@ -25,6 +25,8 @@
     "DataConsumerScheme",
     "DataProvider",
     "DataProviderScheme",
+    "HierarchicalCode",
+    "Level",
     "NamePersonalisation",
     "NamePersonalisationScheme",
     "Ruleset",
@@ -39,17 +41,14 @@
     ("model.Agency", "str:Agency"),  # Order matters
     ("model.Agency", "mes:Receiver"),
     ("model.Agency", "mes:Sender"),
+    ("model.AttributeDescriptor", "str:AttributeList"),
     ("model.Concept", "str:ConceptIdentity"),
     ("model.Codelist", "str:Enumeration"),  # This could possibly be ItemScheme
     ("model.Dimension", "str:Dimension"),  # Order matters
     ("model.Dimension", "str:DimensionReference"),
     ("model.Dimension", "str:GroupDimension"),
-    ("model.StructureUsage", "com:StructureUsage"),
-    ("model.AttributeDescriptor", "str:AttributeList"),
     ("model.DataAttribute", "str:Attribute"),
     ("model.DataStructureDefinition", "str:DataStructure"),
-    ("model.DataStructureDefinition", "com:Structure"),
-    ("model.DataStructureDefinition", "str:Structure"),
     ("model.DimensionDescriptor", "str:DimensionList"),
     ("model.GroupDimensionDescriptor", "str:Group"),
     ("model.GroupDimensionDescriptor", "str:AttachmentGroup"),
@@ -58,10 +57,14 @@
     ("model.MeasureDescriptor", "str:MeasureList"),
     ("model.MetadataStructureDefinition", "str:MetadataStructure"),
     ("model.SeriesKey", "gen:SeriesKey"),
+    ("model.Structure", "com:Structure"),
+    ("model.Structure", "str:Structure"),
     ("model.StructureUsage", "com:StructureUsage"),
     ("model.VTLMappingScheme", "str:VtlMappingScheme"),
     # Message classes
     ("message.DataMessage", "mes:StructureSpecificData"),
+    ("message.MetadataMessage", "mes:GenericMetadata"),
+    ("message.MetadataMessage", "mes:StructureSpecificMetadata"),
     ("message.ErrorMessage", "mes:Error"),
     ("message.StructureMessage", "mes:Structure"),
 ]
@@ -72,6 +75,7 @@
     "xsi": "http://www.w3.org/2001/XMLSchema-instance",
     # To be formatted
     "com": "{}/common",
+    "md": "{}/metadata/generic",
     "data": "{}/data/structurespecific",
     "str": "{}/structure",
     "mes": "{}/message",
@@ -123,15 +127,16 @@ def qname(self, ns_or_name, name=None) -> QName:
         else:
             if name is None:
                 match = re.fullmatch(
-                    r"(\{(?P<ns_full>.*)\}|(?P<ns_key>.*):)(?P<name>.*)", ns_or_name
+                    r"(\{(?P<ns_full>.*)\}|(?P<ns_key>.*):)?(?P<name>.*)", ns_or_name
                 )
                 assert match
                 name = match.group("name")
-                ns_key = match.group("ns_key")
-                if ns_key:
+                if ns_key := match.group("ns_key"):
                     ns = self.NS[ns_key]
+                elif ns := match.group("ns_full"):
+                    pass
                 else:
-                    ns = match.group("ns_full")
+                    ns = None
             else:
                 ns = self.NS[ns_or_name]