From e2c9b987125098290e4c36b9962586eeb9bae241 Mon Sep 17 00:00:00 2001 From: Ivan Ruiz Manuel <72193617+irm-codebase@users.noreply.github.com> Date: Wed, 27 Nov 2024 20:54:29 +0100 Subject: [PATCH 01/13] Move YAML functionality to calliope.io (pre AttrDict YAML removal). --- src/calliope/__init__.py | 2 +- src/calliope/io.py | 105 +++++++++++++++++++++- src/calliope/model.py | 5 ++ tests/test_core_attrdict.py | 159 +-------------------------------- tests/test_io.py | 172 ++++++++++++++++++++++++++++++++++++ 5 files changed, 284 insertions(+), 159 deletions(-) diff --git a/src/calliope/__init__.py b/src/calliope/__init__.py index 794ebc82..22d02097 100644 --- a/src/calliope/__init__.py +++ b/src/calliope/__init__.py @@ -3,7 +3,7 @@ from calliope import examples, exceptions from calliope._version import __version__ from calliope.attrdict import AttrDict -from calliope.model import Model, read_netcdf +from calliope.model import Model, read_netcdf, read_yaml from calliope.util.logging import set_log_verbosity __title__ = "Calliope" diff --git a/src/calliope/io.py b/src/calliope/io.py index 205ffe7f..c466c901 100644 --- a/src/calliope/io.py +++ b/src/calliope/io.py @@ -1,8 +1,11 @@ # Copyright (C) since 2013 Calliope contributors listed in AUTHORS. # Licensed under the Apache 2.0 License (see LICENSE file). -"""Functions to read and save model results.""" +"""Functions to read and save model results and configuration.""" import importlib.resources +import logging +import os +from collections.abc import Iterable from copy import deepcopy from pathlib import Path @@ -11,11 +14,14 @@ import netCDF4 # noqa: F401 import numpy as np import pandas as pd +import ruamel.yaml as ruamel_yaml import xarray as xr from calliope import exceptions from calliope.attrdict import AttrDict -from calliope.util.tools import listify +from calliope.util.tools import climb_template_tree, listify, relative_path + +logger = logging.getLogger(__name__) CONFIG_DIR = importlib.resources.files("calliope") / "config" @@ -208,5 +214,98 @@ def save_csv( def load_config(filename: str): """Load model configuration from a file.""" with importlib.resources.as_file(CONFIG_DIR / filename) as f: - loaded = AttrDict.from_yaml(f) + loaded = read_rich_yaml(f) return loaded + + +def read_rich_yaml( + yaml: str | Path, + allow_override: bool = False, + template_sections: None | Iterable[str] = None, +) -> AttrDict: + """Returns an AttrDict initialised from the given YAML file or string. + + Uses calliope's "flavour" for YAML files. + + Args: + yaml (str | Path): YAML file path or string. + resolve_imports (bool, optional): Solve imports recursively. Defaults to True. + allow_override (bool, optional): Allow overrides for already defined keys. Defaults to False. + template_sections: (Iterable[str], optional): Replace tempalte for the requested sections. Defaults to False. + + Raises: + ValueError: Import solving requested for non-file input YAML. + """ + if isinstance(yaml, str) and not os.path.exists(yaml): + yaml_path = None + yaml_text = yaml + else: + yaml_path = Path(yaml) + yaml_text = yaml_path.read_text(encoding="utf-8") + + yaml_dict = AttrDict(_yaml_load(yaml_text)) + yaml_dict = _resolve_yaml_imports( + yaml_dict, base_path=yaml_path, allow_override=allow_override + ) + if template_sections: + yaml_dict = _resolve_yaml_templates(yaml_dict, template_sections) + return yaml_dict + + +def _yaml_load(src): + """Load YAML from a file object or path with useful parser errors.""" + yaml = ruamel_yaml.YAML(typ="safe") + if not isinstance(src, str): + try: + src_name = src.name + except AttributeError: + src_name = "" + # Force-load file streams as that allows the parser to print + # much more context when it encounters an error + src = src.read() + else: + src_name = "" + try: + result = yaml.load(src) + if not isinstance(result, dict): + raise ValueError(f"Could not parse {src_name} as YAML") + return result + except ruamel_yaml.YAMLError: + logger.error(f"Parser error when reading YAML from {src_name}.") + raise + + +def _resolve_yaml_imports( + loaded: AttrDict, base_path: str | Path | None, allow_override: bool +) -> AttrDict: + loaded_dict = loaded + imports = loaded_dict.get_key("import", None) + if imports: + if not isinstance(imports, list): + raise ValueError("`import` must be a list.") + if base_path is None: + raise ValueError("Imports are not possible for non-file yaml inputs.") + + for k in imports: + path = relative_path(base_path, k) + imported = read_rich_yaml(path) + # loaded is added to imported (i.e. it takes precedence) + imported.union(loaded_dict, allow_override=allow_override) + loaded_dict = imported + # 'import' key itself is no longer needed + loaded_dict.del_key("import") + + return loaded_dict + + +def _resolve_yaml_templates(data: AttrDict, sections: Iterable[str]) -> AttrDict: + """Fill and then remove template definitions in the given sections.""" + if "templates" in data: + templates = data.pop("templates") + for section in sections: + for item, values in data[section].items(): + if "template" in values: + filled, _ = climb_template_tree(values, templates) + data[section][item] = filled + data[section][item].pop("template") + return data diff --git a/src/calliope/model.py b/src/calliope/model.py index ee8c5a77..65cd913d 100644 --- a/src/calliope/model.py +++ b/src/calliope/model.py @@ -39,6 +39,11 @@ def read_netcdf(path): return Model(model_definition=model_data) +def read_yaml(path: str | Path): + """Return a dictionary constructed using Calliope-flavoured YAML.""" + return io.read_rich_yaml(path, template_sections={"techs", "nodes", "data_tables"}) + + class Model: """A Calliope Model.""" diff --git a/tests/test_core_attrdict.py b/tests/test_core_attrdict.py index c65ab18e..32a8dd5b 100644 --- a/tests/test_core_attrdict.py +++ b/tests/test_core_attrdict.py @@ -1,10 +1,4 @@ -import os -import tempfile -from pathlib import Path - -import numpy as np import pytest -import ruamel.yaml as ruamel_yaml from calliope.attrdict import _MISSING, AttrDict @@ -22,30 +16,6 @@ def regular_dict(self): } return d - setup_string = """ - # a comment - a: 1 - b: 2 - # a comment about `c` - c: # a comment inline with `c` - x: foo # a comment on foo - - # - y: bar # - z: - I: 1 - II: 2 - d: - """ - - @pytest.fixture - def yaml_filepath(self): - this_path = Path(__file__).parent - return this_path / "common" / "yaml_file.yaml" - - @pytest.fixture - def yaml_string(self): - return self.setup_string @pytest.fixture def attr_dict(self, regular_dict): @@ -71,56 +41,6 @@ def test_init_from_dict_with_nested_keys(self): d = AttrDict({"foo.bar.baz": 1}) assert d.foo.bar.baz == 1 - def test_from_yaml_path(self, yaml_filepath): - d = AttrDict.from_yaml(yaml_filepath) - assert d.a == 1 - assert d.c.z.II == 2 - - def test_from_yaml_string(self, yaml_string): - d = AttrDict.from_yaml_string(yaml_string) - assert d.a == 1 - assert d.c.z.II == 2 - - def test_from_yaml_string_dot_strings(self): - yaml_string = "a.b.c: 1\na.b.foo: 2" - d = AttrDict.from_yaml_string(yaml_string) - assert d.a.b.c == 1 - assert d.a.b.foo == 2 - - def test_from_yaml_string_dot_strings_duplicate(self): - yaml_string = "a.b.c: 1\na.b.c: 2" - with pytest.raises(ruamel_yaml.constructor.DuplicateKeyError): - AttrDict.from_yaml_string(yaml_string) - - def test_simple_invalid_yaml(self): - yaml_string = "1 this is not valid yaml" - with pytest.raises(ValueError) as excinfo: # noqa: PT011, false positive - AttrDict.from_yaml_string(yaml_string) - assert check_error_or_warning(excinfo, "Could not parse as YAML") - - def test_parser_error(self): - with pytest.raises(ruamel_yaml.YAMLError): - AttrDict.from_yaml_string( - """ - foo: bar - baz: 1 - - foobar - bar: baz - - """ - ) - - def test_order_of_subdicts(self): - d = AttrDict.from_yaml_string( - """ - A.B.C: 10 - A.B: - E: 20 - """ - ) - assert d.A.B.C == 10 - assert d.A.B.E == 20 - def test_dot_access_first(self, attr_dict): d = attr_dict assert d.a == 1 @@ -130,7 +50,7 @@ def test_dot_access_second(self, attr_dict): assert d.c.x == "foo" def test_dot_access_list(self): - d = AttrDict.from_yaml_string("a: [{x: 1}, {y: 2}]") + d = AttrDict({"a": [{"x": 1}, {"y": 2}]}) assert d.a[0].x == 1 def test_set_key_first(self, attr_dict): @@ -226,12 +146,6 @@ def test_as_dict(self, attr_dict): assert dd["a"] == 1 assert dd["c"]["x"] == "foo" - def test_as_dict_with_sublists(self): - d = AttrDict.from_yaml_string("a: [{x: 1}, {y: 2}]") - dd = d.as_dict() - assert dd["a"][0]["x"] == 1 - assert isinstance(dd["a"][0], dict) # Not AttrDict! - def test_as_dict_flat(self, attr_dict): dd = attr_dict.as_dict(flat=True) assert dd["c.x"] == "foo" @@ -264,16 +178,10 @@ def test_union_duplicate_keys(self, attr_dict): @pytest.mark.parametrize("to_replace", ["foo", [], {}, 1]) def test_union_replacement(self, attr_dict, to_replace): d = attr_dict - d_new = AttrDict.from_yaml_string(f"c._REPLACE_: {to_replace}") + d_new = AttrDict({"c._REPLACE_": to_replace}) d.union(d_new, allow_override=True, allow_replacement=True) assert d.c == to_replace - def test_union_replacement_null(self, attr_dict): - d = attr_dict - d_new = AttrDict.from_yaml_string("c._REPLACE_: null") - d.union(d_new, allow_override=True, allow_replacement=True) - assert d.c is None - def test_union_empty_dicts(self, attr_dict): d = attr_dict d_new = AttrDict({"1": {"foo": {}}, "baz": {"bar": {}}}) @@ -300,66 +208,7 @@ def test_del_key_nested(self, attr_dict): attr_dict.del_key("c.z.I") assert "I" not in attr_dict.c.z - def test_to_yaml(self, yaml_filepath): - d = AttrDict.from_yaml(yaml_filepath) - d.set_key("numpy.some_int", np.int32(10)) - d.set_key("numpy.some_float", np.float64(0.5)) - d.a_list = [0, 1, 2] - with tempfile.TemporaryDirectory() as tempdir: - out_file = os.path.join(tempdir, "test.yaml") - d.to_yaml(out_file) - - with open(out_file) as f: - result = f.read() - - assert "some_int: 10" in result - assert "some_float: 0.5" in result - assert "a_list:\n- 0\n- 1\n- 2" in result - - def test_to_yaml_string(self, yaml_filepath): - d = AttrDict.from_yaml(yaml_filepath) - result = d.to_yaml() + def test_to_yaml_string(self, attr_dict): + result = attr_dict.to_yaml() assert "a: 1" in result - def test_import_must_be_list(self): - yaml_string = """ - import: 'somefile.yaml' - """ - with pytest.raises(ValueError) as excinfo: # noqa: PT011, false positive - AttrDict.from_yaml_string(yaml_string, resolve_imports=True) - assert check_error_or_warning(excinfo, "`import` must be a list.") - - def test_do_not_resolve_imports(self): - yaml_string = """ - import: ['somefile.yaml'] - """ - d = AttrDict.from_yaml_string(yaml_string, resolve_imports=False) - # Should not raise an error about a missing file, as we ask for - # imports not to be resolved - assert d["import"] == ["somefile.yaml"] - - def test_nested_import(self): - with tempfile.TemporaryDirectory() as tempdir: - imported_file = os.path.join(tempdir, "test_import.yaml") - imported_yaml = """ - somekey: 1 - anotherkey: 2 - """ - with open(imported_file, "w") as f: - f.write(imported_yaml) - - yaml_string = f""" - foobar: - import: - - {imported_file} - foo: - bar: 1 - baz: 2 - 3: - 4: 5 - """ - - d = AttrDict.from_yaml_string(yaml_string, resolve_imports="foobar") - - assert "foobar.somekey" in d.keys_nested() - assert d.get_key("foobar.anotherkey") == 2 diff --git a/tests/test_io.py b/tests/test_io.py index b496db6b..ad79b423 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -1,12 +1,16 @@ import os import tempfile +from pathlib import Path +import numpy as np import pytest # noqa: F401 +import ruamel.yaml as ruamel_yaml import xarray as xr import calliope import calliope.io from calliope import exceptions +from calliope.attrdict import AttrDict from .common.util import check_error_or_warning @@ -233,3 +237,171 @@ def test_save_per_spore(self): for i in ["0", "1", "2", "3"]: assert os.path.isfile(os.path.join(tempdir, "output", f"spore_{i}.nc")) assert not os.path.isfile(os.path.join(tempdir, "output.nc")) + + +class TestYaml: + + @pytest.fixture + def dummy_yaml_import(self): + return """ + import: ['somefile.yaml'] + """ + + def test_do_not_resolve_imports(self, dummy_yaml_import): + """Text inputs that attempt to import files should raise an error.""" + + with pytest.raises(ValueError) as exinfo: # noqa: PT011, false positive + calliope.io.read_rich_yaml(dummy_yaml_import) + + assert check_error_or_warning( + exinfo, + "Imports are not possible for non-file yaml inputs." + ) + + @pytest.fixture + def dummy_imported_file(self, tmp_path) -> Path: + file = tmp_path / "test_import.yaml" + text = """ + somekey.nested: 1 + anotherkey: 2 + """ + with open(file, "w") as f: + f.write(text) + return file + + def test_import(self, dummy_imported_file): + file = dummy_imported_file.parent / "main_file.yaml" + text = """ + import: + - test_import.yaml + foo: + bar: 1 + baz: 2 + 3: + 4: 5 + """ + with open(file, "w") as f: + f.write(text) + d = calliope.io.read_rich_yaml(file) + + assert "somekey.nested" in d.keys_nested() + assert d.get_key("anotherkey") == 2 + + def test_import_must_be_list(self, tmp_path): + file = tmp_path / "non_list_import.yaml" + text = """ + import: test_import.yaml + foo: + bar: 1 + baz: 2 + 3: + 4: 5 + """ + with open(file, "w") as f: + f.write(text) + + with pytest.raises(ValueError) as excinfo: # noqa: PT011, false positive + calliope.io.read_rich_yaml(file) + assert check_error_or_warning(excinfo, "`import` must be a list.") + + def test_from_yaml_string(self): + yaml_string = """ + # a comment + a: 1 + b: 2 + # a comment about `c` + c: # a comment inline with `c` + x: foo # a comment on foo + + # + y: bar # + z: + I: 1 + II: 2 + d: + """ + d = calliope.io.read_rich_yaml(yaml_string) + assert d.a == 1 + assert d.c.z.II == 2 + + def test_from_yaml_string_dot_strings(self): + yaml_string = "a.b.c: 1\na.b.foo: 2" + d = calliope.io.read_rich_yaml(yaml_string) + assert d.a.b.c == 1 + assert d.a.b.foo == 2 + + def test_from_yaml_string_dot_strings_duplicate(self): + yaml_string = "a.b.c: 1\na.b.c: 2" + with pytest.raises(ruamel_yaml.constructor.DuplicateKeyError): + calliope.io.read_rich_yaml(yaml_string) + + def test_simple_invalid_yaml(self): + yaml_string = "1 this is not valid yaml" + with pytest.raises(ValueError) as excinfo: # noqa: PT011, false positive + calliope.io.read_rich_yaml(yaml_string) + assert check_error_or_warning(excinfo, "Could not parse as YAML") + + def test_parser_error(self): + with pytest.raises(ruamel_yaml.YAMLError): + calliope.io.read_rich_yaml( + """ + foo: bar + baz: 1 + - foobar + bar: baz + + """ + ) + + @pytest.fixture + def multi_order_yaml(self): + return calliope.io.read_rich_yaml( + """ + A.B.C: 10 + A.B: + E: 20 + C: "foobar" + """ + ) + + def test_order_of_subdicts(self, multi_order_yaml): + + assert multi_order_yaml.A.B.C == 10 + assert multi_order_yaml.A.B.E == 20 + assert multi_order_yaml.C == "foobar" + + def test_as_dict_with_sublists(self): + d = calliope.io.read_rich_yaml("a: [{x: 1}, {y: 2}]") + dd = d.as_dict() + assert dd["a"][0]["x"] == 1 + assert all([isinstance(dd["a"][0], dict), not isinstance(dd["a"][0], AttrDict)]) # Not AttrDict! + + + def test_replacement_null_from_file(self, multi_order_yaml): + replacement = calliope.io.read_rich_yaml("C._REPLACE_: null") + multi_order_yaml.union(replacement, allow_override=True, allow_replacement=True) + assert multi_order_yaml.C is None + + @pytest.fixture + def yaml_from_path(self): + this_path = Path(__file__).parent + return calliope.io.read_rich_yaml(this_path / "common" / "yaml_file.yaml") + + def test_from_yaml_path(self, yaml_from_path): + assert yaml_from_path.a == 1 + assert yaml_from_path.c.z.II == 2 + + def test_to_yaml(self, yaml_from_path): + yaml_from_path.set_key("numpy.some_int", np.int32(10)) + yaml_from_path.set_key("numpy.some_float", np.float64(0.5)) + yaml_from_path.a_list = [0, 1, 2] + with tempfile.TemporaryDirectory() as tempdir: + out_file = os.path.join(tempdir, "test.yaml") + yaml_from_path.to_yaml(out_file) + + with open(out_file) as f: + result = f.read() + + assert "some_int: 10" in result + assert "some_float: 0.5" in result + assert "a_list:\n- 0\n- 1\n- 2" in result From 361f4a841dfe228b3a3c2cbad4dc968b61f49a0e Mon Sep 17 00:00:00 2001 From: Ivan Ruiz Manuel <72193617+irm-codebase@users.noreply.github.com> Date: Thu, 28 Nov 2024 18:18:49 +0100 Subject: [PATCH 02/13] Add YAML templating w/tests (pre AttrDict YAML removal). --- src/calliope/__init__.py | 2 +- src/calliope/attrdict.py | 3 + src/calliope/io.py | 81 +++++++++++++++++++++----- src/calliope/model.py | 5 -- tests/test_io.py | 121 ++++++++++++++++++++++++++++++++++++--- 5 files changed, 184 insertions(+), 28 deletions(-) diff --git a/src/calliope/__init__.py b/src/calliope/__init__.py index 22d02097..794ebc82 100644 --- a/src/calliope/__init__.py +++ b/src/calliope/__init__.py @@ -3,7 +3,7 @@ from calliope import examples, exceptions from calliope._version import __version__ from calliope.attrdict import AttrDict -from calliope.model import Model, read_netcdf, read_yaml +from calliope.model import Model, read_netcdf from calliope.util.logging import set_log_verbosity __title__ = "Calliope" diff --git a/src/calliope/attrdict.py b/src/calliope/attrdict.py index bd94df7b..2cb4dd0f 100644 --- a/src/calliope/attrdict.py +++ b/src/calliope/attrdict.py @@ -410,6 +410,9 @@ def union( Raises: KeyError: `other` has an already defined key and `allow_override == False` """ + if not isinstance(other, AttrDict): + # FIXME-yaml: remove AttrDict wrapping in uses of this function. + other = AttrDict(other) self_keys = self.keys_nested() other_keys = other.keys_nested() if allow_replacement: diff --git a/src/calliope/io.py b/src/calliope/io.py index c466c901..10b1b09a 100644 --- a/src/calliope/io.py +++ b/src/calliope/io.py @@ -221,7 +221,6 @@ def load_config(filename: str): def read_rich_yaml( yaml: str | Path, allow_override: bool = False, - template_sections: None | Iterable[str] = None, ) -> AttrDict: """Returns an AttrDict initialised from the given YAML file or string. @@ -247,8 +246,6 @@ def read_rich_yaml( yaml_dict = _resolve_yaml_imports( yaml_dict, base_path=yaml_path, allow_override=allow_override ) - if template_sections: - yaml_dict = _resolve_yaml_templates(yaml_dict, template_sections) return yaml_dict @@ -298,14 +295,70 @@ def _resolve_yaml_imports( return loaded_dict -def _resolve_yaml_templates(data: AttrDict, sections: Iterable[str]) -> AttrDict: - """Fill and then remove template definitions in the given sections.""" - if "templates" in data: - templates = data.pop("templates") - for section in sections: - for item, values in data[section].items(): - if "template" in values: - filled, _ = climb_template_tree(values, templates) - data[section][item] = filled - data[section][item].pop("template") - return data +class TemplateSolver: + """Resolves templates in dictionaries obtained from YAML files.""" + + TEMPLATES_SECTION: str = "templates" + TEMPLATE_CALL: str = "template" + + def __init__(self, data: AttrDict): + """Initialise the solver.""" + self._raw_templates: AttrDict = data.get_key(self.TEMPLATES_SECTION, None) + self._raw_data: AttrDict = data + self.resolved_templates: AttrDict + self.resolved_data: AttrDict + self.resolve() + + def resolve(self): + """Fill in template references and remove template definitions and calls.""" + self.resolved_templates = AttrDict() + for key, value in self._raw_templates.items(): + if not isinstance(value, dict): + raise ValueError("Template definitions must be YAML blocks.") + self.resolved_templates[key] = self._resolve_template(key) + self.resolved_data = self._resolve_data(self._raw_data) + + def _resolve_template(self, name: str, stack: None | set[str] = None) -> AttrDict: + """Resolves templates recursively. + + Catches circular template definitions. + """ + if stack is None: + stack = set() + elif name in stack: + raise ValueError(f"Circular template reference detected for '{name}'.") + stack.add(name) + + result = AttrDict() + raw_data = self._raw_templates[name] + if self.TEMPLATE_CALL in raw_data: + # Current template takes precedence when overriding values + inherited_name = raw_data[self.TEMPLATE_CALL] + if inherited_name in self.resolved_templates: + inherited_data = self.resolved_templates[inherited_name] + else: + inherited_data = self._resolve_template(inherited_name, stack) + result.union(inherited_data) + + local_data = {k: raw_data[k] for k in raw_data.keys() - {self.TEMPLATE_CALL}} + result.union(local_data, allow_override=True) + + stack.remove(name) + return result + + def _resolve_data(self, section, level: int = 0): + if isinstance(section, dict): + result = AttrDict() + if self.TEMPLATES_SECTION in section: + if level != 0: + raise ValueError("Template definitions must be placed at the top level of the YAML file.") + if self.TEMPLATE_CALL in section: + # Prefill template first so it can be overwritten by local values. + result.update(self.resolved_templates[section[self.TEMPLATE_CALL]]) + keys = section.keys() - {self.TEMPLATE_CALL, self.TEMPLATES_SECTION} + for key in keys: + result[key] = self._resolve_data(section[key], level=level+1) + else: + result = section + return result + diff --git a/src/calliope/model.py b/src/calliope/model.py index 65cd913d..ee8c5a77 100644 --- a/src/calliope/model.py +++ b/src/calliope/model.py @@ -39,11 +39,6 @@ def read_netcdf(path): return Model(model_definition=model_data) -def read_yaml(path: str | Path): - """Return a dictionary constructed using Calliope-flavoured YAML.""" - return io.read_rich_yaml(path, template_sections={"techs", "nodes", "data_tables"}) - - class Model: """A Calliope Model.""" diff --git a/tests/test_io.py b/tests/test_io.py index ad79b423..5f993e6e 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -240,7 +240,6 @@ def test_save_per_spore(self): class TestYaml: - @pytest.fixture def dummy_yaml_import(self): return """ @@ -254,8 +253,7 @@ def test_do_not_resolve_imports(self, dummy_yaml_import): calliope.io.read_rich_yaml(dummy_yaml_import) assert check_error_or_warning( - exinfo, - "Imports are not possible for non-file yaml inputs." + exinfo, "Imports are not possible for non-file yaml inputs." ) @pytest.fixture @@ -270,7 +268,7 @@ def dummy_imported_file(self, tmp_path) -> Path: return file def test_import(self, dummy_imported_file): - file = dummy_imported_file.parent / "main_file.yaml" + file = dummy_imported_file.parent / "main_file.yaml" text = """ import: - test_import.yaml @@ -288,7 +286,7 @@ def test_import(self, dummy_imported_file): assert d.get_key("anotherkey") == 2 def test_import_must_be_list(self, tmp_path): - file = tmp_path / "non_list_import.yaml" + file = tmp_path / "non_list_import.yaml" text = """ import: test_import.yaml foo: @@ -365,7 +363,6 @@ def multi_order_yaml(self): ) def test_order_of_subdicts(self, multi_order_yaml): - assert multi_order_yaml.A.B.C == 10 assert multi_order_yaml.A.B.E == 20 assert multi_order_yaml.C == "foobar" @@ -374,8 +371,9 @@ def test_as_dict_with_sublists(self): d = calliope.io.read_rich_yaml("a: [{x: 1}, {y: 2}]") dd = d.as_dict() assert dd["a"][0]["x"] == 1 - assert all([isinstance(dd["a"][0], dict), not isinstance(dd["a"][0], AttrDict)]) # Not AttrDict! - + assert all( + [isinstance(dd["a"][0], dict), not isinstance(dd["a"][0], AttrDict)] + ) # Not AttrDict! def test_replacement_null_from_file(self, multi_order_yaml): replacement = calliope.io.read_rich_yaml("C._REPLACE_: null") @@ -405,3 +403,110 @@ def test_to_yaml(self, yaml_from_path): assert "some_int: 10" in result assert "some_float: 0.5" in result assert "a_list:\n- 0\n- 1\n- 2" in result + + +class TestYAMLTemplates: + + @pytest.fixture + def dummy_solved_template(self) -> calliope.io.TemplateSolver: + text = """ + templates: + T1: + A: ["foo", "bar"] + B: 1 + T2: + C: bar + template: T1 + T3: + template: T1 + B: 11 + T4: + template: T3 + A: ["bar", "foobar"] + B: "1" + C: {"foo": "bar"} + D: true + a: + template: T1 + a1: 1 + b: + template: T3 + c: + template: T4 + D: false + """ + yaml_data = calliope.io.read_rich_yaml(text) + return calliope.io.TemplateSolver(yaml_data) + + def test_inheritance_templates(self, dummy_solved_template): + templates = dummy_solved_template.resolved_templates + assert all( + [ + templates.T1 == {"A": ["foo", "bar"], "B": 1}, + templates.T2 == {"A": ["foo", "bar"], "B": 1, "C": "bar"}, + templates.T3 == {"A": ["foo", "bar"], "B": 11}, + templates.T4 == {"A": ["bar", "foobar"], "B": "1", "C": {"foo": "bar"}, "D": True} + ] + ) + + def test_template_inheritance_data(self, dummy_solved_template): + data = dummy_solved_template.resolved_data + assert all( + [ + data.a == {"A": ["foo", "bar"], "B": 1, "a1": 1}, + data.b == {"A": ["foo", "bar"], "B": 11}, + data.c == {"A": ["bar", "foobar"], "B": "1", "C": {"foo": "bar"}, "D": False} + ] + ) + + def test_invalid_template_error(self): + text = calliope.io.read_rich_yaml( + """ + templates: + T1: "not_a_yaml_block" + T2: + foo: bar + a: + template: T2 + """ + ) + with pytest.raises(ValueError, match="Template definitions must be YAML blocks."): + calliope.io.TemplateSolver(text) + + def test_circular_template_error(self): + text = calliope.io.read_rich_yaml( + """ + templates: + T1: + template: T2 + bar: foo + T2: + template: T1 + foo: bar + a: + template: T2 + """ + ) + with pytest.raises(ValueError, match="Circular template reference detected"): + calliope.io.TemplateSolver(text) + + def test_incorrect_template_placement_error(self): + text = calliope.io.read_rich_yaml( + """ + templates: + T1: + stuff: null + T2: + foo: bar + a: + template: T2 + b: + templates: + T3: + this: "should not be here" + """ + ) + with pytest.raises(ValueError, match="Template definitions must be placed at the top level of the YAML file."): + calliope.io.TemplateSolver(text) + + From 14839de779c87fb9bdef538363ea228e2697fce7 Mon Sep 17 00:00:00 2001 From: Ivan Ruiz Manuel <72193617+irm-codebase@users.noreply.github.com> Date: Fri, 29 Nov 2024 10:09:10 +0100 Subject: [PATCH 03/13] Simplify model initialisation to two main cases (pre-removal AttrDict YAML / inheritance functions --- .../examples/annual_energy_balance.yaml | 1 - src/calliope/config/protected_parameters.yaml | 2 +- src/calliope/io.py | 10 +++- src/calliope/model.py | 51 ++++++++++--------- 4 files changed, 37 insertions(+), 27 deletions(-) diff --git a/docs/user_defined_math/examples/annual_energy_balance.yaml b/docs/user_defined_math/examples/annual_energy_balance.yaml index 783ddbdb..85b5358e 100644 --- a/docs/user_defined_math/examples/annual_energy_balance.yaml +++ b/docs/user_defined_math/examples/annual_energy_balance.yaml @@ -15,7 +15,6 @@ # # Helper functions used: # -# - `inheritance` (where) # - `sum` (expression) # # --- diff --git a/src/calliope/config/protected_parameters.yaml b/src/calliope/config/protected_parameters.yaml index 6d0efbd3..2b055ea6 100644 --- a/src/calliope/config/protected_parameters.yaml +++ b/src/calliope/config/protected_parameters.yaml @@ -6,4 +6,4 @@ definition_matrix: >- `definition_matrix` is a protected array. It will be generated internally based on the values you assign to the `carrier_in` and `carrier_out` parameters. template: >- - Technology/Node template inheritance (`template`) can only be used in the YAML model definition. + Template inheritance (`template`) can only be used in the YAML model definition. diff --git a/src/calliope/io.py b/src/calliope/io.py index 10b1b09a..af25ec90 100644 --- a/src/calliope/io.py +++ b/src/calliope/io.py @@ -307,9 +307,9 @@ def __init__(self, data: AttrDict): self._raw_data: AttrDict = data self.resolved_templates: AttrDict self.resolved_data: AttrDict - self.resolve() + self._resolve() - def resolve(self): + def _resolve(self): """Fill in template references and remove template definitions and calls.""" self.resolved_templates = AttrDict() for key, value in self._raw_templates.items(): @@ -362,3 +362,9 @@ def _resolve_data(self, section, level: int = 0): result = section return result + @classmethod + def resolve_templates(cls, data: AttrDict) -> AttrDict: + """Resolve calliope-flavoured templates.""" + solver = cls(data) + return solver.resolved_data + diff --git a/src/calliope/model.py b/src/calliope/model.py index ee8c5a77..1b6ff431 100644 --- a/src/calliope/model.py +++ b/src/calliope/model.py @@ -90,18 +90,8 @@ def __init__( if isinstance(model_definition, xr.Dataset): self._init_from_model_data(model_definition) else: - if isinstance(model_definition, dict): - model_def_dict = AttrDict(model_definition) - else: - self._def_path = str(model_definition) - model_def_dict = AttrDict.from_yaml(model_definition) - - (model_def, applied_overrides) = preprocess.load_scenario_overrides( - model_def_dict, scenario, override_dict, **kwargs - ) - - self._init_from_model_def_dict( - model_def, applied_overrides, scenario, data_table_dfs + self._init_from_model_definition( + model_definition, scenario, override_dict, data_table_dfs, **kwargs ) self._model_data.attrs["timestamp_model_creation"] = timestamp_model_creation @@ -138,23 +128,38 @@ def is_solved(self): """Get solved status.""" return self._is_solved - def _init_from_model_def_dict( + def _init_from_model_definition( self, - model_definition: calliope.AttrDict, - applied_overrides: str, + model_definition: dict | str, scenario: str | None, - data_table_dfs: dict[str, pd.DataFrame] | None = None, + override_dict: dict | None, + data_table_dfs: dict[str, pd.DataFrame] | None, + **kwargs ) -> None: """Initialise the model using pre-processed YAML files and optional dataframes/dicts. Args: model_definition (calliope.AttrDict): preprocessed model configuration. - applied_overrides (str): overrides specified by users scenario (str | None): scenario specified by users - data_table_dfs (dict[str, pd.DataFrame] | None, optional): files with additional model information. Defaults to None. + override_dict (dict | None): overrides to apply after scenarios. + data_table_dfs (dict[str, pd.DataFrame] | None): files with additional model information. + **kwargs: initialisation overrides. """ + if isinstance(model_definition, dict): + model_def_raw = AttrDict(model_definition) + else: + self._def_path = str(model_definition) + model_def_raw = io.read_rich_yaml(model_definition) + + (model_def_full, applied_overrides) = preprocess.load_scenario_overrides( + model_def_raw, scenario, override_dict, **kwargs + ) + + # FIXME-yaml: reintroduce after cleaning inheritance + # model_def_full = io.TemplateSolver.resolve_templates(model_def_overridden) + # First pass to check top-level keys are all good - validate_dict(model_definition, CONFIG_SCHEMA, "Model definition") + validate_dict(model_def_full, CONFIG_SCHEMA, "Model definition") log_time( LOGGER, @@ -163,7 +168,7 @@ def _init_from_model_def_dict( comment="Model: preprocessing stage 1 (model_run)", ) model_config = AttrDict(extract_from_schema(CONFIG_SCHEMA, "default")) - model_config.union(model_definition.pop("config"), allow_override=True) + model_config.union(model_def_full.pop("config"), allow_override=True) init_config = update_then_validate_config("init", model_config) @@ -180,9 +185,9 @@ def _init_from_model_def_dict( "scenario": scenario, "defaults": param_metadata["default"], } - templates = model_definition.get("templates", AttrDict()) + templates = model_def_full.get("templates", AttrDict()) data_tables: list[DataTable] = [] - for table_name, table_dict in model_definition.pop("data_tables", {}).items(): + for table_name, table_dict in model_def_full.pop("data_tables", {}).items(): table_dict, _ = climb_template_tree(table_dict, templates, table_name) data_tables.append( DataTable( @@ -191,7 +196,7 @@ def _init_from_model_def_dict( ) model_data_factory = ModelDataFactory( - init_config, model_definition, data_tables, attributes, param_metadata + init_config, model_def_full, data_tables, attributes, param_metadata ) model_data_factory.build() From 003f122de0d135f803ef21b7ee39100f6f005d8a Mon Sep 17 00:00:00 2001 From: Ivan Ruiz Manuel <72193617+irm-codebase@users.noreply.github.com> Date: Fri, 29 Nov 2024 19:24:32 +0100 Subject: [PATCH 04/13] Replace AttrDict and remove redundant 'inheritance' helper functionality --- docs/examples/loading_tabular_data.py | 11 +- docs/examples/piecewise_constraints.py | 5 +- docs/hooks/generate_math_docs.py | 3 +- src/calliope/attrdict.py | 115 ----------------- src/calliope/backend/helper_functions.py | 98 -------------- src/calliope/config/config_schema.yaml | 14 +- src/calliope/config/protected_parameters.yaml | 4 +- src/calliope/io.py | 82 +----------- src/calliope/model.py | 24 ++-- src/calliope/preprocess/__init__.py | 2 +- src/calliope/preprocess/data_tables.py | 3 +- src/calliope/preprocess/model_data.py | 29 ++--- src/calliope/preprocess/model_math.py | 3 +- src/calliope/preprocess/scenarios.py | 122 ++++++++++++++++-- src/calliope/util/generate_runs.py | 6 +- src/calliope/util/tools.py | 52 -------- tests/test_backend_helper_functions.py | 31 +---- tests/test_backend_where_parser.py | 9 +- tests/test_cli.py | 4 +- tests/test_core_preprocess.py | 23 ++-- tests/test_core_util.py | 87 +------------ tests/test_example_models.py | 3 +- tests/test_io.py | 107 --------------- tests/test_math.py | 7 +- tests/test_preprocess_model_data.py | 57 ++------ tests/test_preprocess_model_definition.py | 110 ++++++++++++++++ tests/test_preprocess_model_math.py | 3 +- tests/test_preprocess_time.py | 13 +- 28 files changed, 316 insertions(+), 711 deletions(-) create mode 100644 tests/test_preprocess_model_definition.py diff --git a/docs/examples/loading_tabular_data.py b/docs/examples/loading_tabular_data.py index 35fe8398..dc41a7ea 100644 --- a/docs/examples/loading_tabular_data.py +++ b/docs/examples/loading_tabular_data.py @@ -25,6 +25,7 @@ import pandas as pd import calliope +from calliope.io import read_rich_yaml calliope.set_log_verbosity("INFO", include_solver_output=False) @@ -92,7 +93,7 @@ # When this is used to initialise a Calliope model, it is processed into a set of data tables ([xarray.DataArray](https://docs.xarray.dev/en/stable/generated/xarray.DataArray.html)) internally: # %% -model_def = calliope.AttrDict.from_yaml_string( +model_def = read_rich_yaml( """ techs: supply_tech: @@ -324,7 +325,7 @@ # ``` # %% -model_def = calliope.AttrDict.from_yaml_string( +model_def = read_rich_yaml( """ data_tables: tech_data: @@ -362,7 +363,7 @@ # You can do that by setting `data` as the name of a key in a dictionary that you supply when you load the model: # %% -model_def = calliope.AttrDict.from_yaml_string( +model_def = read_rich_yaml( """ data_tables: tech_data: @@ -590,7 +591,7 @@ # # %% -model_def = calliope.AttrDict.from_yaml_string( +model_def = read_rich_yaml( """ data_tables: tech_data: @@ -676,7 +677,7 @@ # ``` # %% -model_def = calliope.AttrDict.from_yaml_string( +model_def = read_rich_yaml( """ data_tables: tech_data: diff --git a/docs/examples/piecewise_constraints.py b/docs/examples/piecewise_constraints.py index 064be53d..f050ecce 100644 --- a/docs/examples/piecewise_constraints.py +++ b/docs/examples/piecewise_constraints.py @@ -25,6 +25,7 @@ import plotly.express as px import calliope +from calliope.io import read_rich_yaml calliope.set_log_verbosity("INFO", include_solver_output=False) @@ -80,7 +81,7 @@ dims: "breakpoints" """ print(new_params) -new_params_as_dict = calliope.AttrDict.from_yaml_string(new_params) +new_params_as_dict = read_rich_yaml(new_params) m = calliope.examples.national_scale(override_dict=new_params_as_dict) # %% @@ -133,7 +134,7 @@ # With our piecewise constraint defined, we can build our optimisation problem and inject this new math. # %% -new_math_as_dict = calliope.AttrDict.from_yaml_string(new_math) +new_math_as_dict = read_rich_yaml(new_math) m.build(add_math_dict=new_math_as_dict) # %% [markdown] diff --git a/docs/hooks/generate_math_docs.py b/docs/hooks/generate_math_docs.py index b513ebd2..e7b2370a 100644 --- a/docs/hooks/generate_math_docs.py +++ b/docs/hooks/generate_math_docs.py @@ -11,6 +11,7 @@ from mkdocs.structure.files import File import calliope +from calliope.io import read_rich_yaml from calliope.postprocess.math_documentation import MathDocumentation logger = logging.getLogger("mkdocs") @@ -42,7 +43,7 @@ def on_files(files: list, config: dict, **kwargs): """Process documentation for pre-defined calliope math files.""" - model_config = calliope.AttrDict.from_yaml(MODEL_PATH) + model_config = read_rich_yaml(MODEL_PATH) base_documentation = generate_base_math_documentation() write_file( diff --git a/src/calliope/attrdict.py b/src/calliope/attrdict.py index 2cb4dd0f..c056c3b4 100644 --- a/src/calliope/attrdict.py +++ b/src/calliope/attrdict.py @@ -107,121 +107,6 @@ def init_from_dict(self, d): else: self.set_key(k, v) - @classmethod - def _resolve_imports( - cls, - loaded: Self, - resolve_imports: bool | str, - base_path: str | Path | None = None, - allow_override: bool = False, - ) -> Self: - if ( - isinstance(resolve_imports, bool) - and resolve_imports is True - and "import" in loaded - ): - loaded_dict = loaded - elif ( - isinstance(resolve_imports, str) - and resolve_imports + ".import" in loaded.keys_nested() - ): - loaded_dict = loaded.get_key(resolve_imports) - else: # Return right away if no importing to be done - return loaded - - # If we end up here, we have something to import - imports = loaded_dict.get_key("import") - if not isinstance(imports, list): - raise ValueError("`import` must be a list.") - - for k in imports: - path = relative_path(base_path, k) - imported = cls.from_yaml(path) - # loaded is added to imported (i.e. it takes precedence) - imported.union(loaded_dict, allow_override=allow_override) - loaded_dict = imported - # 'import' key itself is no longer needed - loaded_dict.del_key("import") - - if isinstance(resolve_imports, str): - loaded.set_key(resolve_imports, loaded_dict) - else: - loaded = loaded_dict - - return loaded - - @classmethod - def from_yaml( - cls, - filename: str | Path, - resolve_imports: bool | str = True, - allow_override: bool = False, - ) -> Self: - """Returns an AttrDict initialized from the given path or file path. - - If `resolve_imports` is True, top-level `import:` statements - are resolved recursively. - If `resolve_imports` is False, top-level `import:` statements - are treated like any other key and not further processed. - If `resolve_imports` is a string, such as `foobar`, import - statements underneath that key are resolved, i.e. `foobar.import:`. - When resolving import statements, anything defined locally - overrides definitions in the imported file. - - Args: - filename (str | Path): YAML file. - resolve_imports (bool | str, optional): top-level `import:` solving option. - Defaults to True. - allow_override (bool, optional): whether or not to allow overrides of already defined keys. - Defaults to False. - - Returns: - Self: constructed AttrDict - """ - filename = Path(filename) - loaded = cls(_yaml_load(filename.read_text(encoding="utf-8"))) - loaded = cls._resolve_imports( - loaded, resolve_imports, filename, allow_override=allow_override - ) - return loaded - - @classmethod - def from_yaml_string( - cls, - string: str, - resolve_imports: bool | str = True, - allow_override: bool = False, - ) -> Self: - """Returns an AttrDict initialized from the given string. - - Input string must be valid YAML. - - If `resolve_imports` is True, top-level `import:` statements - are resolved recursively. - If `resolve_imports` is False, top-level `import:` statements - are treated like any other key and not further processed. - If `resolve_imports` is a string, such as `foobar`, import - statements underneath that key are resolved, i.e. `foobar.import:`. - When resolving import statements, anything defined locally - overrides definitions in the imported file. - - Args: - string (str): Valid YAML string. - resolve_imports (bool | str, optional): top-level `import:` solving option. - Defaults to True. - allow_override (bool, optional): whether or not to allow overrides of already defined keys. - Defaults to False. - - Returns: - calliope.AttrDict: - - """ - loaded = cls(_yaml_load(string)) - loaded = cls._resolve_imports( - loaded, resolve_imports, allow_override=allow_override - ) - return loaded - def set_key(self, key, value): """Set the given ``key`` to the given ``value``. diff --git a/src/calliope/backend/helper_functions.py b/src/calliope/backend/helper_functions.py index f8cef607..7159ee00 100644 --- a/src/calliope/backend/helper_functions.py +++ b/src/calliope/backend/helper_functions.py @@ -162,104 +162,6 @@ def _listify(self, vals: list[str] | str) -> list[str]: return vals -class Inheritance(ParsingHelperFunction): - """Find all nodes / techs that inherit from a template.""" - - #: - ALLOWED_IN = ["where"] - #: - NAME = "inheritance" - - def as_math_string( # noqa: D102, override - self, nodes: str | None = None, techs: str | None = None - ) -> str: - strings = [] - if nodes is not None: - strings.append(f"nodes={nodes}") - if techs is not None: - strings.append(f"techs={techs}") - return rf"\text{{inherits({','.join(strings)})}}" - - def as_array( - self, *, nodes: str | None = None, techs: str | None = None - ) -> xr.DataArray: - """Find all technologies and/or nodes which inherit from a particular template. - - The group items being referenced must be defined by the user in `templates`. - - Args: - nodes (str | None, optional): group name to search for inheritance of on the `nodes` dimension. Default is None. - techs (str | None, optional): group name to search for inheritance of on the `techs` dimension. Default is None. - - Returns: - xr.Dataset: Boolean array where values are True where the group is inherited, False otherwise. Array dimensions will equal the number of non-None inputs. - - Examples: - With: - ```yaml - templates: - foo: - available_area: 1 - bar: - flow_cap_max: 1 - baz: - template: bar - flow_out_eff: 0.5 - nodes: - node_1: - template: foo - techs: {tech_1, tech_2} - node_2: - techs: {tech_1, tech_2} - techs: - tech_1: - ... - template: bar - tech_2: - ... - template: baz - ``` - - >>> inheritance(nodes=foo) - - array([True, False]) - Coordinates: - * nodes (nodes) >> inheritance(techs=bar) # tech_2 inherits `bar` via `baz`. - - array([True, True]) - Coordinates: - * techs (techs) >> inheritance(techs=baz) - - array([False, True]) - Coordinates: - * techs (techs) >> inheritance(nodes=foo, techs=baz) - - array([[False, False], - [True, False]]) - Coordinates: - * nodes (nodes) - - Abstract technology/node templates from which techs/nodes can `inherit`. - See the model definition schema for more guidance on content. - additionalProperties: false - patternProperties: *nested_pattern + # templates: + # type: [object, "null"] + # description: >- + # Abstract technology/node templates from which techs/nodes can `inherit`. + # See the model definition schema for more guidance on content. + # additionalProperties: false + # patternProperties: *nested_pattern overrides: type: [object, "null"] diff --git a/src/calliope/config/protected_parameters.yaml b/src/calliope/config/protected_parameters.yaml index 2b055ea6..53bba311 100644 --- a/src/calliope/config/protected_parameters.yaml +++ b/src/calliope/config/protected_parameters.yaml @@ -6,4 +6,6 @@ definition_matrix: >- `definition_matrix` is a protected array. It will be generated internally based on the values you assign to the `carrier_in` and `carrier_out` parameters. template: >- - Template inheritance (`template`) can only be used in the YAML model definition. + Template calls (`template`) can only be used in the YAML model definition. +templates: >- + Template definitions (`templates`) can only be used in the YAML model definition. diff --git a/src/calliope/io.py b/src/calliope/io.py index af25ec90..93ec19a0 100644 --- a/src/calliope/io.py +++ b/src/calliope/io.py @@ -5,7 +5,6 @@ import importlib.resources import logging import os -from collections.abc import Iterable from copy import deepcopy from pathlib import Path @@ -19,7 +18,7 @@ from calliope import exceptions from calliope.attrdict import AttrDict -from calliope.util.tools import climb_template_tree, listify, relative_path +from calliope.util.tools import listify, relative_path logger = logging.getLogger(__name__) @@ -120,7 +119,7 @@ def _deserialise(attrs: dict) -> None: Changes will be made in-place, so be sure to supply a copy of your dictionary if you want access to its original state. """ for attr in _pop_serialised_list(attrs, "serialised_dicts"): - attrs[attr] = AttrDict.from_yaml_string(attrs[attr]) + attrs[attr] = read_rich_yaml(attrs[attr]) for attr in _pop_serialised_list(attrs, "serialised_bools"): attrs[attr] = bool(attrs[attr]) for attr in _pop_serialised_list(attrs, "serialised_nones"): @@ -228,9 +227,7 @@ def read_rich_yaml( Args: yaml (str | Path): YAML file path or string. - resolve_imports (bool, optional): Solve imports recursively. Defaults to True. allow_override (bool, optional): Allow overrides for already defined keys. Defaults to False. - template_sections: (Iterable[str], optional): Replace tempalte for the requested sections. Defaults to False. Raises: ValueError: Import solving requested for non-file input YAML. @@ -293,78 +290,3 @@ def _resolve_yaml_imports( loaded_dict.del_key("import") return loaded_dict - - -class TemplateSolver: - """Resolves templates in dictionaries obtained from YAML files.""" - - TEMPLATES_SECTION: str = "templates" - TEMPLATE_CALL: str = "template" - - def __init__(self, data: AttrDict): - """Initialise the solver.""" - self._raw_templates: AttrDict = data.get_key(self.TEMPLATES_SECTION, None) - self._raw_data: AttrDict = data - self.resolved_templates: AttrDict - self.resolved_data: AttrDict - self._resolve() - - def _resolve(self): - """Fill in template references and remove template definitions and calls.""" - self.resolved_templates = AttrDict() - for key, value in self._raw_templates.items(): - if not isinstance(value, dict): - raise ValueError("Template definitions must be YAML blocks.") - self.resolved_templates[key] = self._resolve_template(key) - self.resolved_data = self._resolve_data(self._raw_data) - - def _resolve_template(self, name: str, stack: None | set[str] = None) -> AttrDict: - """Resolves templates recursively. - - Catches circular template definitions. - """ - if stack is None: - stack = set() - elif name in stack: - raise ValueError(f"Circular template reference detected for '{name}'.") - stack.add(name) - - result = AttrDict() - raw_data = self._raw_templates[name] - if self.TEMPLATE_CALL in raw_data: - # Current template takes precedence when overriding values - inherited_name = raw_data[self.TEMPLATE_CALL] - if inherited_name in self.resolved_templates: - inherited_data = self.resolved_templates[inherited_name] - else: - inherited_data = self._resolve_template(inherited_name, stack) - result.union(inherited_data) - - local_data = {k: raw_data[k] for k in raw_data.keys() - {self.TEMPLATE_CALL}} - result.union(local_data, allow_override=True) - - stack.remove(name) - return result - - def _resolve_data(self, section, level: int = 0): - if isinstance(section, dict): - result = AttrDict() - if self.TEMPLATES_SECTION in section: - if level != 0: - raise ValueError("Template definitions must be placed at the top level of the YAML file.") - if self.TEMPLATE_CALL in section: - # Prefill template first so it can be overwritten by local values. - result.update(self.resolved_templates[section[self.TEMPLATE_CALL]]) - keys = section.keys() - {self.TEMPLATE_CALL, self.TEMPLATES_SECTION} - for key in keys: - result[key] = self._resolve_data(section[key], level=level+1) - else: - result = section - return result - - @classmethod - def resolve_templates(cls, data: AttrDict) -> AttrDict: - """Resolve calliope-flavoured templates.""" - solver = cls(data) - return solver.resolved_data - diff --git a/src/calliope/model.py b/src/calliope/model.py index 1b6ff431..006c6a90 100644 --- a/src/calliope/model.py +++ b/src/calliope/model.py @@ -25,7 +25,7 @@ update_then_validate_config, validate_dict, ) -from calliope.util.tools import climb_template_tree, relative_path +from calliope.util.tools import relative_path if TYPE_CHECKING: from calliope.backend.backend_model import BackendModel @@ -90,6 +90,9 @@ def __init__( if isinstance(model_definition, xr.Dataset): self._init_from_model_data(model_definition) else: + if not isinstance(model_definition, dict): + # Only file definitions allow relative files. + self._def_path = str(model_definition) self._init_from_model_definition( model_definition, scenario, override_dict, data_table_dfs, **kwargs ) @@ -134,7 +137,7 @@ def _init_from_model_definition( scenario: str | None, override_dict: dict | None, data_table_dfs: dict[str, pd.DataFrame] | None, - **kwargs + **kwargs, ) -> None: """Initialise the model using pre-processed YAML files and optional dataframes/dicts. @@ -145,19 +148,10 @@ def _init_from_model_definition( data_table_dfs (dict[str, pd.DataFrame] | None): files with additional model information. **kwargs: initialisation overrides. """ - if isinstance(model_definition, dict): - model_def_raw = AttrDict(model_definition) - else: - self._def_path = str(model_definition) - model_def_raw = io.read_rich_yaml(model_definition) - - (model_def_full, applied_overrides) = preprocess.load_scenario_overrides( - model_def_raw, scenario, override_dict, **kwargs + (model_def_full, applied_overrides) = preprocess.prepare_model_definition( + model_definition, scenario, override_dict ) - - # FIXME-yaml: reintroduce after cleaning inheritance - # model_def_full = io.TemplateSolver.resolve_templates(model_def_overridden) - + model_def_full.union(AttrDict({"config.init": kwargs}), allow_override=True) # First pass to check top-level keys are all good validate_dict(model_def_full, CONFIG_SCHEMA, "Model definition") @@ -185,10 +179,8 @@ def _init_from_model_definition( "scenario": scenario, "defaults": param_metadata["default"], } - templates = model_def_full.get("templates", AttrDict()) data_tables: list[DataTable] = [] for table_name, table_dict in model_def_full.pop("data_tables", {}).items(): - table_dict, _ = climb_template_tree(table_dict, templates, table_name) data_tables.append( DataTable( init_config, table_name, table_dict, data_table_dfs, self._def_path diff --git a/src/calliope/preprocess/__init__.py b/src/calliope/preprocess/__init__.py index 2b9584be..f1998f20 100644 --- a/src/calliope/preprocess/__init__.py +++ b/src/calliope/preprocess/__init__.py @@ -3,4 +3,4 @@ from calliope.preprocess.data_tables import DataTable from calliope.preprocess.model_data import ModelDataFactory from calliope.preprocess.model_math import CalliopeMath -from calliope.preprocess.scenarios import load_scenario_overrides +from calliope.preprocess.scenarios import prepare_model_definition diff --git a/src/calliope/preprocess/data_tables.py b/src/calliope/preprocess/data_tables.py index 4a90fbf3..1ba59e63 100644 --- a/src/calliope/preprocess/data_tables.py +++ b/src/calliope/preprocess/data_tables.py @@ -40,7 +40,6 @@ class DataTableDict(TypedDict): add_dims: NotRequired[dict[str, str | list[str]]] select: NotRequired[dict[str, str | bool | int]] drop: NotRequired[Hashable | list[Hashable]] - template: NotRequired[str] class DataTable: @@ -131,7 +130,7 @@ def node_dict(self, techs_incl_inheritance: AttrDict) -> AttrDict: Args: techs_incl_inheritance (AttrDict): Technology definition dictionary which is a union of any YAML definition and the result of calling `self.tech_dict` across all data tables. - Technologies should have their entire definition inheritance chain resolved. + Technologies should have their definition inheritance resolved. """ node_tech_vars = self.dataset[ [ diff --git a/src/calliope/preprocess/model_data.py b/src/calliope/preprocess/model_data.py index 2fa0c3fe..fa14f120 100644 --- a/src/calliope/preprocess/model_data.py +++ b/src/calliope/preprocess/model_data.py @@ -17,7 +17,7 @@ from calliope.attrdict import AttrDict from calliope.preprocess import data_tables, time from calliope.util.schema import MODEL_SCHEMA, validate_dict -from calliope.util.tools import climb_template_tree, listify +from calliope.util.tools import listify LOGGER = logging.getLogger(__name__) @@ -148,8 +148,7 @@ def init_from_data_tables(self, data_tables: list[data_tables.DataTable]): def add_node_tech_data(self): """For each node, extract technology definitions and node-level parameters and convert them to arrays. - The node definition will first be updated according to any defined inheritance (via `template`), - before processing each defined tech (which will also be updated according to its inheritance tree). + The node definition will be updated with each defined tech (which will also be updated according to its inheritance tree). Node and tech definitions will be validated against the model definition schema here. """ @@ -516,7 +515,7 @@ def _inherit_defs( ) -> AttrDict: """For a set of node/tech definitions, climb the inheritance tree to build a final definition dictionary. - For `techs` at `nodes`, the first step is to inherit the technology definition from `techs`, _then_ to climb `template` references. + For `techs` at `nodes`, they inherit the technology definition from `techs`. Base definitions will take precedence over inherited ones and more recent inherited definitions will take precedence over older ones. @@ -540,11 +539,11 @@ def _inherit_defs( AttrDict: Dictionary containing all active tech/node definitions with inherited parameters. """ if connected_dims: - err_message_prefix = ( + debug_message_prefix = ( ", ".join([f"({k}, {v})" for k, v in connected_dims.items()]) + ", " ) else: - err_message_prefix = "" + debug_message_prefix = "" updated_defs = AttrDict() if dim_dict is None: @@ -557,7 +556,7 @@ def _inherit_defs( base_def = self.model_definition["techs"] if item_name not in base_def: raise KeyError( - f"{err_message_prefix}({dim_name}, {item_name}) | Reference to item not defined in base {dim_name}" + f"{debug_message_prefix}({dim_name}, {item_name}) | Reference to item not defined in base {dim_name}" ) item_base_def = deepcopy(base_def[item_name]) @@ -568,23 +567,15 @@ def _inherit_defs( item_base_def = _data_table_dict else: item_base_def = item_def - templates = self.model_definition.get("templates", AttrDict()) - updated_item_def, inheritance = climb_template_tree( - item_base_def, templates, item_name - ) - if not updated_item_def.get("active", True): + if not item_base_def.get("active", True): LOGGER.debug( - f"{err_message_prefix}({dim_name}, {item_name}) | Deactivated." + f"{debug_message_prefix}({dim_name}, {item_name}) | Deactivated." ) self._deactivate_item(**{dim_name: item_name, **connected_dims}) continue - if inheritance is not None: - updated_item_def[f"{dim_name}_inheritance"] = ",".join(inheritance) - del updated_item_def["template"] - - updated_defs[item_name] = updated_item_def + updated_defs[item_name] = item_base_def return updated_defs @@ -746,7 +737,7 @@ def _update_numeric_dims(ds: xr.Dataset, id_: str) -> xr.Dataset: def _raise_error_on_transmission_tech_def( self, tech_def_dict: AttrDict, node_name: str ): - """Do not allow any transmission techs are defined in the node-level tech dict. + """Do not allow any transmission techs to be defined in the node-level tech dict. Args: tech_def_dict (dict): Tech definition dict (after full inheritance) at a node. diff --git a/src/calliope/preprocess/model_math.py b/src/calliope/preprocess/model_math.py index a05a6a12..721c8d94 100644 --- a/src/calliope/preprocess/model_math.py +++ b/src/calliope/preprocess/model_math.py @@ -8,6 +8,7 @@ from calliope.attrdict import AttrDict from calliope.exceptions import ModelError +from calliope.io import read_rich_yaml from calliope.util.schema import MATH_SCHEMA, validate_dict from calliope.util.tools import relative_path @@ -166,7 +167,7 @@ def _init_from_string( def _add_file(self, yaml_filepath: Path, name: str) -> None: try: - math = AttrDict.from_yaml(yaml_filepath, allow_override=True) + math = read_rich_yaml(yaml_filepath, allow_override=True) except FileNotFoundError: raise ModelError( f"Math preprocessing | File does not exist: {yaml_filepath}" diff --git a/src/calliope/preprocess/scenarios.py b/src/calliope/preprocess/scenarios.py index 473544fb..d4e1950b 100644 --- a/src/calliope/preprocess/scenarios.py +++ b/src/calliope/preprocess/scenarios.py @@ -1,21 +1,53 @@ # Copyright (C) since 2013 Calliope contributors listed in AUTHORS. # Licensed under the Apache 2.0 License (see LICENSE file). -"""Preprocessing of base model definition and overrides/scenarios into a unified dictionary.""" +"""Preprocessing of model definition into a unified dictionary.""" import logging +from pathlib import Path from calliope import exceptions from calliope.attrdict import AttrDict +from calliope.io import read_rich_yaml from calliope.util.tools import listify LOGGER = logging.getLogger(__name__) -def load_scenario_overrides( - model_definition: dict, +def prepare_model_definition( + data: str | Path | dict, scenario: str | None = None, override_dict: dict | None = None, - **kwargs, +) -> tuple[AttrDict, str]: + """Arrenge model definition data folloging our standardised order of priority. + + Should always be called when defining calliope models from configuration files. + The order of priority is: + + - scenarios/overrides > templates > regular data sections + + Args: + data (str | Path | dict): _description_ + scenario (str | None, optional): _description_. Defaults to None. + override_dict (dict | None, optional): _description_. Defaults to None. + + Returns: + tuple[AttrDict, str]: _description_ + """ + if isinstance(data, dict): + model_def = AttrDict(data) + else: + model_def = read_rich_yaml(data) + model_def, applied_overrides = _load_scenario_overrides(model_def, scenario, override_dict) + template_solver = TemplateSolver(model_def) + model_def = template_solver.resolved_data + + return model_def, applied_overrides + + +def _load_scenario_overrides( + model_definition: dict, + scenario: str | None = None, + override_dict: dict | None = None ) -> tuple[AttrDict, str]: """Apply user-defined overrides to the model definition. @@ -44,7 +76,7 @@ def load_scenario_overrides( # First pass of applying override dict before applying scenarios, # so that can override scenario definitions by override_dict if isinstance(override_dict, str): - override_dict = AttrDict.from_yaml_string(override_dict) + override_dict = read_rich_yaml(override_dict) if isinstance(override_dict, dict): override_dict = AttrDict(override_dict) @@ -88,10 +120,6 @@ def load_scenario_overrides( _log_overrides(model_def_dict, model_def_with_overrides) - model_def_with_overrides.union( - AttrDict({"config.init": kwargs}), allow_override=True - ) - return (model_def_with_overrides, ";".join(applied_overrides)) @@ -100,7 +128,7 @@ def _combine_overrides(overrides: AttrDict, scenario_overrides: list): for override in scenario_overrides: try: yaml_string = overrides[override].to_yaml() - override_with_imports = AttrDict.from_yaml_string(yaml_string) + override_with_imports = read_rich_yaml(yaml_string) except KeyError: raise exceptions.ModelError(f"Override `{override}` is not defined.") try: @@ -153,3 +181,77 @@ def _log_overrides(init_model_def: AttrDict, overriden_model_def: AttrDict) -> N else: continue LOGGER.debug(message) + + +class TemplateSolver: + """Resolves templates before they reach Calliope models.""" + + TEMPLATES_SECTION: str = "templates" + TEMPLATE_CALL: str = "template" + + def __init__(self, data: AttrDict): + """Initialise the solver.""" + self._raw_templates: AttrDict = data.get_key(self.TEMPLATES_SECTION, AttrDict()) + self._raw_data: AttrDict = data + self.resolved_templates: AttrDict + self.resolved_data: AttrDict + self._resolve() + + def _resolve(self): + """Fill in template references and remove template definitions and calls.""" + self.resolved_templates = AttrDict() + for key, value in self._raw_templates.items(): + if not isinstance(value, dict): + raise exceptions.ModelError("Template definitions must be YAML blocks.") + self.resolved_templates[key] = self._resolve_template(key) + self.resolved_data = self._resolve_data(self._raw_data) + + def _resolve_template(self, name: str, stack: None | set[str] = None) -> AttrDict: + """Resolves templates recursively. + + Catches circular template definitions. + """ + if stack is None: + stack = set() + elif name in stack: + raise exceptions.ModelError(f"Circular template reference detected for '{name}'.") + stack.add(name) + + result = AttrDict() + raw_data = self._raw_templates[name] + if self.TEMPLATE_CALL in raw_data: + # Current template takes precedence when overriding values + inherited_name = raw_data[self.TEMPLATE_CALL] + if inherited_name in self.resolved_templates: + inherited_data = self.resolved_templates[inherited_name] + else: + inherited_data = self._resolve_template(inherited_name, stack) + result.union(inherited_data) + + local_data = {k: raw_data[k] for k in raw_data.keys() - {self.TEMPLATE_CALL}} + result.union(local_data, allow_override=True) + + stack.remove(name) + return result + + def _resolve_data(self, section, level: int = 0): + if isinstance(section, dict): + if self.TEMPLATES_SECTION in section: + if level != 0: + raise exceptions.ModelError("Template definitions must be placed at the top level of the YAML file.") + if self.TEMPLATE_CALL in section: + template = self.resolved_templates[section[self.TEMPLATE_CALL]].copy() + else: + template = AttrDict() + + local = AttrDict() + for key in section.keys() - {self.TEMPLATE_CALL, self.TEMPLATES_SECTION}: + local[key] = self._resolve_data(section[key], level=level+1) + + # Local values have priority. + template.union(local, allow_override=True) + result = template + else: + result = section + return result + diff --git a/src/calliope/util/generate_runs.py b/src/calliope/util/generate_runs.py index 37d1b987..0169e00e 100644 --- a/src/calliope/util/generate_runs.py +++ b/src/calliope/util/generate_runs.py @@ -11,7 +11,7 @@ import pandas as pd -from calliope.attrdict import AttrDict +from calliope.io import read_rich_yaml def generate_runs(model_file, scenarios=None, additional_args=None, override_dict=None): @@ -29,9 +29,9 @@ def generate_runs(model_file, scenarios=None, additional_args=None, override_dic """ if scenarios is None: - config = AttrDict.from_yaml(model_file) + config = read_rich_yaml(model_file) if override_dict: - override = AttrDict.from_yaml_string(override_dict) + override = read_rich_yaml(override_dict) config.union(override, allow_override=True, allow_replacement=True) if "scenarios" in config: diff --git a/src/calliope/util/tools.py b/src/calliope/util/tools.py index dee2f6ca..8575409b 100644 --- a/src/calliope/util/tools.py +++ b/src/calliope/util/tools.py @@ -51,55 +51,3 @@ def listify(var: Any) -> list: else: var = [var] return var - - -def climb_template_tree( - input_dict: "AttrDict", - templates: "AttrDict", - item_name: str | None = None, - inheritance: list | None = None, -) -> tuple["AttrDict", list | None]: - """Follow the `template` references from model definition elements to `templates`. - - Model definition elements can inherit template entries (those in `templates`). - Template entries can also inherit each other, to create an inheritance chain. - - This function will be called recursively until a definition dictionary without `template` is reached. - - Args: - input_dict (AttrDict): Dictionary (possibly) containing `template`. - templates (AttrDict): Dictionary of available templates. - item_name (str | None, optional): - The current position in the inheritance tree. - If given, used only for a more expressive KeyError. - Defaults to None. - inheritance (list | None, optional): - A list of items that have been inherited (starting with the oldest). - If the first `input_dict` does not contain `template`, this will remain as None. - Defaults to None. - - Raises: - KeyError: Must inherit from a named template item in `templates`. - - Returns: - tuple[AttrDict, list | None]: Definition dictionary with inherited data and a list of the inheritance tree climbed to get there. - """ - to_inherit = input_dict.get("template", None) - if to_inherit is None: - updated_input_dict = input_dict - elif to_inherit not in templates: - message = f"Cannot find `{to_inherit}` in template inheritance tree." - if item_name is not None: - message = f"{item_name} | {message}" - raise KeyError(message) - else: - base_def_dict, inheritance = climb_template_tree( - templates[to_inherit], templates, to_inherit, inheritance - ) - updated_input_dict = deepcopy(base_def_dict) - updated_input_dict.union(input_dict, allow_override=True) - if inheritance is not None: - inheritance.append(to_inherit) - else: - inheritance = [to_inherit] - return updated_input_dict, inheritance diff --git a/tests/test_backend_helper_functions.py b/tests/test_backend_helper_functions.py index cf77d69d..f94fb5f7 100644 --- a/tests/test_backend_helper_functions.py +++ b/tests/test_backend_helper_functions.py @@ -18,11 +18,6 @@ def where(): return helper_functions._registry["where"] -@pytest.fixture(scope="class") -def where_inheritance(where, parsing_kwargs): - return where["inheritance"](**parsing_kwargs) - - @pytest.fixture(scope="class") def where_any(where, parsing_kwargs): return where["any"](**parsing_kwargs) @@ -100,7 +95,7 @@ def _is_defined(drop_dims, dims): return _is_defined @pytest.mark.parametrize( - ("string_type", "func_name"), [("where", "inheritance"), ("expression", "sum")] + ("string_type", "func_name"), [("where", "defined"), ("expression", "sum")] ) def test_duplicate_name_exception(self, string_type, func_name): with pytest.raises(ValueError, match=rf".*{string_type}.*{func_name}.*"): @@ -126,18 +121,6 @@ def __call__(self): assert all(func_name in helper_functions._registry[i] for i in string_types) - def test_nodes_inheritance(self, where_inheritance, dummy_model_data): - boo_bool = where_inheritance(nodes="boo") - assert boo_bool.equals(dummy_model_data.nodes_inheritance_boo_bool) - - def test_techs_inheritance(self, where_inheritance, dummy_model_data): - boo_bool = where_inheritance(techs="boo") - assert boo_bool.equals(dummy_model_data.techs_inheritance_boo_bool) - - def test_techs_and_nodes_inheritance(self, where_inheritance, dummy_model_data): - boo_bool = where_inheritance(techs="boo", nodes="boo") - assert boo_bool.equals(dummy_model_data.multi_inheritance_boo_bool) - def test_any_not_exists(self, where_any): summed = where_any("foo", over="techs") assert summed.equals(xr.DataArray(False)) @@ -397,18 +380,6 @@ def parsing_kwargs(self, dummy_model_data): "equation_name": "foo", } - def test_techs_inheritance(self, where_inheritance): - assert where_inheritance(techs="boo") == r"\text{inherits(techs=boo)}" - - def test_nodes_inheritance(self, where_inheritance): - assert where_inheritance(nodes="boo") == r"\text{inherits(nodes=boo)}" - - def test_techs_and_nodes_inheritance(self, where_inheritance): - assert ( - where_inheritance(nodes="boo", techs="bar") - == r"\text{inherits(nodes=boo,techs=bar)}" - ) - def test_any_not_exists(self, where_any): summed_string = where_any("foo", over="techs") assert summed_string == r"\bigvee\limits_{\text{tech} \in \text{techs}} (foo)" diff --git a/tests/test_backend_where_parser.py b/tests/test_backend_where_parser.py index 69620155..def6f621 100644 --- a/tests/test_backend_where_parser.py +++ b/tests/test_backend_where_parser.py @@ -3,7 +3,6 @@ import pytest import xarray as xr -from calliope.attrdict import AttrDict from calliope.backend import expression_parser, helper_functions, where_parser from calliope.exceptions import BackendError @@ -14,10 +13,6 @@ BASE_DIMS = ["nodes", "techs", "carriers", "costs", "timesteps"] -def parse_yaml(yaml_string): - return AttrDict.from_yaml_string(yaml_string) - - @pytest.fixture def base_parser_elements(): number, identifier = expression_parser.setup_base_parser_elements() @@ -388,7 +383,7 @@ def test_subsetting_parser(self, subset, subset_string, expected_subset): "[bar] in", # missing set name "foo in [bar]", # Wrong order of subset and set name "[foo=bar] in foo", # comparison string in subset - "[inheritance(techs=a)] in foo" # helper function in subset + "[defined(techs=[tech1, tech2], within=nodes, how=any)] in foo", # helper function in subset "(bar) in foo", # wrong brackets ], ) @@ -419,7 +414,7 @@ class TestParserMasking: [ ("all_inf", "all_false"), ("config.foo=True", True), - ("inheritance(nodes=boo)", "nodes_inheritance_boo_bool"), + ("get_val_at_index(nodes=0)", "foo"), ], ) def test_no_aggregation( diff --git a/tests/test_cli.py b/tests/test_cli.py index 7a3e3374..8e7876a9 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -7,7 +7,7 @@ from click.testing import CliRunner import calliope -from calliope import AttrDict, cli +from calliope import cli, io _MODEL_NATIONAL = ( importlib_resources.files("calliope") @@ -220,7 +220,7 @@ def test_generate_scenarios(self): ) assert result.exit_code == 0 assert os.path.isfile(out_file) - scenarios = AttrDict.from_yaml(out_file) + scenarios = io.read_rich_yaml(out_file) assert "scenario_0" not in scenarios["scenarios"] assert scenarios["scenarios"]["scenario_1"] == [ "cold_fusion", diff --git a/tests/test_core_preprocess.py b/tests/test_core_preprocess.py index b0f286f4..6122ab4d 100644 --- a/tests/test_core_preprocess.py +++ b/tests/test_core_preprocess.py @@ -6,6 +6,7 @@ import calliope import calliope.exceptions as exceptions from calliope.attrdict import AttrDict +from calliope.io import read_rich_yaml from .common.util import build_test_model as build_model from .common.util import check_error_or_warning @@ -16,7 +17,7 @@ def test_model_from_dict(self, data_source_dir): """Test creating a model from dict/AttrDict instead of from YAML""" model_dir = data_source_dir.parent model_location = model_dir / "model.yaml" - model_dict = AttrDict.from_yaml(model_location) + model_dict = read_rich_yaml(model_location) node_dict = AttrDict( { "nodes": { @@ -39,7 +40,7 @@ def test_model_from_dict(self, data_source_dir): ) def test_valid_scenarios(self, dummy_int): """Test that valid scenario definition from overrides raises no error and results in applied scenario.""" - override = AttrDict.from_yaml_string( + override = read_rich_yaml( f""" scenarios: scenario_1: ['one', 'two'] @@ -72,7 +73,7 @@ def test_valid_scenario_of_scenarios(self, dummy_int): """Test that valid scenario definition which groups scenarios and overrides raises no error and results in applied scenario. """ - override = AttrDict.from_yaml_string( + override = read_rich_yaml( f""" scenarios: scenario_1: ['one', 'two'] @@ -107,7 +108,7 @@ def test_valid_scenario_of_scenarios(self, dummy_int): def test_invalid_scenarios_dict(self): """Test that invalid scenario definition raises appropriate error""" - override = AttrDict.from_yaml_string( + override = read_rich_yaml( """ scenarios: scenario_1: @@ -123,7 +124,7 @@ def test_invalid_scenarios_dict(self): def test_invalid_scenarios_str(self): """Test that invalid scenario definition raises appropriate error""" - override = AttrDict.from_yaml_string( + override = read_rich_yaml( """ scenarios: scenario_1: 'foo' @@ -138,7 +139,7 @@ def test_invalid_scenarios_str(self): def test_undefined_carriers(self): """Test that user has input either carrier or carrier_in/_out for each tech""" - override = AttrDict.from_yaml_string( + override = read_rich_yaml( """ techs: test_undefined_carrier: @@ -159,7 +160,7 @@ def test_incorrect_subset_time(self): """ def override(param): - return AttrDict.from_yaml_string(f"config.init.time_subset: {param}") + return read_rich_yaml(f"config.init.time_subset: {param}") # should fail: one string in list with pytest.raises(exceptions.ModelError): @@ -211,7 +212,7 @@ def test_inconsistent_time_indices_fails(self): varying input data are consistent with each other """ # should fail: wrong length of demand_heat csv vs demand_elec - override = AttrDict.from_yaml_string( + override = read_rich_yaml( "data_tables.demand_elec.data: data_tables/demand_heat_wrong_length.csv" ) # check in output error that it points to: 07/01/2005 10:00:00 @@ -222,7 +223,7 @@ def test_inconsistent_time_indices_fails(self): ) def test_inconsistent_time_indices_passes_thanks_to_time_subsetting(self): - override = AttrDict.from_yaml_string( + override = read_rich_yaml( "data_tables.demand_elec.data: data_tables/demand_heat_wrong_length.csv" ) # should pass: wrong length of demand_heat csv, but time subsetting removes the difference @@ -278,7 +279,7 @@ def test_model_version_mismatch(self): def test_unspecified_base_tech(self): """All technologies must specify a base_tech""" - override = AttrDict.from_yaml_string( + override = read_rich_yaml( """ techs.test_supply_no_base_tech: name: Supply tech @@ -294,7 +295,7 @@ def test_unspecified_base_tech(self): def test_tech_as_base_tech(self): """All technologies must specify a base_tech""" - override1 = AttrDict.from_yaml_string( + override1 = read_rich_yaml( """ techs.test_supply_tech_base_tech: name: Supply tech diff --git a/tests/test_core_util.py b/tests/test_core_util.py index 32ea38e9..a0423d37 100644 --- a/tests/test_core_util.py +++ b/tests/test_core_util.py @@ -10,10 +10,10 @@ import pytest import calliope +from calliope.io import read_rich_yaml from calliope.util import schema from calliope.util.generate_runs import generate_runs from calliope.util.logging import log_time -from calliope.util.tools import climb_template_tree from .common.util import check_error_or_warning @@ -184,7 +184,7 @@ def test_invalid_dict(self, to_validate, expected_path): @pytest.fixture def base_math(self): - return calliope.AttrDict.from_yaml( + return read_rich_yaml( Path(calliope.__file__).parent / "math" / "plan.yaml" ) @@ -192,11 +192,11 @@ def base_math(self): "dict_path", glob.glob(str(Path(calliope.__file__).parent / "math" / "*.yaml")) ) def test_validate_math(self, base_math, dict_path): - math_schema = calliope.AttrDict.from_yaml( + math_schema = read_rich_yaml( Path(calliope.__file__).parent / "config" / "math_schema.yaml" ) to_validate = base_math.union( - calliope.AttrDict.from_yaml(dict_path, allow_override=True), + read_rich_yaml(dict_path, allow_override=True), allow_override=True, ) schema.validate_dict(to_validate, math_schema, "") @@ -247,7 +247,7 @@ def sample_config_schema(self): default: false description: operate use cap results. """ - return calliope.AttrDict.from_yaml_string(schema_string) + return read_rich_yaml(schema_string) @pytest.fixture(scope="class") def sample_model_def_schema(self): @@ -321,7 +321,7 @@ def sample_model_def_schema(self): title: Foobar. description: foobar. """ - return calliope.AttrDict.from_yaml_string(schema_string) + return read_rich_yaml(schema_string) @pytest.fixture def expected_config_defaults(self): @@ -466,78 +466,3 @@ def test_reset_schema(self): "^[^_^\\d][\\w]*$" ]["properties"] ) - - -class TestClimbTemplateTree: - @pytest.fixture - def templates(self) -> "calliope.AttrDict": - return calliope.AttrDict( - { - "foo_group": {"template": "bar_group", "my_param": 1}, - "bar_group": {"my_param": 2, "my_other_param": 2}, - "data_table_group": {"rows": ["foobar"]}, - } - ) - - @pytest.mark.parametrize( - ("starting_dict", "expected_dict", "expected_inheritance"), - [ - ({"my_param": 1}, {"my_param": 1}, None), - ( - {"template": "foo_group"}, - {"my_param": 1, "my_other_param": 2, "template": "foo_group"}, - ["bar_group", "foo_group"], - ), - ( - {"template": "bar_group"}, - {"my_param": 2, "my_other_param": 2, "template": "bar_group"}, - ["bar_group"], - ), - ( - {"template": "bar_group", "my_param": 3, "my_own_param": 1}, - { - "my_param": 3, - "my_other_param": 2, - "my_own_param": 1, - "template": "bar_group", - }, - ["bar_group"], - ), - ( - {"template": "data_table_group", "columns": "techs"}, - { - "columns": "techs", - "rows": ["foobar"], - "template": "data_table_group", - }, - ["data_table_group"], - ), - ], - ) - def test_climb_template_tree( - self, templates, starting_dict, expected_dict, expected_inheritance - ): - """Templates should be found and applied in order of 'ancestry' (newer dict keys replace older ones if they overlap).""" - - new_dict, inheritance = climb_template_tree( - calliope.AttrDict(starting_dict), templates, "A" - ) - assert new_dict == expected_dict - assert inheritance == expected_inheritance - - @pytest.mark.parametrize( - ("item_name", "expected_message_prefix"), [("A", "A | "), (None, "")] - ) - def test_climb_template_tree_missing_ancestor( - self, templates, item_name, expected_message_prefix - ): - """Referencing a template that doesn't exist in `templates` raises an error.""" - with pytest.raises(KeyError) as excinfo: - climb_template_tree( - calliope.AttrDict({"template": "not_there"}), templates, item_name - ) - - assert check_error_or_warning( - excinfo, - f"{expected_message_prefix}Cannot find `not_there` in template inheritance tree.", - ) diff --git a/tests/test_example_models.py b/tests/test_example_models.py index 507e3d50..e2448773 100755 --- a/tests/test_example_models.py +++ b/tests/test_example_models.py @@ -7,6 +7,7 @@ import calliope from calliope import exceptions +from calliope.io import read_rich_yaml from .common.util import check_error_or_warning @@ -400,7 +401,7 @@ def example_tester(self, source_unit, solver="cbc", solver_io=None): data_tables = f"data_tables.pv_resource.select.scaler: {source_unit}" unit_override = { "techs.pv.source_unit": source_unit, - **calliope.AttrDict.from_yaml_string(data_tables), + **read_rich_yaml(data_tables), } model = calliope.examples.urban_scale( diff --git a/tests/test_io.py b/tests/test_io.py index 5f993e6e..c4db1d2b 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -403,110 +403,3 @@ def test_to_yaml(self, yaml_from_path): assert "some_int: 10" in result assert "some_float: 0.5" in result assert "a_list:\n- 0\n- 1\n- 2" in result - - -class TestYAMLTemplates: - - @pytest.fixture - def dummy_solved_template(self) -> calliope.io.TemplateSolver: - text = """ - templates: - T1: - A: ["foo", "bar"] - B: 1 - T2: - C: bar - template: T1 - T3: - template: T1 - B: 11 - T4: - template: T3 - A: ["bar", "foobar"] - B: "1" - C: {"foo": "bar"} - D: true - a: - template: T1 - a1: 1 - b: - template: T3 - c: - template: T4 - D: false - """ - yaml_data = calliope.io.read_rich_yaml(text) - return calliope.io.TemplateSolver(yaml_data) - - def test_inheritance_templates(self, dummy_solved_template): - templates = dummy_solved_template.resolved_templates - assert all( - [ - templates.T1 == {"A": ["foo", "bar"], "B": 1}, - templates.T2 == {"A": ["foo", "bar"], "B": 1, "C": "bar"}, - templates.T3 == {"A": ["foo", "bar"], "B": 11}, - templates.T4 == {"A": ["bar", "foobar"], "B": "1", "C": {"foo": "bar"}, "D": True} - ] - ) - - def test_template_inheritance_data(self, dummy_solved_template): - data = dummy_solved_template.resolved_data - assert all( - [ - data.a == {"A": ["foo", "bar"], "B": 1, "a1": 1}, - data.b == {"A": ["foo", "bar"], "B": 11}, - data.c == {"A": ["bar", "foobar"], "B": "1", "C": {"foo": "bar"}, "D": False} - ] - ) - - def test_invalid_template_error(self): - text = calliope.io.read_rich_yaml( - """ - templates: - T1: "not_a_yaml_block" - T2: - foo: bar - a: - template: T2 - """ - ) - with pytest.raises(ValueError, match="Template definitions must be YAML blocks."): - calliope.io.TemplateSolver(text) - - def test_circular_template_error(self): - text = calliope.io.read_rich_yaml( - """ - templates: - T1: - template: T2 - bar: foo - T2: - template: T1 - foo: bar - a: - template: T2 - """ - ) - with pytest.raises(ValueError, match="Circular template reference detected"): - calliope.io.TemplateSolver(text) - - def test_incorrect_template_placement_error(self): - text = calliope.io.read_rich_yaml( - """ - templates: - T1: - stuff: null - T2: - foo: bar - a: - template: T2 - b: - templates: - T3: - this: "should not be here" - """ - ) - with pytest.raises(ValueError, match="Template definitions must be placed at the top level of the YAML file."): - calliope.io.TemplateSolver(text) - - diff --git a/tests/test_math.py b/tests/test_math.py index 35aed4e7..e3c03195 100644 --- a/tests/test_math.py +++ b/tests/test_math.py @@ -7,11 +7,12 @@ from pyomo.repn.tests import lp_diff from calliope import AttrDict +from calliope.io import read_rich_yaml from .common.util import build_lp, build_test_model CALLIOPE_DIR: Path = importlib.resources.files("calliope") -PLAN_MATH: AttrDict = AttrDict.from_yaml(CALLIOPE_DIR / "math" / "plan.yaml") +PLAN_MATH: AttrDict = read_rich_yaml(CALLIOPE_DIR / "math" / "plan.yaml") @pytest.fixture(scope="class") @@ -47,7 +48,7 @@ class TestBaseMath: @pytest.fixture(scope="class") def base_math(self): - return AttrDict.from_yaml(CALLIOPE_DIR / "math" / "plan.yaml") + return read_rich_yaml(CALLIOPE_DIR / "math" / "plan.yaml") def test_flow_cap(self, compare_lps): self.TEST_REGISTER.add("variables.flow_cap") @@ -232,7 +233,7 @@ def abs_filepath(self): @pytest.fixture(scope="class") def custom_math(self): - return AttrDict.from_yaml(self.CUSTOM_MATH_DIR / self.YAML_FILEPATH) + return read_rich_yaml(self.CUSTOM_MATH_DIR / self.YAML_FILEPATH) @pytest.fixture def build_and_compare(self, abs_filepath, compare_lps): diff --git a/tests/test_preprocess_model_data.py b/tests/test_preprocess_model_data.py index c6036087..06816090 100644 --- a/tests/test_preprocess_model_data.py +++ b/tests/test_preprocess_model_data.py @@ -7,7 +7,7 @@ import xarray as xr from calliope import AttrDict, exceptions -from calliope.preprocess import scenarios +from calliope.preprocess import prepare_model_definition from calliope.preprocess.model_data import ModelDataFactory from .common.util import build_test_model as build_model @@ -17,9 +17,8 @@ @pytest.fixture def model_def(): model_def_path = Path(__file__).parent / "common" / "test_model" / "model.yaml" - model_dict = AttrDict.from_yaml(model_def_path) - model_def_override, _ = scenarios.load_scenario_overrides( - model_dict, scenario="simple_supply,empty_tech_node" + model_def_override, _ = prepare_model_definition( + model_def_path, scenario="simple_supply,empty_tech_node" ) return model_def_override, model_def_path @@ -87,9 +86,7 @@ def test_add_node_tech_data(self, model_data_factory_w_params: ModelDataFactory) "heat", } assert set(model_data_factory_w_params.dataset.data_vars.keys()) == { - "nodes_inheritance", "distance", - "techs_inheritance", "name", "carrier_out", "carrier_in", @@ -444,7 +441,7 @@ def test_prepare_param_dict_no_broadcast_allowed( f"foo | Length mismatch between data ({param_data}) and index ([['foo'], ['bar']]) for parameter definition", ) - def test_template_defs_inactive( + def test_inherit_defs_inactive( self, my_caplog, model_data_factory: ModelDataFactory ): def_dict = {"A": {"active": False}} @@ -454,30 +451,12 @@ def test_template_defs_inactive( assert "(nodes, A) | Deactivated." in my_caplog.text assert not new_def_dict - def test_template_defs_nodes_inherit(self, model_data_factory: ModelDataFactory): - def_dict = { - "A": {"template": "init_nodes", "my_param": 1}, - "B": {"my_param": 2}, - } - new_def_dict = model_data_factory._inherit_defs( - dim_name="nodes", dim_dict=AttrDict(def_dict) - ) - - assert new_def_dict == { - "A": { - "nodes_inheritance": "init_nodes", - "my_param": 1, - "techs": {"test_demand_elec": None}, - }, - "B": {"my_param": 2}, - } - - def test_template_defs_nodes_from_base(self, model_data_factory: ModelDataFactory): + def test_inherit_defs_nodes_from_base(self, model_data_factory: ModelDataFactory): """Without a `dim_dict` to start off inheritance chaining, the `dim_name` will be used to find keys.""" new_def_dict = model_data_factory._inherit_defs(dim_name="nodes") assert set(new_def_dict.keys()) == {"a", "b", "c"} - def test_template_defs_techs(self, model_data_factory: ModelDataFactory): + def test_inherit_defs_techs(self, model_data_factory: ModelDataFactory): """`dim_dict` overrides content of base model definition.""" model_data_factory.model_definition.set_key("techs.foo.base_tech", "supply") model_data_factory.model_definition.set_key("techs.foo.my_param", 2) @@ -488,27 +467,7 @@ def test_template_defs_techs(self, model_data_factory: ModelDataFactory): ) assert new_def_dict == {"foo": {"my_param": 1, "base_tech": "supply"}} - def test_template_defs_techs_inherit(self, model_data_factory: ModelDataFactory): - """Use of template is tracked in updated definition dictionary (as `techs_inheritance` here).""" - model_data_factory.model_definition.set_key( - "techs.foo.template", "test_controller" - ) - model_data_factory.model_definition.set_key("techs.foo.base_tech", "supply") - model_data_factory.model_definition.set_key("techs.foo.my_param", 2) - - def_dict = {"foo": {"my_param": 1}} - new_def_dict = model_data_factory._inherit_defs( - dim_name="techs", dim_dict=AttrDict(def_dict) - ) - assert new_def_dict == { - "foo": { - "my_param": 1, - "base_tech": "supply", - "techs_inheritance": "test_controller", - } - } - - def test_template_defs_techs_empty_def(self, model_data_factory: ModelDataFactory): + def test_inherit_defs_techs_empty_def(self, model_data_factory: ModelDataFactory): """An empty `dim_dict` entry can be handled, by returning the model definition for that entry.""" model_data_factory.model_definition.set_key("techs.foo.base_tech", "supply") model_data_factory.model_definition.set_key("techs.foo.my_param", 2) @@ -519,7 +478,7 @@ def test_template_defs_techs_empty_def(self, model_data_factory: ModelDataFactor ) assert new_def_dict == {"foo": {"my_param": 2, "base_tech": "supply"}} - def test_template_defs_techs_missing_base_def( + def test_inherit_defs_techs_missing_base_def( self, model_data_factory: ModelDataFactory ): """If inheriting from a template, checks against the schema will still be undertaken.""" diff --git a/tests/test_preprocess_model_definition.py b/tests/test_preprocess_model_definition.py new file mode 100644 index 00000000..bb1eb015 --- /dev/null +++ b/tests/test_preprocess_model_definition.py @@ -0,0 +1,110 @@ +import pytest + +from calliope.exceptions import ModelError +from calliope.io import read_rich_yaml +from calliope.preprocess.scenarios import TemplateSolver + + +class TestTemplateSolver: + + @pytest.fixture + def dummy_solved_template(self) -> TemplateSolver: + text = """ + templates: + T1: + A: ["foo", "bar"] + B: 1 + T2: + C: bar + template: T1 + T3: + template: T1 + B: 11 + T4: + template: T3 + A: ["bar", "foobar"] + B: "1" + C: {"foo": "bar"} + D: true + a: + template: T1 + a1: 1 + b: + template: T3 + c: + template: T4 + D: false + """ + yaml_data = read_rich_yaml(text) + return TemplateSolver(yaml_data) + + def test_inheritance_templates(self, dummy_solved_template): + templates = dummy_solved_template.resolved_templates + assert all( + [ + templates.T1 == {"A": ["foo", "bar"], "B": 1}, + templates.T2 == {"A": ["foo", "bar"], "B": 1, "C": "bar"}, + templates.T3 == {"A": ["foo", "bar"], "B": 11}, + templates.T4 == {"A": ["bar", "foobar"], "B": "1", "C": {"foo": "bar"}, "D": True} + ] + ) + + def test_template_inheritance_data(self, dummy_solved_template): + data = dummy_solved_template.resolved_data + assert all( + [ + data.a == {"A": ["foo", "bar"], "B": 1, "a1": 1}, + data.b == {"A": ["foo", "bar"], "B": 11}, + data.c == {"A": ["bar", "foobar"], "B": "1", "C": {"foo": "bar"}, "D": False} + ] + ) + + def test_invalid_template_error(self): + text = read_rich_yaml( + """ + templates: + T1: "not_a_yaml_block" + T2: + foo: bar + a: + template: T2 + """ + ) + with pytest.raises(ModelError, match="Template definitions must be YAML blocks."): + TemplateSolver(text) + + def test_circular_template_error(self): + text = read_rich_yaml( + """ + templates: + T1: + template: T2 + bar: foo + T2: + template: T1 + foo: bar + a: + template: T2 + """ + ) + with pytest.raises(ModelError, match="Circular template reference detected"): + TemplateSolver(text) + + def test_incorrect_template_placement_error(self): + text = read_rich_yaml( + """ + templates: + T1: + stuff: null + T2: + foo: bar + a: + template: T2 + b: + templates: + T3: + this: "should not be here" + """ + ) + with pytest.raises(ModelError, match="Template definitions must be placed at the top level of the YAML file."): + TemplateSolver(text) diff --git a/tests/test_preprocess_model_math.py b/tests/test_preprocess_model_math.py index 46af363e..c62ee4d7 100644 --- a/tests/test_preprocess_model_math.py +++ b/tests/test_preprocess_model_math.py @@ -9,6 +9,7 @@ import calliope from calliope.exceptions import ModelError +from calliope.io import read_rich_yaml from calliope.preprocess import CalliopeMath @@ -105,7 +106,7 @@ def model_math_w_mode_user(self, model_math_w_mode, user_math_path, def_path): @pytest.fixture(scope="class") def predefined_mode_data(self, pre_defined_mode): path = Path(calliope.__file__).parent / "math" / f"{pre_defined_mode}.yaml" - math = calliope.AttrDict.from_yaml(path) + math = read_rich_yaml(path) return math def test_predefined_add(self, model_math_w_mode, predefined_mode_data): diff --git a/tests/test_preprocess_time.py b/tests/test_preprocess_time.py index 3272872d..e5c4e037 100644 --- a/tests/test_preprocess_time.py +++ b/tests/test_preprocess_time.py @@ -2,6 +2,7 @@ import pytest # noqa: F401 from calliope import AttrDict, exceptions +from calliope.io import read_rich_yaml from .common.util import build_test_model @@ -14,7 +15,7 @@ def test_change_date_format(self): """ # should pass: changing datetime format from default - override = AttrDict.from_yaml_string( + override = read_rich_yaml( """ config.init.time_format: "%d/%m/%Y %H:%M" data_tables: @@ -30,7 +31,7 @@ def test_change_date_format(self): def test_incorrect_date_format_one(self): # should fail: wrong dateformat input for one file - override = AttrDict.from_yaml_string( + override = read_rich_yaml( "data_tables.demand_elec.data: data_tables/demand_heat_diff_dateformat.csv" ) @@ -46,7 +47,7 @@ def test_incorrect_date_format_multi(self): def test_incorrect_date_format_one_value_only(self): # should fail: one value wrong in file - override = AttrDict.from_yaml_string( + override = read_rich_yaml( "data_tables.test_demand_elec.data: data_tables/demand_heat_wrong_dateformat.csv" ) # check in output error that it points to: 07/01/2005 10:00:00 @@ -151,7 +152,7 @@ class TestResampling: def test_15min_resampling_to_6h(self): # The data is identical for '2005-01-01' and '2005-01-03' timesteps, # it is only different for '2005-01-02' - override = AttrDict.from_yaml_string( + override = read_rich_yaml( "data_tables.demand_elec.data: data_tables/demand_elec_15mins.csv" ) @@ -178,7 +179,7 @@ def test_15min_to_2h_resampling_to_2h(self): """ CSV has daily timeseries varying from 15min to 2h resolution, resample all to 2h """ - override = AttrDict.from_yaml_string( + override = read_rich_yaml( "data_tables.demand_elec.data: data_tables/demand_elec_15T_to_2h.csv" ) @@ -212,7 +213,7 @@ def test_15min_to_2h_resampling_to_2h(self): def test_different_ts_resolutions_resampling_to_6h(self): # The data is identical for '2005-01-01' and '2005-01-03' timesteps, # it is only different for '2005-01-02' - override = AttrDict.from_yaml_string( + override = read_rich_yaml( """ data_tables: demand_elec: From 2a1330d13ce5cf14e6d7234a31b4f259f2695945 Mon Sep 17 00:00:00 2001 From: Ivan Ruiz Manuel <72193617+irm-codebase@users.noreply.github.com> Date: Fri, 29 Nov 2024 19:52:58 +0100 Subject: [PATCH 05/13] Rename scenarios to model_definition and rearrange test location. --- src/calliope/attrdict.py | 3 - src/calliope/config/config_schema.yaml | 8 -- src/calliope/io.py | 5 +- src/calliope/preprocess/__init__.py | 2 +- .../{scenarios.py => model_definition.py} | 17 ++- src/calliope/util/tools.py | 6 +- tests/test_core_attrdict.py | 2 - tests/test_core_preprocess.py | 107 +-------------- tests/test_core_util.py | 7 +- tests/test_preprocess_model_definition.py | 125 +++++++++++++++++- tests/test_preprocess_time.py | 2 +- 11 files changed, 138 insertions(+), 146 deletions(-) rename src/calliope/preprocess/{scenarios.py => model_definition.py} (95%) diff --git a/src/calliope/attrdict.py b/src/calliope/attrdict.py index c056c3b4..3c171090 100644 --- a/src/calliope/attrdict.py +++ b/src/calliope/attrdict.py @@ -5,14 +5,11 @@ import copy import io import logging -from pathlib import Path import numpy as np import ruamel.yaml as ruamel_yaml from typing_extensions import Self -from calliope.util.tools import relative_path - logger = logging.getLogger(__name__) diff --git a/src/calliope/config/config_schema.yaml b/src/calliope/config/config_schema.yaml index 7348bd18..f6fbeb51 100644 --- a/src/calliope/config/config_schema.yaml +++ b/src/calliope/config/config_schema.yaml @@ -228,14 +228,6 @@ properties: additionalProperties: false patternProperties: *nested_pattern - # templates: - # type: [object, "null"] - # description: >- - # Abstract technology/node templates from which techs/nodes can `inherit`. - # See the model definition schema for more guidance on content. - # additionalProperties: false - # patternProperties: *nested_pattern - overrides: type: [object, "null"] description: >- diff --git a/src/calliope/io.py b/src/calliope/io.py index 93ec19a0..275ed98d 100644 --- a/src/calliope/io.py +++ b/src/calliope/io.py @@ -217,10 +217,7 @@ def load_config(filename: str): return loaded -def read_rich_yaml( - yaml: str | Path, - allow_override: bool = False, -) -> AttrDict: +def read_rich_yaml(yaml: str | Path, allow_override: bool = False) -> AttrDict: """Returns an AttrDict initialised from the given YAML file or string. Uses calliope's "flavour" for YAML files. diff --git a/src/calliope/preprocess/__init__.py b/src/calliope/preprocess/__init__.py index f1998f20..b5bd90c8 100644 --- a/src/calliope/preprocess/__init__.py +++ b/src/calliope/preprocess/__init__.py @@ -2,5 +2,5 @@ from calliope.preprocess.data_tables import DataTable from calliope.preprocess.model_data import ModelDataFactory +from calliope.preprocess.model_definition import prepare_model_definition from calliope.preprocess.model_math import CalliopeMath -from calliope.preprocess.scenarios import prepare_model_definition diff --git a/src/calliope/preprocess/scenarios.py b/src/calliope/preprocess/model_definition.py similarity index 95% rename from src/calliope/preprocess/scenarios.py rename to src/calliope/preprocess/model_definition.py index d4e1950b..4fcf1ba3 100644 --- a/src/calliope/preprocess/scenarios.py +++ b/src/calliope/preprocess/model_definition.py @@ -37,7 +37,9 @@ def prepare_model_definition( model_def = AttrDict(data) else: model_def = read_rich_yaml(data) - model_def, applied_overrides = _load_scenario_overrides(model_def, scenario, override_dict) + model_def, applied_overrides = _load_scenario_overrides( + model_def, scenario, override_dict + ) template_solver = TemplateSolver(model_def) model_def = template_solver.resolved_data @@ -47,7 +49,7 @@ def prepare_model_definition( def _load_scenario_overrides( model_definition: dict, scenario: str | None = None, - override_dict: dict | None = None + override_dict: dict | None = None, ) -> tuple[AttrDict, str]: """Apply user-defined overrides to the model definition. @@ -214,7 +216,9 @@ def _resolve_template(self, name: str, stack: None | set[str] = None) -> AttrDic if stack is None: stack = set() elif name in stack: - raise exceptions.ModelError(f"Circular template reference detected for '{name}'.") + raise exceptions.ModelError( + f"Circular template reference detected for '{name}'." + ) stack.add(name) result = AttrDict() @@ -238,7 +242,9 @@ def _resolve_data(self, section, level: int = 0): if isinstance(section, dict): if self.TEMPLATES_SECTION in section: if level != 0: - raise exceptions.ModelError("Template definitions must be placed at the top level of the YAML file.") + raise exceptions.ModelError( + "Template definitions must be placed at the top level of the YAML file." + ) if self.TEMPLATE_CALL in section: template = self.resolved_templates[section[self.TEMPLATE_CALL]].copy() else: @@ -246,7 +252,7 @@ def _resolve_data(self, section, level: int = 0): local = AttrDict() for key in section.keys() - {self.TEMPLATE_CALL, self.TEMPLATES_SECTION}: - local[key] = self._resolve_data(section[key], level=level+1) + local[key] = self._resolve_data(section[key], level=level + 1) # Local values have priority. template.union(local, allow_override=True) @@ -254,4 +260,3 @@ def _resolve_data(self, section, level: int = 0): else: result = section return result - diff --git a/src/calliope/util/tools.py b/src/calliope/util/tools.py index 8575409b..51920d88 100644 --- a/src/calliope/util/tools.py +++ b/src/calliope/util/tools.py @@ -2,15 +2,11 @@ # Licensed under the Apache 2.0 License (see LICENSE file). """Assorted helper tools.""" -from copy import deepcopy from pathlib import Path -from typing import TYPE_CHECKING, Any, TypeVar +from typing import Any, TypeVar from typing_extensions import ParamSpec -if TYPE_CHECKING: - from calliope import AttrDict - P = ParamSpec("P") T = TypeVar("T") diff --git a/tests/test_core_attrdict.py b/tests/test_core_attrdict.py index 32a8dd5b..340e72e2 100644 --- a/tests/test_core_attrdict.py +++ b/tests/test_core_attrdict.py @@ -16,7 +16,6 @@ def regular_dict(self): } return d - @pytest.fixture def attr_dict(self, regular_dict): d = regular_dict @@ -211,4 +210,3 @@ def test_del_key_nested(self, attr_dict): def test_to_yaml_string(self, attr_dict): result = attr_dict.to_yaml() assert "a: 1" in result - diff --git a/tests/test_core_preprocess.py b/tests/test_core_preprocess.py index 6122ab4d..0ee2f38c 100644 --- a/tests/test_core_preprocess.py +++ b/tests/test_core_preprocess.py @@ -5,7 +5,6 @@ import calliope import calliope.exceptions as exceptions -from calliope.attrdict import AttrDict from calliope.io import read_rich_yaml from .common.util import build_test_model as build_model @@ -17,8 +16,8 @@ def test_model_from_dict(self, data_source_dir): """Test creating a model from dict/AttrDict instead of from YAML""" model_dir = data_source_dir.parent model_location = model_dir / "model.yaml" - model_dict = read_rich_yaml(model_location) - node_dict = AttrDict( + model_dict = calliope.io.read_rich_yaml(model_location) + node_dict = calliope.AttrDict( { "nodes": { "a": {"techs": {"test_supply_elec": {}, "test_demand_elec": {}}}, @@ -35,108 +34,6 @@ def test_model_from_dict(self, data_source_dir): # test as dict calliope.Model(model_dict.as_dict()) - @pytest.mark.filterwarnings( - "ignore:(?s).*(links, test_link_a_b_elec) | Deactivated:calliope.exceptions.ModelWarning" - ) - def test_valid_scenarios(self, dummy_int): - """Test that valid scenario definition from overrides raises no error and results in applied scenario.""" - override = read_rich_yaml( - f""" - scenarios: - scenario_1: ['one', 'two'] - - overrides: - one: - techs.test_supply_gas.flow_cap_max: {dummy_int} - two: - techs.test_supply_elec.flow_cap_max: {dummy_int/2} - - nodes: - a: - techs: - test_supply_gas: - test_supply_elec: - test_demand_elec: - """ - ) - model = build_model(override_dict=override, scenario="scenario_1") - - assert ( - model._model_data.sel(techs="test_supply_gas")["flow_cap_max"] == dummy_int - ) - assert ( - model._model_data.sel(techs="test_supply_elec")["flow_cap_max"] - == dummy_int / 2 - ) - - def test_valid_scenario_of_scenarios(self, dummy_int): - """Test that valid scenario definition which groups scenarios and overrides raises - no error and results in applied scenario. - """ - override = read_rich_yaml( - f""" - scenarios: - scenario_1: ['one', 'two'] - scenario_2: ['scenario_1', 'new_location'] - - overrides: - one: - techs.test_supply_gas.flow_cap_max: {dummy_int} - two: - techs.test_supply_elec.flow_cap_max: {dummy_int/2} - new_location: - nodes.b.techs: - test_supply_elec: - - nodes: - a: - techs: - test_supply_gas: - test_supply_elec: - test_demand_elec: - """ - ) - model = build_model(override_dict=override, scenario="scenario_2") - - assert ( - model._model_data.sel(techs="test_supply_gas")["flow_cap_max"] == dummy_int - ) - assert ( - model._model_data.sel(techs="test_supply_elec")["flow_cap_max"] - == dummy_int / 2 - ) - - def test_invalid_scenarios_dict(self): - """Test that invalid scenario definition raises appropriate error""" - override = read_rich_yaml( - """ - scenarios: - scenario_1: - techs.foo.bar: 1 - """ - ) - with pytest.raises(exceptions.ModelError) as excinfo: - build_model(override_dict=override, scenario="scenario_1") - - assert check_error_or_warning( - excinfo, "(scenarios, scenario_1) | Unrecognised override name: techs." - ) - - def test_invalid_scenarios_str(self): - """Test that invalid scenario definition raises appropriate error""" - override = read_rich_yaml( - """ - scenarios: - scenario_1: 'foo' - """ - ) - with pytest.raises(exceptions.ModelError) as excinfo: - build_model(override_dict=override, scenario="scenario_1") - - assert check_error_or_warning( - excinfo, "(scenarios, scenario_1) | Unrecognised override name: foo." - ) - def test_undefined_carriers(self): """Test that user has input either carrier or carrier_in/_out for each tech""" override = read_rich_yaml( diff --git a/tests/test_core_util.py b/tests/test_core_util.py index a0423d37..e1c2baff 100644 --- a/tests/test_core_util.py +++ b/tests/test_core_util.py @@ -184,9 +184,7 @@ def test_invalid_dict(self, to_validate, expected_path): @pytest.fixture def base_math(self): - return read_rich_yaml( - Path(calliope.__file__).parent / "math" / "plan.yaml" - ) + return read_rich_yaml(Path(calliope.__file__).parent / "math" / "plan.yaml") @pytest.mark.parametrize( "dict_path", glob.glob(str(Path(calliope.__file__).parent / "math" / "*.yaml")) @@ -196,8 +194,7 @@ def test_validate_math(self, base_math, dict_path): Path(calliope.__file__).parent / "config" / "math_schema.yaml" ) to_validate = base_math.union( - read_rich_yaml(dict_path, allow_override=True), - allow_override=True, + read_rich_yaml(dict_path, allow_override=True), allow_override=True ) schema.validate_dict(to_validate, math_schema, "") diff --git a/tests/test_preprocess_model_definition.py b/tests/test_preprocess_model_definition.py index bb1eb015..1501165e 100644 --- a/tests/test_preprocess_model_definition.py +++ b/tests/test_preprocess_model_definition.py @@ -2,11 +2,117 @@ from calliope.exceptions import ModelError from calliope.io import read_rich_yaml -from calliope.preprocess.scenarios import TemplateSolver +from calliope.preprocess.model_definition import TemplateSolver +from .common.util import build_test_model as build_model +from .common.util import check_error_or_warning -class TestTemplateSolver: +class TestScenarioOverrides: + @pytest.mark.filterwarnings( + "ignore:(?s).*(links, test_link_a_b_elec) | Deactivated:calliope.exceptions.ModelWarning" + ) + def test_valid_scenarios(self, dummy_int): + """Test that valid scenario definition from overrides raises no error and results in applied scenario.""" + override = read_rich_yaml( + f""" + scenarios: + scenario_1: ['one', 'two'] + + overrides: + one: + techs.test_supply_gas.flow_cap_max: {dummy_int} + two: + techs.test_supply_elec.flow_cap_max: {dummy_int/2} + + nodes: + a: + techs: + test_supply_gas: + test_supply_elec: + test_demand_elec: + """ + ) + model = build_model(override_dict=override, scenario="scenario_1") + + assert ( + model._model_data.sel(techs="test_supply_gas")["flow_cap_max"] == dummy_int + ) + assert ( + model._model_data.sel(techs="test_supply_elec")["flow_cap_max"] + == dummy_int / 2 + ) + + def test_valid_scenario_of_scenarios(self, dummy_int): + """Test that valid scenario definition which groups scenarios and overrides raises + no error and results in applied scenario. + """ + override = read_rich_yaml( + f""" + scenarios: + scenario_1: ['one', 'two'] + scenario_2: ['scenario_1', 'new_location'] + + overrides: + one: + techs.test_supply_gas.flow_cap_max: {dummy_int} + two: + techs.test_supply_elec.flow_cap_max: {dummy_int/2} + new_location: + nodes.b.techs: + test_supply_elec: + + nodes: + a: + techs: + test_supply_gas: + test_supply_elec: + test_demand_elec: + """ + ) + model = build_model(override_dict=override, scenario="scenario_2") + + assert ( + model._model_data.sel(techs="test_supply_gas")["flow_cap_max"] == dummy_int + ) + assert ( + model._model_data.sel(techs="test_supply_elec")["flow_cap_max"] + == dummy_int / 2 + ) + + def test_invalid_scenarios_dict(self): + """Test that invalid scenario definition raises appropriate error""" + override = read_rich_yaml( + """ + scenarios: + scenario_1: + techs.foo.bar: 1 + """ + ) + with pytest.raises(ModelError) as excinfo: + build_model(override_dict=override, scenario="scenario_1") + + assert check_error_or_warning( + excinfo, "(scenarios, scenario_1) | Unrecognised override name: techs." + ) + + def test_invalid_scenarios_str(self): + """Test that invalid scenario definition raises appropriate error""" + override = read_rich_yaml( + """ + scenarios: + scenario_1: 'foo' + """ + ) + with pytest.raises(ModelError) as excinfo: + build_model(override_dict=override, scenario="scenario_1") + + assert check_error_or_warning( + excinfo, "(scenarios, scenario_1) | Unrecognised override name: foo." + ) + + +class TestTemplateSolver: @pytest.fixture def dummy_solved_template(self) -> TemplateSolver: text = """ @@ -45,7 +151,8 @@ def test_inheritance_templates(self, dummy_solved_template): templates.T1 == {"A": ["foo", "bar"], "B": 1}, templates.T2 == {"A": ["foo", "bar"], "B": 1, "C": "bar"}, templates.T3 == {"A": ["foo", "bar"], "B": 11}, - templates.T4 == {"A": ["bar", "foobar"], "B": "1", "C": {"foo": "bar"}, "D": True} + templates.T4 + == {"A": ["bar", "foobar"], "B": "1", "C": {"foo": "bar"}, "D": True}, ] ) @@ -55,7 +162,8 @@ def test_template_inheritance_data(self, dummy_solved_template): [ data.a == {"A": ["foo", "bar"], "B": 1, "a1": 1}, data.b == {"A": ["foo", "bar"], "B": 11}, - data.c == {"A": ["bar", "foobar"], "B": "1", "C": {"foo": "bar"}, "D": False} + data.c + == {"A": ["bar", "foobar"], "B": "1", "C": {"foo": "bar"}, "D": False}, ] ) @@ -70,7 +178,9 @@ def test_invalid_template_error(self): template: T2 """ ) - with pytest.raises(ModelError, match="Template definitions must be YAML blocks."): + with pytest.raises( + ModelError, match="Template definitions must be YAML blocks." + ): TemplateSolver(text) def test_circular_template_error(self): @@ -106,5 +216,8 @@ def test_incorrect_template_placement_error(self): this: "should not be here" """ ) - with pytest.raises(ModelError, match="Template definitions must be placed at the top level of the YAML file."): + with pytest.raises( + ModelError, + match="Template definitions must be placed at the top level of the YAML file.", + ): TemplateSolver(text) diff --git a/tests/test_preprocess_time.py b/tests/test_preprocess_time.py index e5c4e037..dab2cbd6 100644 --- a/tests/test_preprocess_time.py +++ b/tests/test_preprocess_time.py @@ -1,7 +1,7 @@ import pandas as pd import pytest # noqa: F401 -from calliope import AttrDict, exceptions +from calliope import exceptions from calliope.io import read_rich_yaml from .common.util import build_test_model From 5526a65df87078de856aa198de24872403640dd7 Mon Sep 17 00:00:00 2001 From: Ivan Ruiz Manuel <72193617+irm-codebase@users.noreply.github.com> Date: Fri, 29 Nov 2024 20:11:18 +0100 Subject: [PATCH 06/13] Remove redundant AttrDict call in AttrDict.union(). Fill CHANGELOG. --- CHANGELOG.md | 6 ++++++ src/calliope/model.py | 2 +- src/calliope/preprocess/data_tables.py | 18 ++++++++---------- src/calliope/preprocess/model_data.py | 10 ++++------ src/calliope/preprocess/model_definition.py | 10 +++++----- src/calliope/util/schema.py | 4 ++-- tests/common/util.py | 4 ++-- 7 files changed, 28 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b58be7ca..abea82cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ### User-facing changes +|changed| `template:` can now be used anywhere within YAML definition files, not just in the `nodes`, `techs` and `data_tables` sections. + |changed| Single data entries defined in YAML indexed parameters will not be automatically broadcast along indexed dimensions. To achieve the same functionality as in ` tuple[AttrDict, AttrDict]: for param in self.PARAMS_TO_INITIALISE_YAML: if param in self.dataset: base_tech_dict = self.dataset[param].to_dataframe().dropna().T.to_dict() - base_tech_data.union(AttrDict(base_tech_dict)) + base_tech_data.union(base_tech_dict) return tech_dict, base_tech_data @@ -229,15 +229,13 @@ def __extract_data(grouped_series): ) else: lookup_dict.union( - AttrDict( - self.dataset[param] - .to_series() - .reset_index(lookup_dim) - .groupby("techs") - .apply(__extract_data) - .dropna() - .to_dict() - ) + self.dataset[param] + .to_series() + .reset_index(lookup_dim) + .groupby("techs") + .apply(__extract_data) + .dropna() + .to_dict() ) return lookup_dict diff --git a/src/calliope/preprocess/model_data.py b/src/calliope/preprocess/model_data.py index fa14f120..b94e1c9f 100644 --- a/src/calliope/preprocess/model_data.py +++ b/src/calliope/preprocess/model_data.py @@ -637,12 +637,10 @@ def _links_to_node_format(self, active_node_dict: AttrDict) -> AttrDict: self._update_one_way_links(node_from_data, node_to_data) link_tech_dict.union( - AttrDict( - { - node_from: {link_name: node_from_data}, - node_to: {link_name: node_to_data}, - } - ) + { + node_from: {link_name: node_from_data}, + node_to: {link_name: node_to_data}, + } ) return link_tech_dict diff --git a/src/calliope/preprocess/model_definition.py b/src/calliope/preprocess/model_definition.py index 4fcf1ba3..b6a32ae3 100644 --- a/src/calliope/preprocess/model_definition.py +++ b/src/calliope/preprocess/model_definition.py @@ -18,17 +18,17 @@ def prepare_model_definition( scenario: str | None = None, override_dict: dict | None = None, ) -> tuple[AttrDict, str]: - """Arrenge model definition data folloging our standardised order of priority. + """Arrange model definition data folloging our standardised order of priority. Should always be called when defining calliope models from configuration files. The order of priority is: - - scenarios/overrides > templates > regular data sections + - override_dict > scenarios > data section > template Args: - data (str | Path | dict): _description_ - scenario (str | None, optional): _description_. Defaults to None. - override_dict (dict | None, optional): _description_. Defaults to None. + data (str | Path | dict): model data file or dictionary. + scenario (str | None, optional): scenario to run. Defaults to None. + override_dict (dict | None, optional): additional overrides. Defaults to None. Returns: tuple[AttrDict, str]: _description_ diff --git a/src/calliope/util/schema.py b/src/calliope/util/schema.py index bd98cc77..86613580 100644 --- a/src/calliope/util/schema.py +++ b/src/calliope/util/schema.py @@ -30,7 +30,7 @@ def update_then_validate_config( ) -> AttrDict: """Return an updated version of the configuration schema.""" to_validate = deepcopy(config_dict[config_key]) - to_validate.union(AttrDict(update_kwargs), allow_override=True) + to_validate.union(update_kwargs, allow_override=True) validate_dict( {"config": {config_key: to_validate}}, CONFIG_SCHEMA, @@ -70,7 +70,7 @@ def update_model_schema( "^[^_^\\d][\\w]*$" ]["properties"] - to_update.union(AttrDict(new_entries), allow_override=allow_override) + to_update.union(new_entries, allow_override=allow_override) validator = jsonschema.Draft202012Validator validator.META_SCHEMA["unevaluatedProperties"] = False diff --git a/tests/common/util.py b/tests/common/util.py index 8ae70da8..9d658637 100644 --- a/tests/common/util.py +++ b/tests/common/util.py @@ -103,7 +103,7 @@ def build_lp( if isinstance(math_data, dict): for component_group, component_math in math_data.items(): if isinstance(component_math, dict): - math_to_add.union(calliope.AttrDict({component_group: component_math})) + math_to_add.union({component_group: component_math}) elif isinstance(component_math, list): for name in component_math: math_to_add.set_key( @@ -113,7 +113,7 @@ def build_lp( obj = { "dummy_obj": {"equations": [{"expression": "1 + 1"}], "sense": "minimize"} } - math_to_add.union(calliope.AttrDict({"objectives": obj})) + math_to_add.union({"objectives": obj}) obj_to_activate = "dummy_obj" else: obj_to_activate = list(math_to_add["objectives"].keys())[0] From 3b005bc33593e424708deea30f78345c34229d69 Mon Sep 17 00:00:00 2001 From: Ivan Ruiz Manuel <72193617+irm-codebase@users.noreply.github.com> Date: Mon, 2 Dec 2024 15:34:54 +0100 Subject: [PATCH 07/13] Update documentation: templates now in YAML section --- docs/creating/data_tables.md | 3 +- docs/creating/index.md | 3 +- docs/creating/techs.md | 2 +- docs/creating/templates.md | 144 ---------- docs/creating/yaml.md | 311 ++++++++++++++++----- docs/examples/national_scale/index.md | 2 +- docs/examples/urban_scale/index.md | 2 +- docs/migrating.md | 5 +- docs/user_defined_math/helper_functions.md | 2 +- mkdocs.yml | 1 - 10 files changed, 244 insertions(+), 231 deletions(-) delete mode 100644 docs/creating/templates.md diff --git a/docs/creating/data_tables.md b/docs/creating/data_tables.md index 2319ed04..d0d194c6 100644 --- a/docs/creating/data_tables.md +++ b/docs/creating/data_tables.md @@ -18,7 +18,6 @@ In brief it is: * [**drop**](#selecting-dimension-values-and-dropping-dimensions): dimensions to drop from your rows/columns, e.g., a "comment" row. * [**add_dims**](#adding-dimensions): dimensions to add to the table after loading it in, with the corresponding value(s) to assign to the dimension index. * [**rename_dims**](#renaming-dimensions-on-load): dimension names to map from those defined in the data table (e.g `time`) to those used in the Calliope model (e.g. `timesteps`). -* [**template**](#using-a-template): Reference to a [template](templates.md) from which to inherit common configuration options. When we refer to "dimensions", we mean the sets over which data is indexed in the model: `nodes`, `techs`, `timesteps`, `carriers`, `costs`. In addition, when loading from file, there is the _required_ dimension `parameters`. @@ -271,7 +270,7 @@ In this section we will show some examples of loading data and provide the equiv cost_storage_cap.data: 150 ``` - 1. To limit repetition, we have defined [templates](templates.md) for our costs. + 1. To limit repetition, we have defined [templates](yaml.md#reusing-definitions-through-templates) for our costs. !!! info "See also" Our [data table loading tutorial][loading-tabular-data] has more examples of loading tabular data into your model. diff --git a/docs/creating/index.md b/docs/creating/index.md index 35ddefb2..b242aef8 100644 --- a/docs/creating/index.md +++ b/docs/creating/index.md @@ -35,7 +35,7 @@ We distinguish between: - the model **definition** (your representation of a physical system in YAML). Model configuration is everything under the top-level YAML key [`config`](config.md). -Model definition is everything else, under the top-level YAML keys [`parameters`](parameters.md), [`techs`](techs.md), [`nodes`](nodes.md), [`templates`](templates.md), and [`data_tables`](data_tables.md). +Model definition is everything else, under the top-level YAML keys [`parameters`](parameters.md), [`techs`](techs.md), [`nodes`](nodes.md), and [`data_tables`](data_tables.md). It is possible to define alternatives to the model configuration/definition that you can refer to when you initialise your model. These are defined under the top-level YAML keys [`scenarios` and `overrides`](scenarios.md). @@ -84,5 +84,4 @@ The rest of this section discusses everything you need to know to set up a model - An overview of [YAML as it is used in Calliope](yaml.md) - though this comes first here, you can also safely skip it and refer back to it as a reference as questions arise when you go through the model configuration and definition examples. - More details on the [model configuration](config.md). - The key parts of the model definition, first, the [technologies](techs.md), then, the [nodes](nodes.md), the locations in space where technologies can be placed. -- How to use [technology and node templates](templates.md) to reduce repetition in the model definition. - Other important features to be aware of when defining your model: defining [indexed parameters](parameters.md), i.e. parameter which are not indexed over technologies and nodes, [loading tabular data](data_tables.md), and defining [scenarios and overrides](scenarios.md). diff --git a/docs/creating/techs.md b/docs/creating/techs.md index 60d0479d..f2a2a73a 100644 --- a/docs/creating/techs.md +++ b/docs/creating/techs.md @@ -15,7 +15,7 @@ This establishes the basic characteristics in the optimisation model (decision v ??? info "Sharing configuration with templates" To share definitions between technologies and/or nodes, you can use configuration templates (the `template` key). - This allows a technology/node to inherit definitions from [`template` definitions](templates.md). + This allows a technology/node to inherit definitions from [`template` definitions](yaml.md#reusing-definitions-through-templates). Note that `template` is different to setting a `base_tech`. Setting a base_tech does not entail any configuration options being inherited; `base_tech` is only used when building the optimisation problem (i.e., in the `math`). diff --git a/docs/creating/templates.md b/docs/creating/templates.md deleted file mode 100644 index c723c8c8..00000000 --- a/docs/creating/templates.md +++ /dev/null @@ -1,144 +0,0 @@ - -# Inheriting from templates: `templates` - -For larger models, duplicate entries can start to crop up and become cumbersome. -To streamline data entry, technologies, nodes, and data tables can inherit common data from a `template`. - -## Templates in technologies - -If we want to set interest rate to `0.1` across all our technologies, we could define: - -```yaml -templates: - interest_rate_setter: - cost_interest_rate: - data: 0.1 - index: monetary - dims: costs -techs: - ccgt: - template: interest_rate_setter - ... - ac_transmission: - template: interest_rate_setter - ... -``` - -## Templates in nodes - -Similarly, if we want to allow the same technologies at all our nodes: - -```yaml -templates: - standard_tech_list: - techs: {ccgt, battery, demand_power} # (1)! -nodes: - region1: - template: standard_tech_list - ... - region2: - template: standard_tech_list - ... - ... - region100: - template: standard_tech_list -``` - -1. this YAML syntax is shortform for: - ```yaml - techs: - ccgt: - battery: - demand_power: - ``` - -## Templates in data tables - -Data tables can also store common options under the `templates` key, for example: - -```yaml -templates: - common_data_options: - rows: timesteps - columns: nodes - add_dims: - parameters: source_use_max -data_tables: - pv_data: - data: /path/to/pv_timeseries.csv - template: common_data_options - add_dims: - techs: pv - wind_data: - data: /path/to/wind_timeseries.csv - template: common_data_options - add_dims: - techs: wind - hydro_data: - data: /path/to/hydro_timeseries.csv - template: common_data_options - add_dims: - techs: hydro -``` - -## Inheritance chains - -Inheritance chains can also be created. -That is, templates can inherit from other templates. -E.g.: - -```yaml -templates: - interest_rate_setter: - cost_interest_rate: - data: 0.1 - index: monetary - dims: costs - investment_cost_setter: - template: interest_rate_setter - cost_flow_cap: - data: 100 - index: monetary - dims: costs - cost_area_use: - data: 1 - index: monetary - dims: costs -techs: - ccgt: - template: investment_cost_setter - ... - ac_transmission: - template: interest_rate_setter - ... -``` - -## Overriding template values - -Template properties can always be overridden by the inheriting component. -This can be useful to streamline setting costs, e.g.: - -```yaml -templates: - interest_rate_setter: - cost_interest_rate: - data: 0.1 - index: monetary - dims: costs - investment_cost_setter: - template: interest_rate_setter - cost_interest_rate.data: 0.2 # this will replace `0.1` in the `interest_rate_setter`. - cost_flow_cap: - data: null - index: monetary - dims: costs - cost_area_use: - data: null - index: monetary - dims: costs -techs: - ccgt: - template: investment_cost_setter - cost_flow_cap.data: 100 # this will replace `null` in the `investment_cost_setter`. - ... -``` diff --git a/docs/creating/yaml.md b/docs/creating/yaml.md index 6264afa2..4ede042f 100644 --- a/docs/creating/yaml.md +++ b/docs/creating/yaml.md @@ -2,9 +2,90 @@ All model configuration/definition files (with the exception of tabular data files) are in the YAML format, "a human friendly data serialisation standard for all programming languages". +## A quick introduction to YAML + Configuration for Calliope is usually specified as `option: value` entries, where `value` might be a number, a text string, or a list (e.g. a list of further settings). -## Abbreviated nesting +!!! info "See also" + See the [YAML website](https://yaml.org/) for more general information about YAML. + +### Data types + +Using quotation marks (`'` or `"`) to enclose strings is optional, but can help with readability. +The three ways of setting `option` to `text` below are equivalent: + +```yaml +option1: "text" +option2: 'text' +option3: text +``` + +Without quotations, the following values in YAML will be converted to different Python types: + +- Any unquoted number will be interpreted as numeric (e.g., `1`, `1e6` `1e-10`). +- `true` or `false` values will be interpreted as boolean. +- `.inf` and `.nan` values will be interpreted as the float values `np.inf` (infinite) and `np.nan` (not a number), respectively. +- `null` values will interpreted as `None`. + +### Comments + +Comments can be inserted anywhere in YAML files with the `#` symbol. +The remainder of a line after `#` is interpreted as a comment. +Therefore, if you have a string with a `#` in it, make sure to use explicit quotation marks. + +```yaml +# This is a comment +option1: "text with ##hashtags## needs quotation marks" +``` + +### Lists and dictionaries + +Lists in YAML can be of the form `[...]` or a series of lines starting with `-`. +These two lists are equivalent: + +```yaml +key: [option1, option2] +``` + +```yaml +key: + - option1 + - option2 +``` + +Dictionaries can be of the form `{...}` or a series of lines _without_ a starting `-`. +These two dictionaries are equivalent: + +```yaml +key: {option1: value1, option2: value2} +``` + +```yaml +key: + option1: value1 + option2: value2 +``` + +To continue dictionary nesting, you can add more `{}` parentheses or you can indent your lines further. +We prefer to use 2 spaces for indenting as this makes the nested data structures more readable than the often-used 4 spaces. + +We sometimes also use lists of dictionaries in Calliope, e.g.: + +```yaml +key: + - option1: value1 + option2: value2 + - option3: value3 + option4: value4 +``` + +Which is equivalent in Python to `#!python {"key": [{"option1": value1, "option2": value2}, {"option3": value3, "option4": value4}]}`. + +## Calliope's additional YAML features + +To make model definition easier, we add some extra features that go beyond regular YAML formatting. + +### Abbreviated nesting Calliope allows an abbreviated form for long, nested settings: @@ -20,7 +101,7 @@ can be written as: one.two.three: x ``` -## Relative file imports +### Relative file imports Calliope also allows a special `import:` directive in any YAML file. This can specify one or several YAML files to import, e.g.: @@ -125,9 +206,160 @@ scenarios: * The imported files may include further files, so arbitrary degrees of nested configurations are possible. * The `import` statement can either give an absolute path or a path relative to the importing file. -## Overriding one file with another +### Reusing definitions through templates + +For larger models, duplicate entries can start to crop up and become cumbersome. +To streamline data entry, any section can inherit common data from a `template` which is defined in the top-level `templates` section. + +???+ example "Example 1: templates in technologies" + + If we want to set interest rate to `0.1` across all our technologies, we could define: + + ```yaml + templates: + interest_rate_setter: + cost_interest_rate: + data: 0.1 + index: monetary + dims: costs + techs: + ccgt: + template: interest_rate_setter + ... + ac_transmission: + template: interest_rate_setter + ... + ``` + +??? example "Example 2: templates in nodes" + + Similarly, if we want to allow the same technologies at all our nodes: + + ```yaml + templates: + standard_tech_list: + techs: {ccgt, battery, demand_power} # (1)! + nodes: + region1: + template: standard_tech_list + ... + region2: + template: standard_tech_list + ... + ... + region100: + template: standard_tech_list + ``` + + This YAML syntax is shortform for: + + ```yaml + nodes: + region1: + techs: + ccgt: + battery: + demand_power: + ... + ... + ``` + +??? example "Example 3: templates in data tables" + + Storing common options under the `templates` key is also useful for data tables, for example: -While generally, as stated above, if an the imported file and the current file define the same option, Calliope will raise an exception. + ```yaml + templates: + common_data_options: + rows: timesteps + columns: nodes + add_dims: + parameters: source_use_max + data_tables: + pv_data: + data: /path/to/pv_timeseries.csv + template: common_data_options + add_dims: + techs: pv + wind_data: + data: /path/to/wind_timeseries.csv + template: common_data_options + add_dims: + techs: wind + hydro_data: + data: /path/to/hydro_timeseries.csv + template: common_data_options + add_dims: + techs: hydro + ``` + +Inheritance chains can also be created. +That is, templates can inherit from other templates. + +??? example "Example 4: template inheritance chain" + + ```yaml + templates: + interest_rate_setter: + cost_interest_rate: + data: 0.1 + index: monetary + dims: costs + investment_cost_setter: + template: interest_rate_setter + cost_flow_cap: + data: 100 + index: monetary + dims: costs + cost_area_use: + data: 1 + index: monetary + dims: costs + techs: + ccgt: + template: investment_cost_setter + ... + ac_transmission: + template: interest_rate_setter + ... + ``` + +Template properties can always be overridden by the inheriting component. +That is, the 'local' value has priority over the inherited template value. +This can be useful to streamline setting costs for different technologies. + +??? example "Example 5: overriding template values" + + In this example, a technology overrides a single templated cost. + + ```yaml + templates: + interest_rate_setter: + cost_interest_rate: + data: 0.1 + index: monetary + dims: costs + investment_cost_setter: + template: interest_rate_setter + cost_interest_rate.data: 0.2 # this will replace `0.1` in the `interest_rate_setter`. + cost_flow_cap: + data: null + index: monetary + dims: costs + cost_area_use: + data: null + index: monetary + dims: costs + techs: + ccgt: + template: investment_cost_setter + cost_flow_cap.data: 100 # this will replace `null` in the `investment_cost_setter`. + ... + ``` + +### Overriding one file with another + +Generally, if an the imported file and the current file define the same option, Calliope will raise an exception. However, you can define `overrides` which you can then reference when loading your Calliope model (see [Scenarios and overrides](scenarios.md)). These `override` settings will override any data that match the same name and will add new data if it wasn't already there. @@ -167,74 +399,3 @@ Will lead to: one.four: y four.five.six: y ``` - -## Data types - -Using quotation marks (`'` or `"`) to enclose strings is optional, but can help with readability. -The three ways of setting `option` to `text` below are equivalent: - -```yaml -option: "text" -option: 'text' -option: text -``` - -Without quotations, the following values in YAML will be converted to different Python types: - -- Any unquoted number will be interpreted as numeric. -- `true` or `false` values will be interpreted as boolean. -- `.inf` and `.nan` values will be interpreted as the float values `np.inf` (infinite) and `np.nan` (not a number), respectively. -- `null` values will interpreted as `None`. - -## Comments - -Comments can be inserted anywhere in YAML files with the `#` symbol. -The remainder of a line after `#` is interpreted as a comment. -Therefore, if you have a string with a `#` in it, make sure to use explicit quotation marks. - - -## Lists and dictionaries - -Lists in YAML can be of the form `[...]` or a series of lines starting with `-`. -These two lists are equivalent: - -```yaml -key: [option1, option2] -``` - -```yaml -key: - - option1 - - option2 -``` - -Dictionaries can be of the form `{...}` or a series of lines _without_ a starting `-`. -These two dictionaries are equivalent: - -```yaml -key: {option1: value1, option2: value2} -``` - -```yaml -key: - option1: value1 - option2: value2 -``` - -To continue dictionary nesting, you can add more `{}` parentheses or you can indent your lines further. -We prefer to use 2 spaces for indenting as this makes the nested data structures more readable than the often-used 4 spaces. - -We sometimes also use lists of dictionaries in Calliope, e.g.: - -```yaml -key: - - option1: value1 - option2: value2 - - option3: value3 - option4: value4 -``` - -Which is equivalent in Python to `#!python {"key": [{"option1": value1, "option2": value2}, {"option3": value3, "option4": value4}]}`. - -!!! info "See also" - See the [YAML website](https://yaml.org/) for more general information about YAML. diff --git a/docs/examples/national_scale/index.md b/docs/examples/national_scale/index.md index dc4bfe28..0b820b7f 100644 --- a/docs/examples/national_scale/index.md +++ b/docs/examples/national_scale/index.md @@ -201,7 +201,7 @@ As the name suggests, it applies no cost or efficiency losses to this transmissi We can see that those technologies which rely on `free_transmission` inherit a lot of this information from elsewhere in the model definition. `free_transmission` is defined in `templates`, which makes it inheritable. -[Templates](../../creating/templates.md) allow us to avoid excessive repetition in our model definition. +[Templates](../../creating/yaml.md#reusing-definitions-through-templates) allow us to avoid excessive repetition in our model definition. Technologies and nodes can inherit from anything defined in `templates`. items in `templates` can also inherit from each other, so you can create inheritance chains. diff --git a/docs/examples/urban_scale/index.md b/docs/examples/urban_scale/index.md index 241385ac..f0c54898 100644 --- a/docs/examples/urban_scale/index.md +++ b/docs/examples/urban_scale/index.md @@ -226,7 +226,7 @@ Gas is made available in each node without consideration of transmission. --8<-- "src/calliope/example_models/urban_scale/model_config/techs.yaml:transmission" ``` -To avoid excessive duplication in model definition, our transmission technologies inherit most of the their parameters from [templates](../../creating/templates.md): +To avoid excessive duplication in model definition, our transmission technologies inherit most of the their parameters from [templates](../../creating/yaml.md#reusing-definitions-through-templates): ```yaml --8<-- "src/calliope/example_models/urban_scale/model_config/techs.yaml:transmission-templates" diff --git a/docs/migrating.md b/docs/migrating.md index f361358d..930bb7af 100644 --- a/docs/migrating.md +++ b/docs/migrating.md @@ -309,7 +309,7 @@ Instead, links are defined as separate transmission technologies in `techs`, inc ``` !!! note - You can use [`templates`](creating/templates.md) to minimise duplications in the new transmission technology definition. + You can use [`templates`](creating/yaml.md#reusing-definitions-through-templates) to minimise duplications in the new transmission technology definition. ### Renaming parameters/decision variables without core changes in function @@ -746,8 +746,7 @@ This means you could define different output carriers for a `supply` technology, ### `templates` for nodes -The new [`templates` key](creating/templates.md) can be applied to `nodes` as well as `techs`. -This makes up for the [removal of grouping node names in keys by comma separation](#comma-separated-node-definitions). +The new [`templates` key](creating/yaml.md#reusing-definitions-through-templates) makes up for the [removal of grouping node names in keys by comma separation](#comma-separated-node-definitions). So, to achieve this result: diff --git a/docs/user_defined_math/helper_functions.md b/docs/user_defined_math/helper_functions.md index 8b08dad8..9a4c7b59 100644 --- a/docs/user_defined_math/helper_functions.md +++ b/docs/user_defined_math/helper_functions.md @@ -8,7 +8,7 @@ Helper functions generally require a good understanding of their functionality, ## inheritance -Using `inheritance(...)` in a `where` string allows you to grab a subset of technologies / nodes that all share the same [`template`](../creating/templates.md) in the technology's / node's `template` key. +Using `inheritance(...)` in a `where` string allows you to grab a subset of technologies / nodes that all share the same [`template`](../creating/yaml.md#reusing-definitions-through-templates) in the technology's / node's `template` key. If a `template` also inherits from another `template` (chained inheritance), you will get all `techs`/`nodes` that are children along that inheritance chain. So, for the definition: diff --git a/mkdocs.yml b/mkdocs.yml index 6b41e180..2db9eee6 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -104,7 +104,6 @@ nav: - Model configuration: creating/config.md - Technologies: creating/techs.md - Nodes: creating/nodes.md - - Inheriting from templates: creating/templates.md - Indexed parameters: creating/parameters.md - Loading data tables: creating/data_tables.md - Scenarios and overrides: creating/scenarios.md From 3af68b781647133f9a27b70da9bc0a4fe66ab41b Mon Sep 17 00:00:00 2001 From: Ivan Ruiz Manuel <72193617+irm-codebase@users.noreply.github.com> Date: Mon, 2 Dec 2024 15:43:02 +0100 Subject: [PATCH 08/13] YAML title shortening --- docs/creating/yaml.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/creating/yaml.md b/docs/creating/yaml.md index 4ede042f..de68d86c 100644 --- a/docs/creating/yaml.md +++ b/docs/creating/yaml.md @@ -1,4 +1,4 @@ -# Brief introduction to YAML as used in Calliope +# YAML as used in Calliope All model configuration/definition files (with the exception of tabular data files) are in the YAML format, "a human friendly data serialisation standard for all programming languages". @@ -359,7 +359,7 @@ This can be useful to streamline setting costs for different technologies. ### Overriding one file with another -Generally, if an the imported file and the current file define the same option, Calliope will raise an exception. +Generally, if the imported file and the current file define the same option, Calliope will raise an exception. However, you can define `overrides` which you can then reference when loading your Calliope model (see [Scenarios and overrides](scenarios.md)). These `override` settings will override any data that match the same name and will add new data if it wasn't already there. From 9f978d9b6d1085b0a715c684d06acc3761db71f3 Mon Sep 17 00:00:00 2001 From: Ivan Ruiz Manuel <72193617+irm-codebase@users.noreply.github.com> Date: Mon, 2 Dec 2024 16:19:51 +0100 Subject: [PATCH 09/13] Remove references to the 'inheritance' helper function from the documentation --- docs/migrating.md | 5 ++--- docs/user_defined_math/helper_functions.md | 23 ---------------------- docs/user_defined_math/syntax.md | 10 ++++------ 3 files changed, 6 insertions(+), 32 deletions(-) diff --git a/docs/migrating.md b/docs/migrating.md index 930bb7af..8db9b419 100644 --- a/docs/migrating.md +++ b/docs/migrating.md @@ -181,10 +181,9 @@ This split means you can change configuration options on-the-fly if you are work `locations` (abbreviated to `locs` in the Calliope data dimensions) has been renamed to `nodes` (no abbreviation). This allows us to not require an abbreviation and is a disambiguation from the [pandas.DataFrame.loc][] and [xarray.DataArray.loc][] methods. -### `parent` → `base_tech` + `template` +### `parent` → `base_tech` -Technology inheritance has been unlinked from its abstract "base" technology. -`template` allows for inheriting attributes from `templates` while `base_tech` is fixed to be one of [`demand`, `supply`, `conversion`, `transmission`, `storage`]. +Technology inheritance has been renamed to `base_tech`, which is fixed to be one of [`demand`, `supply`, `conversion`, `transmission`, `storage`]. === "v0.6" diff --git a/docs/user_defined_math/helper_functions.md b/docs/user_defined_math/helper_functions.md index 9a4c7b59..3d4eb4fc 100644 --- a/docs/user_defined_math/helper_functions.md +++ b/docs/user_defined_math/helper_functions.md @@ -6,29 +6,6 @@ Their functionality is detailed in the [helper function API page](../reference/a Here, we give a brief summary. Helper functions generally require a good understanding of their functionality, so make sure you are comfortable with them beforehand. -## inheritance - -Using `inheritance(...)` in a `where` string allows you to grab a subset of technologies / nodes that all share the same [`template`](../creating/yaml.md#reusing-definitions-through-templates) in the technology's / node's `template` key. -If a `template` also inherits from another `template` (chained inheritance), you will get all `techs`/`nodes` that are children along that inheritance chain. - -So, for the definition: - -```yaml -templates: - techgroup1: - template: techgroup2 - flow_cap_max: 10 - techgroup2: - base_tech: supply -techs: - tech1: - template: techgroup1 - tech2: - template: techgroup2 -``` - -`inheritance(techgroup1)` will give the `[tech1]` subset and `inheritance(techgroup2)` will give the `[tech1, tech2]` subset. - ## any Parameters are indexed over multiple dimensions. diff --git a/docs/user_defined_math/syntax.md b/docs/user_defined_math/syntax.md index 955c4ac6..ec29a047 100644 --- a/docs/user_defined_math/syntax.md +++ b/docs/user_defined_math/syntax.md @@ -52,13 +52,11 @@ Configuration options are any that are defined in `config.build`, where you can 1. `get_val_at_index` is a [helper function](helper_functions.md#get_val_at_index)! -1. Checking the `base_tech` of a technology (`storage`, `supply`, etc.) or its inheritance chain (if using `templates` and the `template` parameter). +1. Checking the `base_tech` of a technology (`storage`, `supply`, etc.). ??? example "Examples" - If you want to create a decision variable across only `storage` technologies, you would include `base_tech=storage`. - - If you want to apply a constraint across only your own `rooftop_supply` technologies (e.g., you have defined `rooftop_supply` in `templates` and your technologies `pv` and `solar_thermal` define `#!yaml template: rooftop_supply`), you would include `inheritance(rooftop_supply)`. - Note that `base_tech=...` is a simple check for the given value of `base_tech`, while `inheritance()` is a [helper function](helper_functions.md) which can deal with finding techs/nodes using the same template, e.g. `pv` might inherit the `rooftop_supply` template which in turn might inherit the template `electricity_supply`. 1. Subsetting a set. The sets available to subset are always [`nodes`, `techs`, `carriers`] + any additional sets defined by you in [`foreach`](#foreach-lists). @@ -183,13 +181,13 @@ equations: - expression: flow_out <= $adjusted_flow_in sub_expressions: adjusted_flow_in: - - where: inheritance(storage) + - where: base_tech=storage # main expression becomes `flow_out <= flow_in * flow_eff` expression: flow_in * flow_eff - - where: inheritance(supply) + - where: base_tech=supply # main expression becomes `flow_out <= flow_in * flow_eff * parasitic_eff` expression: flow_in * flow_eff * parasitic_eff - - where: inheritance(conversion) + - where: base_tech=conversion # main expression becomes `flow_out <= flow_in * flow_eff * 0.3` expression: flow_in * flow_eff * 0.3 ``` From 3ba48a2c9001817fa6c62c4da268a6a0a46c887c Mon Sep 17 00:00:00 2001 From: Ivan Ruiz Manuel <72193617+irm-codebase@users.noreply.github.com> Date: Tue, 3 Dec 2024 09:48:25 +0100 Subject: [PATCH 10/13] Remove unused load function in attrdict --- src/calliope/attrdict.py | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/src/calliope/attrdict.py b/src/calliope/attrdict.py index 3c171090..02e2ad28 100644 --- a/src/calliope/attrdict.py +++ b/src/calliope/attrdict.py @@ -21,29 +21,6 @@ def __nonzero__(self): _MISSING = __Missing() -def _yaml_load(src): - """Load YAML from a file object or path with useful parser errors.""" - yaml = ruamel_yaml.YAML(typ="safe") - if not isinstance(src, str): - try: - src_name = src.name - except AttributeError: - src_name = "" - # Force-load file streams as that allows the parser to print - # much more context when it encounters an error - src = src.read() - else: - src_name = "" - try: - result = yaml.load(src) - if not isinstance(result, dict): - raise ValueError(f"Could not parse {src_name} as YAML") - return result - except ruamel_yaml.YAMLError: - logger.error(f"Parser error when reading YAML from {src_name}.") - raise - - class AttrDict(dict): """Extended `dict` class.""" From f9a377e3ebe5a39e958d38f7c266f48935831e2f Mon Sep 17 00:00:00 2001 From: Ivan Ruiz Manuel <72193617+irm-codebase@users.noreply.github.com> Date: Thu, 5 Dec 2024 13:45:59 +0100 Subject: [PATCH 11/13] Move to_yaml from AttrDict to io --- src/calliope/attrdict.py | 38 ---- src/calliope/backend/backend_model.py | 4 +- src/calliope/cli.py | 4 +- src/calliope/io.py | 44 +++- src/calliope/preprocess/model_definition.py | 4 +- tests/test_core_attrdict.py | 4 - tests/test_io.py | 233 +++++++++++--------- tests/test_preprocess_model_math.py | 4 +- 8 files changed, 182 insertions(+), 153 deletions(-) diff --git a/src/calliope/attrdict.py b/src/calliope/attrdict.py index 02e2ad28..b3262f83 100644 --- a/src/calliope/attrdict.py +++ b/src/calliope/attrdict.py @@ -3,11 +3,8 @@ """AttrDict implementation (a subclass of regular dict) used for managing model configuration.""" import copy -import io import logging -import numpy as np -import ruamel.yaml as ruamel_yaml from typing_extensions import Self logger = logging.getLogger(__name__) @@ -187,41 +184,6 @@ def as_dict_flat(self): d[k] = self.get_key(k) return d - def to_yaml(self, path=None): - """Conversion to YAML. - - Saves the AttrDict to the ``path`` as a YAML file or returns a YAML string - if ``path`` is None. - """ - result = self.copy() - yaml_ = ruamel_yaml.YAML() - yaml_.indent = 2 - yaml_.block_seq_indent = 0 - yaml_.sort_base_mapping_type_on_output = False - - # Numpy objects should be converted to regular Python objects, - # so that they are properly displayed in the resulting YAML output - for k in result.keys_nested(): - # Convert numpy numbers to regular python ones - v = result.get_key(k) - if isinstance(v, np.floating): - result.set_key(k, float(v)) - elif isinstance(v, np.integer): - result.set_key(k, int(v)) - # Lists are turned into seqs so that they are formatted nicely - elif isinstance(v, list): - result.set_key(k, yaml_.seq(v)) - - result = result.as_dict() - - if path is not None: - with open(path, "w") as f: - yaml_.dump(result, f) - else: - stream = io.StringIO() - yaml_.dump(result, stream) - return stream.getvalue() - def keys_nested(self, subkeys_as="list"): """Returns all keys in the AttrDict, including nested keys. diff --git a/src/calliope/backend/backend_model.py b/src/calliope/backend/backend_model.py index c52d74ab..f8431513 100644 --- a/src/calliope/backend/backend_model.py +++ b/src/calliope/backend/backend_model.py @@ -30,7 +30,7 @@ from calliope.attrdict import AttrDict from calliope.backend import helper_functions, parsing from calliope.exceptions import warn as model_warn -from calliope.io import load_config +from calliope.io import load_config, to_yaml from calliope.preprocess.model_math import ORDERED_COMPONENTS_T, CalliopeMath from calliope.util.schema import ( MODEL_SCHEMA, @@ -449,7 +449,7 @@ def _add_to_dataset( yaml_snippet_attrs[attr] = val if yaml_snippet_attrs: - add_attrs["yaml_snippet"] = AttrDict(yaml_snippet_attrs).to_yaml() + add_attrs["yaml_snippet"] = to_yaml(yaml_snippet_attrs) da.attrs = { "obj_type": obj_type, diff --git a/src/calliope/cli.py b/src/calliope/cli.py index a9d811d2..ae347dbf 100644 --- a/src/calliope/cli.py +++ b/src/calliope/cli.py @@ -13,7 +13,7 @@ import click -from calliope import AttrDict, Model, examples, read_netcdf +from calliope import Model, examples, io, read_netcdf from calliope._version import __version__ from calliope.exceptions import BackendError from calliope.util.generate_runs import generate @@ -400,4 +400,4 @@ def generate_scenarios( } } - AttrDict(scenarios).to_yaml(out_file) + io.to_yaml(scenarios, path=out_file) diff --git a/src/calliope/io.py b/src/calliope/io.py index 275ed98d..b3cb5ad1 100644 --- a/src/calliope/io.py +++ b/src/calliope/io.py @@ -6,6 +6,7 @@ import logging import os from copy import deepcopy +from io import StringIO from pathlib import Path # We import netCDF4 before xarray to mitigate a numpy warning: @@ -23,6 +24,8 @@ logger = logging.getLogger(__name__) CONFIG_DIR = importlib.resources.files("calliope") / "config" +YAML_INDENT = 2 +YAML_BLOCK_SEQUENCE_INDENT = 0 def read_netcdf(path): @@ -75,7 +78,7 @@ def _serialise(attrs: dict) -> None: dict_attrs = [k for k, v in attrs.items() if isinstance(v, dict)] attrs["serialised_dicts"] = dict_attrs for attr in dict_attrs: - attrs[attr] = AttrDict(attrs[attr]).to_yaml() + attrs[attr] = to_yaml(attrs[attr]) # Convert boolean attrs to ints bool_attrs = [k for k, v in attrs.items() if isinstance(v, bool)] @@ -287,3 +290,42 @@ def _resolve_yaml_imports( loaded_dict.del_key("import") return loaded_dict + + +def to_yaml(data: AttrDict | dict, path: None | str | Path = None) -> str: + """Conversion to YAML. + + Saves the AttrDict to the ``path`` as a YAML file or returns a YAML string + if ``path`` is None. + """ + result = AttrDict(data).copy() + # Prepare YAML parsing settings + yaml_ = ruamel_yaml.YAML() + yaml_.indent = YAML_INDENT + yaml_.block_seq_indent = YAML_BLOCK_SEQUENCE_INDENT + yaml_.sort_base_mapping_type_on_output = ( + False # FIXME: identify if this is necessary + ) + + # Numpy objects should be converted to regular Python objects, + # so that they are properly displayed in the resulting YAML output + for k in result.keys_nested(): + # Convert numpy numbers to regular python ones + v = result.get_key(k) + if isinstance(v, np.floating): + result.set_key(k, float(v)) + elif isinstance(v, np.integer): + result.set_key(k, int(v)) + # Lists are turned into seqs so that they are formatted nicely + elif isinstance(v, list): + result.set_key(k, yaml_.seq(v)) + + result = result.as_dict() + + if path is not None: + with open(path, "w") as f: + yaml_.dump(result, f) + + stream = StringIO() + yaml_.dump(result, stream) + return stream.getvalue() diff --git a/src/calliope/preprocess/model_definition.py b/src/calliope/preprocess/model_definition.py index b6a32ae3..5e5dd16d 100644 --- a/src/calliope/preprocess/model_definition.py +++ b/src/calliope/preprocess/model_definition.py @@ -7,7 +7,7 @@ from calliope import exceptions from calliope.attrdict import AttrDict -from calliope.io import read_rich_yaml +from calliope.io import read_rich_yaml, to_yaml from calliope.util.tools import listify LOGGER = logging.getLogger(__name__) @@ -129,7 +129,7 @@ def _combine_overrides(overrides: AttrDict, scenario_overrides: list): combined_override_dict = AttrDict() for override in scenario_overrides: try: - yaml_string = overrides[override].to_yaml() + yaml_string = to_yaml(overrides[override]) override_with_imports = read_rich_yaml(yaml_string) except KeyError: raise exceptions.ModelError(f"Override `{override}` is not defined.") diff --git a/tests/test_core_attrdict.py b/tests/test_core_attrdict.py index 340e72e2..cdbae0a9 100644 --- a/tests/test_core_attrdict.py +++ b/tests/test_core_attrdict.py @@ -206,7 +206,3 @@ def test_del_key_single(self, attr_dict): def test_del_key_nested(self, attr_dict): attr_dict.del_key("c.z.I") assert "I" not in attr_dict.c.z - - def test_to_yaml_string(self, attr_dict): - result = attr_dict.to_yaml() - assert "a: 1" in result diff --git a/tests/test_io.py b/tests/test_io.py index c4db1d2b..eb3ee69c 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -240,95 +240,134 @@ def test_save_per_spore(self): class TestYaml: - @pytest.fixture - def dummy_yaml_import(self): - return """ - import: ['somefile.yaml'] - """ - - def test_do_not_resolve_imports(self, dummy_yaml_import): - """Text inputs that attempt to import files should raise an error.""" + TEST_TEXT = { + "simple_nested": """ +somekey.nested: 1 +anotherkey: 2 +""", + "triple_nested": """ +foo: + bar: 1 + baz: 2 + nested: + value: 5 +""", + "complex_commented": """ +# a comment +a: 1 +b: 2 +# a comment about `c` +c: # a comment inline with `c` + x: foo # a comment on foo + + # + y: bar # + z: + I: 1 + II: 2 +d: +""", + "nested_string": "a.b.c: 1\na.b.foo: 2\nb.a.c.bar: foo", + } + + TEST_EXPECTED = { + "simple_nested": {"somekey": {"nested": 1}, "anotherkey": 2}, + "triple_nested": {"foo": {"bar": 1, "baz": 2, "nested": {"value": 5}}}, + "complex_commented": { + "a": 1, + "b": 2, + "c": {"x": "foo", "y": "bar", "z": {"I": 1, "II": 2}}, + "d": None, + }, + "nested_string": { + "a": {"b": {"c": 1, "foo": 2}}, + "b": {"a": {"c": {"bar": "foo"}}}, + }, + } + + @pytest.fixture( + params=["simple_nested", "triple_nested", "complex_commented", "nested_string"] + ) + def test_group(self, request) -> str: + return request.param - with pytest.raises(ValueError) as exinfo: # noqa: PT011, false positive - calliope.io.read_rich_yaml(dummy_yaml_import) + @pytest.fixture + def yaml_text(self, test_group) -> str: + return self.TEST_TEXT[test_group] - assert check_error_or_warning( - exinfo, "Imports are not possible for non-file yaml inputs." - ) + @pytest.fixture + def expected_dict(self, test_group) -> dict: + return self.TEST_EXPECTED[test_group] @pytest.fixture def dummy_imported_file(self, tmp_path) -> Path: file = tmp_path / "test_import.yaml" text = """ - somekey.nested: 1 - anotherkey: 2 +# Comment +import_key_a.nested: 1 +import_key_b: 2 +import_key_c: [1, 2, 3] """ with open(file, "w") as f: f.write(text) return file - def test_import(self, dummy_imported_file): - file = dummy_imported_file.parent / "main_file.yaml" - text = """ - import: - - test_import.yaml - foo: - bar: 1 - baz: 2 - 3: - 4: 5 - """ + def test_text_read(self, yaml_text, expected_dict): + """Loading from text strings should be correct.""" + read = calliope.io.read_rich_yaml(yaml_text) + assert read == expected_dict + + def test_file_read(self, test_group, yaml_text, expected_dict, tmp_path): + """Loading from files should be correct.""" + file = tmp_path / f"{test_group}.yaml" with open(file, "w") as f: - f.write(text) + f.write(yaml_text) + read = calliope.io.read_rich_yaml(file) + assert read == expected_dict + + @pytest.mark.parametrize( + "bad_import", + [ + "import: ['somefile.yaml']\n", + "import: ['somefile.yaml', 'other_file.yaml']\n", + ], + ) + def test_text_import_error(self, yaml_text, bad_import): + """Text inputs that attempt to import files should raise an error.""" + with pytest.raises( + ValueError, match="Imports are not possible for non-file yaml inputs." + ): + calliope.io.read_rich_yaml(bad_import + yaml_text) + + def test_import(self, test_group, yaml_text, dummy_imported_file): + """Imported files relative to the main file should load correctly.""" + file = dummy_imported_file.parent / f"{test_group}_relative.yaml" + import_text = f""" +import: + - {dummy_imported_file.name} +""" + with open(file, "w") as f: + f.write(import_text + yaml_text) d = calliope.io.read_rich_yaml(file) - assert "somekey.nested" in d.keys_nested() - assert d.get_key("anotherkey") == 2 + assert "import_key_a.nested" in d.keys_nested() + assert d.get_key("import_key_b") == 2 + assert d["import_key_c"] == [1, 2, 3] - def test_import_must_be_list(self, tmp_path): - file = tmp_path / "non_list_import.yaml" - text = """ - import: test_import.yaml - foo: - bar: 1 - baz: 2 - 3: - 4: 5 - """ + def test_invalid_import_type_error( + self, test_group, yaml_text, dummy_imported_file + ): + file = dummy_imported_file.parent / f"{test_group}_invalid_import_type.yaml" + import_text = f"""import: {dummy_imported_file.name}\n""" with open(file, "w") as f: - f.write(text) + f.write(import_text + yaml_text) with pytest.raises(ValueError) as excinfo: # noqa: PT011, false positive calliope.io.read_rich_yaml(file) assert check_error_or_warning(excinfo, "`import` must be a list.") - def test_from_yaml_string(self): - yaml_string = """ - # a comment - a: 1 - b: 2 - # a comment about `c` - c: # a comment inline with `c` - x: foo # a comment on foo - - # - y: bar # - z: - I: 1 - II: 2 - d: - """ - d = calliope.io.read_rich_yaml(yaml_string) - assert d.a == 1 - assert d.c.z.II == 2 - - def test_from_yaml_string_dot_strings(self): - yaml_string = "a.b.c: 1\na.b.foo: 2" - d = calliope.io.read_rich_yaml(yaml_string) - assert d.a.b.c == 1 - assert d.a.b.foo == 2 - - def test_from_yaml_string_dot_strings_duplicate(self): + def test_duplicate_dot_string_error(self): + """Duplicate entries should result in an error.""" yaml_string = "a.b.c: 1\na.b.c: 2" with pytest.raises(ruamel_yaml.constructor.DuplicateKeyError): calliope.io.read_rich_yaml(yaml_string) @@ -351,9 +390,15 @@ def test_parser_error(self): """ ) - @pytest.fixture - def multi_order_yaml(self): - return calliope.io.read_rich_yaml( + def test_as_dict_with_sublists(self): + """Lists should not be converted to AttrDict.""" + d = calliope.io.read_rich_yaml("a: [{x: 1}, {y: 2}]") + dd = d.as_dict() + assert dd["a"][0]["x"] == 1 + assert all([isinstance(dd["a"][0], dict), not isinstance(dd["a"][0], AttrDict)]) + + def test_replacement_null_from_file(self): + yaml_dict = calliope.io.read_rich_yaml( """ A.B.C: 10 A.B: @@ -361,41 +406,25 @@ def multi_order_yaml(self): C: "foobar" """ ) - - def test_order_of_subdicts(self, multi_order_yaml): - assert multi_order_yaml.A.B.C == 10 - assert multi_order_yaml.A.B.E == 20 - assert multi_order_yaml.C == "foobar" - - def test_as_dict_with_sublists(self): - d = calliope.io.read_rich_yaml("a: [{x: 1}, {y: 2}]") - dd = d.as_dict() - assert dd["a"][0]["x"] == 1 - assert all( - [isinstance(dd["a"][0], dict), not isinstance(dd["a"][0], AttrDict)] - ) # Not AttrDict! - - def test_replacement_null_from_file(self, multi_order_yaml): replacement = calliope.io.read_rich_yaml("C._REPLACE_: null") - multi_order_yaml.union(replacement, allow_override=True, allow_replacement=True) - assert multi_order_yaml.C is None - - @pytest.fixture - def yaml_from_path(self): - this_path = Path(__file__).parent - return calliope.io.read_rich_yaml(this_path / "common" / "yaml_file.yaml") - - def test_from_yaml_path(self, yaml_from_path): - assert yaml_from_path.a == 1 - assert yaml_from_path.c.z.II == 2 - - def test_to_yaml(self, yaml_from_path): - yaml_from_path.set_key("numpy.some_int", np.int32(10)) - yaml_from_path.set_key("numpy.some_float", np.float64(0.5)) - yaml_from_path.a_list = [0, 1, 2] + yaml_dict.union(replacement, allow_override=True, allow_replacement=True) + assert yaml_dict.C is None + + def test_to_yaml_roundtrip(self, expected_dict): + """Saving to a file should result in no data loss.""" + yaml_text = calliope.io.to_yaml(expected_dict) + reloaded = calliope.io.read_rich_yaml(yaml_text) + assert reloaded == expected_dict + + def test_to_yaml_complex(self, yaml_text): + """Saving to a file/string should handle special cases.""" + yaml_dict = calliope.io.read_rich_yaml(yaml_text) + yaml_dict.set_key("numpy.some_int", np.int32(10)) + yaml_dict.set_key("numpy.some_float", np.float64(0.5)) + yaml_dict.a_list = [0, 1, 2] with tempfile.TemporaryDirectory() as tempdir: out_file = os.path.join(tempdir, "test.yaml") - yaml_from_path.to_yaml(out_file) + calliope.io.to_yaml(yaml_dict, path=out_file) with open(out_file) as f: result = f.read() diff --git a/tests/test_preprocess_model_math.py b/tests/test_preprocess_model_math.py index c62ee4d7..86c2c24f 100644 --- a/tests/test_preprocess_model_math.py +++ b/tests/test_preprocess_model_math.py @@ -9,7 +9,7 @@ import calliope from calliope.exceptions import ModelError -from calliope.io import read_rich_yaml +from calliope.io import read_rich_yaml, to_yaml from calliope.preprocess import CalliopeMath @@ -37,7 +37,7 @@ def user_math(dummy_int): @pytest.fixture(scope="module") def user_math_path(def_path, user_math): file_path = def_path / "custom-math.yaml" - user_math.to_yaml(def_path / file_path) + to_yaml(user_math, path=def_path / file_path) return "custom-math.yaml" From 0253194daee02833fd021d6adae36bb8b1e9d8e6 Mon Sep 17 00:00:00 2001 From: Ivan Ruiz Manuel <72193617+irm-codebase@users.noreply.github.com> Date: Thu, 5 Dec 2024 16:46:03 +0100 Subject: [PATCH 12/13] PR review improvements. Fix random data table information loss bug. --- docs/creating/yaml.md | 353 +++++++++++++------- src/calliope/preprocess/model_definition.py | 5 +- tests/test_io.py | 17 +- 3 files changed, 242 insertions(+), 133 deletions(-) diff --git a/docs/creating/yaml.md b/docs/creating/yaml.md index de68d86c..45d74f58 100644 --- a/docs/creating/yaml.md +++ b/docs/creating/yaml.md @@ -215,147 +215,260 @@ To streamline data entry, any section can inherit common data from a `template` If we want to set interest rate to `0.1` across all our technologies, we could define: - ```yaml - templates: - interest_rate_setter: - cost_interest_rate: - data: 0.1 - index: monetary - dims: costs - techs: - ccgt: - template: interest_rate_setter - ... - ac_transmission: - template: interest_rate_setter - ... - ``` + === "Using templates" + + ```yaml + templates: + interest_rate_setter: + cost_interest_rate: + data: 0.1 + index: monetary + dims: costs + techs: + ccgt: + flow_out_eff: 0.5 + template: interest_rate_setter + ac_transmission: + flow_out_eff: 0.98 + template: interest_rate_setter + ``` + === "Without templates" + + ```yaml + techs: + ccgt: + flow_out_eff: 0.5 + cost_interest_rate: + data: 0.1 + index: monetary + dims: costs + ac_transmission: + flow_out_eff: 0.98 + cost_interest_rate: + data: 0.1 + index: monetary + dims: costs + ``` ??? example "Example 2: templates in nodes" Similarly, if we want to allow the same technologies at all our nodes: - ```yaml - templates: - standard_tech_list: - techs: {ccgt, battery, demand_power} # (1)! - nodes: - region1: - template: standard_tech_list - ... - region2: - template: standard_tech_list - ... - ... - region100: - template: standard_tech_list - ``` - - This YAML syntax is shortform for: - - ```yaml - nodes: - region1: - techs: - ccgt: - battery: - demand_power: - ... - ... - ``` + === "Using templates" + + ```yaml + templates: + standard_tech_list: + techs: {ccgt, battery, demand_power} + nodes: + region1: + template: standard_tech_list + latitude: 39 + longitude: -2 + region2: + template: standard_tech_list + latitude: 40 + longitude: 0 + ``` + + === "Without templates" + + ```yaml + nodes: + region1: + techs: + ccgt: + battery: + demand_power: + latitude: 39 + longitude: -2 + region2: + techs: + ccgt: + battery: + demand_power: + latitude: 40 + longitude: 0 + ``` ??? example "Example 3: templates in data tables" - Storing common options under the `templates` key is also useful for data tables, for example: - - ```yaml - templates: - common_data_options: - rows: timesteps - columns: nodes - add_dims: - parameters: source_use_max - data_tables: - pv_data: - data: /path/to/pv_timeseries.csv - template: common_data_options - add_dims: - techs: pv - wind_data: - data: /path/to/wind_timeseries.csv - template: common_data_options - add_dims: - techs: wind - hydro_data: - data: /path/to/hydro_timeseries.csv - template: common_data_options - add_dims: - techs: hydro - ``` + Storing common options under the `templates` key is also useful for data tables. + + === "Using templates" + + ```yaml + templates: + common_data_options: + rows: timesteps + columns: nodes + add_dims: + parameters: source_use_max + data_tables: + pv_data: + data: /path/to/pv_timeseries.csv + template: common_data_options + add_dims: + techs: pv + wind_data: + data: /path/to/wind_timeseries.csv + template: common_data_options + add_dims: + techs: wind + hydro_data: + data: /path/to/hydro_timeseries.csv + template: common_data_options + add_dims: + techs: hydro + ``` + === "Without templates" + + ```yaml + data_tables: + pv_data: + data: /path/to/pv_timeseries.csv + rows: timesteps + columns: nodes + add_dims: + parameters: source_use_max + techs: pv + wind_data: + data: /path/to/wind_timeseries.csv + rows: timesteps + columns: nodes + add_dims: + parameters: source_use_max + techs: wind + hydro_data: + data: /path/to/hydro_timeseries.csv + rows: timesteps + columns: nodes + add_dims: + parameters: source_use_max + techs: hydro + ``` Inheritance chains can also be created. That is, templates can inherit from other templates. ??? example "Example 4: template inheritance chain" - ```yaml - templates: - interest_rate_setter: - cost_interest_rate: - data: 0.1 - index: monetary - dims: costs - investment_cost_setter: - template: interest_rate_setter - cost_flow_cap: - data: 100 - index: monetary - dims: costs - cost_area_use: - data: 1 - index: monetary - dims: costs - techs: - ccgt: - template: investment_cost_setter - ... - ac_transmission: - template: interest_rate_setter - ... - ``` + A two-level template inheritance chain. + + === "Using templates" + + ```yaml + templates: + interest_rate_setter: + cost_interest_rate: + data: 0.1 + index: monetary + dims: costs + investment_cost_setter: + template: interest_rate_setter + cost_flow_cap: + data: 100 + index: monetary + dims: costs + cost_area_use: + data: 1 + index: monetary + dims: costs + techs: + ccgt: + template: investment_cost_setter + flow_out_eff: 0.5 + ac_transmission: + template: interest_rate_setter + flow_out_eff: 0.98 + ``` + + === "Without templates" -Template properties can always be overridden by the inheriting component. -That is, the 'local' value has priority over the inherited template value. + ```yaml + techs: + ccgt: + cost_interest_rate: + data: 0.1 + index: monetary + dims: costs + cost_flow_cap: + data: 100 + index: monetary + dims: costs + cost_area_use: + data: 1 + index: monetary + dims: costs + flow_out_eff: 0.5 + ac_transmission: + cost_interest_rate: + data: 0.1 + index: monetary + dims: costs + cost_flow_cap: + data: 100 + index: monetary + dims: costs + cost_area_use: + data: 1 + index: monetary + dims: costs + flow_out_eff: 0.98 + ``` + +Template properties can always be overwritten by the inheriting component. +That is, a 'local' value has priority over the template value. This can be useful to streamline setting costs for different technologies. ??? example "Example 5: overriding template values" In this example, a technology overrides a single templated cost. - ```yaml - templates: - interest_rate_setter: - cost_interest_rate: - data: 0.1 - index: monetary - dims: costs - investment_cost_setter: - template: interest_rate_setter - cost_interest_rate.data: 0.2 # this will replace `0.1` in the `interest_rate_setter`. - cost_flow_cap: - data: null - index: monetary - dims: costs - cost_area_use: - data: null - index: monetary - dims: costs - techs: - ccgt: - template: investment_cost_setter - cost_flow_cap.data: 100 # this will replace `null` in the `investment_cost_setter`. - ... - ``` + === "Using templates" + + ```yaml + templates: + interest_rate_setter: + cost_interest_rate: + data: 0.1 + index: monetary + dims: costs + investment_cost_setter: + template: interest_rate_setter + cost_interest_rate.data: 0.2 # this will replace `0.1` in the `interest_rate_setter`. + cost_flow_cap: + data: null + index: monetary + dims: costs + cost_area_use: + data: null + index: monetary + dims: costs + techs: + ccgt: + template: investment_cost_setter + cost_flow_cap.data: 100 # this will replace `null` in the `investment_cost_setter`. + ``` + + === "Without templates" + + ```yaml + techs: + ccgt: + cost_interest_rate: + data: 0.2 + index: monetary + dims: costs + cost_flow_cap: + data: 100 + index: monetary + dims: costs + cost_area_use: + data: null + index: monetary + dims: costs + ``` ### Overriding one file with another diff --git a/src/calliope/preprocess/model_definition.py b/src/calliope/preprocess/model_definition.py index 5e5dd16d..8a7f86da 100644 --- a/src/calliope/preprocess/model_definition.py +++ b/src/calliope/preprocess/model_definition.py @@ -251,8 +251,9 @@ def _resolve_data(self, section, level: int = 0): template = AttrDict() local = AttrDict() - for key in section.keys() - {self.TEMPLATE_CALL, self.TEMPLATES_SECTION}: - local[key] = self._resolve_data(section[key], level=level + 1) + for key in section.keys(): + if key not in [self.TEMPLATE_CALL, self.TEMPLATES_SECTION]: + local[key] = self._resolve_data(section[key], level=level + 1) # Local values have priority. template.union(local, allow_override=True) diff --git a/tests/test_io.py b/tests/test_io.py index eb3ee69c..802026a1 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -308,8 +308,7 @@ def dummy_imported_file(self, tmp_path) -> Path: import_key_b: 2 import_key_c: [1, 2, 3] """ - with open(file, "w") as f: - f.write(text) + file.write_text(text) return file def test_text_read(self, yaml_text, expected_dict): @@ -320,8 +319,7 @@ def test_text_read(self, yaml_text, expected_dict): def test_file_read(self, test_group, yaml_text, expected_dict, tmp_path): """Loading from files should be correct.""" file = tmp_path / f"{test_group}.yaml" - with open(file, "w") as f: - f.write(yaml_text) + file.write_text(yaml_text) read = calliope.io.read_rich_yaml(file) assert read == expected_dict @@ -346,8 +344,7 @@ def test_import(self, test_group, yaml_text, dummy_imported_file): import: - {dummy_imported_file.name} """ - with open(file, "w") as f: - f.write(import_text + yaml_text) + file.write_text(import_text + yaml_text) d = calliope.io.read_rich_yaml(file) assert "import_key_a.nested" in d.keys_nested() @@ -359,8 +356,7 @@ def test_invalid_import_type_error( ): file = dummy_imported_file.parent / f"{test_group}_invalid_import_type.yaml" import_text = f"""import: {dummy_imported_file.name}\n""" - with open(file, "w") as f: - f.write(import_text + yaml_text) + file.write_text(import_text + yaml_text) with pytest.raises(ValueError) as excinfo: # noqa: PT011, false positive calliope.io.read_rich_yaml(file) @@ -423,11 +419,10 @@ def test_to_yaml_complex(self, yaml_text): yaml_dict.set_key("numpy.some_float", np.float64(0.5)) yaml_dict.a_list = [0, 1, 2] with tempfile.TemporaryDirectory() as tempdir: - out_file = os.path.join(tempdir, "test.yaml") + out_file = Path(tempdir) / "test.yaml" calliope.io.to_yaml(yaml_dict, path=out_file) - with open(out_file) as f: - result = f.read() + result = out_file.read_text() assert "some_int: 10" in result assert "some_float: 0.5" in result From 8d5889480a561aa99c7c5ba068fcce9f60373cd2 Mon Sep 17 00:00:00 2001 From: Ivan Ruiz Manuel <72193617+irm-codebase@users.noreply.github.com> Date: Fri, 6 Dec 2024 12:43:03 +0100 Subject: [PATCH 13/13] PR updates: migration section improvements, io remove unnecessary code --- docs/migrating.md | 6 ++++-- src/calliope/io.py | 18 ++++-------------- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/docs/migrating.md b/docs/migrating.md index 8db9b419..b704ba6b 100644 --- a/docs/migrating.md +++ b/docs/migrating.md @@ -181,9 +181,11 @@ This split means you can change configuration options on-the-fly if you are work `locations` (abbreviated to `locs` in the Calliope data dimensions) has been renamed to `nodes` (no abbreviation). This allows us to not require an abbreviation and is a disambiguation from the [pandas.DataFrame.loc][] and [xarray.DataArray.loc][] methods. -### `parent` → `base_tech` +### `parent` and `tech_groups` → `base_tech` and `templates` -Technology inheritance has been renamed to `base_tech`, which is fixed to be one of [`demand`, `supply`, `conversion`, `transmission`, `storage`]. +Technology `parent` inheritance has been renamed to `base_tech`, which is fixed to be one of [`demand`, `supply`, `conversion`, `transmission`, `storage`]. + +The `tech_groups` functionality has been removed in favour of a new, more flexible, `templates` functionality. === "v0.6" diff --git a/src/calliope/io.py b/src/calliope/io.py index b3cb5ad1..3b68fa27 100644 --- a/src/calliope/io.py +++ b/src/calliope/io.py @@ -246,19 +246,10 @@ def read_rich_yaml(yaml: str | Path, allow_override: bool = False) -> AttrDict: return yaml_dict -def _yaml_load(src): +def _yaml_load(src: str): """Load YAML from a file object or path with useful parser errors.""" yaml = ruamel_yaml.YAML(typ="safe") - if not isinstance(src, str): - try: - src_name = src.name - except AttributeError: - src_name = "" - # Force-load file streams as that allows the parser to print - # much more context when it encounters an error - src = src.read() - else: - src_name = "" + src_name = "" try: result = yaml.load(src) if not isinstance(result, dict): @@ -303,9 +294,8 @@ def to_yaml(data: AttrDict | dict, path: None | str | Path = None) -> str: yaml_ = ruamel_yaml.YAML() yaml_.indent = YAML_INDENT yaml_.block_seq_indent = YAML_BLOCK_SEQUENCE_INDENT - yaml_.sort_base_mapping_type_on_output = ( - False # FIXME: identify if this is necessary - ) + # Keep dictionary order + yaml_.sort_base_mapping_type_on_output = False # type: ignore[assignment] # Numpy objects should be converted to regular Python objects, # so that they are properly displayed in the resulting YAML output