diff --git a/CHANGELOG.md b/CHANGELOG.md index 867ed39b..2a53b948 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ### User-facing changes +|new| Data tables can inherit options from `templates`, like `techs` and `nodes` (#676). + |new| Math has been removed from `model.math`, and can now be accessed via `model.math.data` (#639). |new| (non-NaN) Default values and data types for parameters appear in math documentation (if they appear in the model definition schema) (#677). diff --git a/docs/creating/data_tables.md b/docs/creating/data_tables.md index e549be03..7e9d631b 100644 --- a/docs/creating/data_tables.md +++ b/docs/creating/data_tables.md @@ -14,9 +14,10 @@ In brief it is: * **data**: path to file or reference name for an in-memory object. * **rows**: the dimension(s) in your table defined per row. * **columns**: the dimension(s) in your table defined per column. -* **select**: values within dimensions that you want to select from your tabular data, discarding the rest. -* **drop**: dimensions to drop from your rows/columns, e.g., a "comment" row. -* **add_dims**: dimensions to add to the table after loading it in, with the corresponding value(s) to assign to the dimension index. +* [**select**](#selecting-dimension-values-and-dropping-dimensions): values within dimensions that you want to select from your tabular data, discarding the rest. +* [**drop**](#selecting-dimension-values-and-dropping-dimensions): dimensions to drop from your rows/columns, e.g., a "comment" row. +* [**add_dims**](#adding-dimensions): dimensions to add to the table after loading it in, with the corresponding value(s) to assign to the dimension index. +* [**template**](#using-a-template): Reference to a [template](templates.md) from which to inherit common configuration options. When we refer to "dimensions", we mean the sets over which data is indexed in the model: `nodes`, `techs`, `timesteps`, `carriers`, `costs`. In addition, when loading from file, there is the _required_ dimension `parameters`. @@ -418,6 +419,65 @@ Or to define the same timeseries source data for two technologies at different n parameters: source_use_max ``` +## Using a template + +Templates allow us to define common options that are inherited by several data tables. +They can be particularly useful if you are storing your data in many small tables. + +To assign the same input timeseries data for (tech1, node1) and (tech2, node2) using [`add_dims`](#adding-dimensions): + +=== "Without `template`" + + | | | + | ---------------: | :-- | + | 2005-01-01 00:00 | 100 | + | 2005-01-01 01:00 | 200 | + + ```yaml + data_tables: + tech_data_1: + data: data_tables/tech_data.csv + rows: timesteps + add_dims: + techs: tech1 + nodes: node1 + parameters: source_use_max + tech_data_2: + data: data_tables/tech_data.csv + rows: timesteps + add_dims: + techs: tech2 + nodes: node2 + parameters: source_use_max + ``` + +=== "With `template`" + + | | | + | ---------------: | :-- | + | 2005-01-01 00:00 | 100 | + | 2005-01-01 01:00 | 200 | + + ```yaml + templates: + common_data_options: + data: data_tables/tech_data.csv + rows: timesteps + add_dims: + parameters: source_use_max + data_tables: + tech_data_1: + template: common_data_options + add_dims: + techs: tech1 + nodes: node1 + tech_data_2: + template: common_data_options + add_dims: + techs: tech2 + nodes: node2 + ``` + ## Loading CSV files vs `pandas` dataframes To load from CSV, set the filepath in `data` to point to your file. diff --git a/docs/creating/templates.md b/docs/creating/templates.md index d1d970dd..c723c8c8 100644 --- a/docs/creating/templates.md +++ b/docs/creating/templates.md @@ -2,9 +2,11 @@ # Inheriting from templates: `templates` For larger models, duplicate entries can start to crop up and become cumbersome. -To streamline data entry, technologies and nodes can inherit common data from a `template`. +To streamline data entry, technologies, nodes, and data tables can inherit common data from a `template`. -For example, if we want to set interest rate to `0.1` across all our technologies, we could define: +## Templates in technologies + +If we want to set interest rate to `0.1` across all our technologies, we could define: ```yaml templates: @@ -22,6 +24,8 @@ techs: ... ``` +## Templates in nodes + Similarly, if we want to allow the same technologies at all our nodes: ```yaml @@ -48,6 +52,37 @@ nodes: demand_power: ``` +## Templates in data tables + +Data tables can also store common options under the `templates` key, for example: + +```yaml +templates: + common_data_options: + rows: timesteps + columns: nodes + add_dims: + parameters: source_use_max +data_tables: + pv_data: + data: /path/to/pv_timeseries.csv + template: common_data_options + add_dims: + techs: pv + wind_data: + data: /path/to/wind_timeseries.csv + template: common_data_options + add_dims: + techs: wind + hydro_data: + data: /path/to/hydro_timeseries.csv + template: common_data_options + add_dims: + techs: hydro +``` + +## Inheritance chains + Inheritance chains can also be created. That is, templates can inherit from other templates. E.g.: @@ -78,7 +113,9 @@ techs: ... ``` -Finally, template properties can always be overridden by the inheriting component. +## Overriding template values + +Template properties can always be overridden by the inheriting component. This can be useful to streamline setting costs, e.g.: ```yaml diff --git a/src/calliope/model.py b/src/calliope/model.py index 42f67875..ee8c5a77 100644 --- a/src/calliope/model.py +++ b/src/calliope/model.py @@ -25,7 +25,7 @@ update_then_validate_config, validate_dict, ) -from calliope.util.tools import relative_path +from calliope.util.tools import climb_template_tree, relative_path if TYPE_CHECKING: from calliope.backend.backend_model import BackendModel @@ -180,15 +180,15 @@ def _init_from_model_def_dict( "scenario": scenario, "defaults": param_metadata["default"], } - - data_tables = [ - DataTable( - init_config, source_name, source_dict, data_table_dfs, self._def_path + templates = model_definition.get("templates", AttrDict()) + data_tables: list[DataTable] = [] + for table_name, table_dict in model_definition.pop("data_tables", {}).items(): + table_dict, _ = climb_template_tree(table_dict, templates, table_name) + data_tables.append( + DataTable( + init_config, table_name, table_dict, data_table_dfs, self._def_path + ) ) - for source_name, source_dict in model_definition.pop( - "data_tables", {} - ).items() - ] model_data_factory = ModelDataFactory( init_config, model_definition, data_tables, attributes, param_metadata diff --git a/src/calliope/preprocess/data_tables.py b/src/calliope/preprocess/data_tables.py index b8151c1d..93fff27c 100644 --- a/src/calliope/preprocess/data_tables.py +++ b/src/calliope/preprocess/data_tables.py @@ -37,6 +37,7 @@ class DataTableDict(TypedDict): add_dims: NotRequired[dict[str, str | list[str]]] select: dict[str, str | bool | int] drop: Hashable | list[Hashable] + template: NotRequired[str] class DataTable: diff --git a/src/calliope/preprocess/model_data.py b/src/calliope/preprocess/model_data.py index 6de0aa3f..7c6d6cc3 100644 --- a/src/calliope/preprocess/model_data.py +++ b/src/calliope/preprocess/model_data.py @@ -17,7 +17,7 @@ from calliope.attrdict import AttrDict from calliope.preprocess import data_tables, time from calliope.util.schema import MODEL_SCHEMA, validate_dict -from calliope.util.tools import listify +from calliope.util.tools import climb_template_tree, listify LOGGER = logging.getLogger(__name__) @@ -555,10 +555,15 @@ def _inherit_defs( item_base_def = deepcopy(base_def[item_name]) item_base_def.union(item_def, allow_override=True) + if item_name in self.tech_data_from_tables: + _data_table_dict = deepcopy(self.tech_data_from_tables[item_name]) + _data_table_dict.union(item_base_def, allow_override=True) + item_base_def = _data_table_dict else: item_base_def = item_def - updated_item_def, inheritance = self._climb_template_tree( - item_base_def, dim_name, item_name + templates = self.model_definition.get("templates", AttrDict()) + updated_item_def, inheritance = climb_template_tree( + item_base_def, templates, item_name ) if not updated_item_def.get("active", True): @@ -576,60 +581,6 @@ def _inherit_defs( return updated_defs - def _climb_template_tree( - self, - dim_item_dict: AttrDict, - dim_name: Literal["nodes", "techs"], - item_name: str, - inheritance: list | None = None, - ) -> tuple[AttrDict, list | None]: - """Follow the `template` references from `nodes` / `techs` to `templates`. - - Abstract template definitions (those in `templates`) can inherit each other, but `nodes`/`techs` cannot. - - This function will be called recursively until a definition dictionary without `template` is reached. - - Args: - dim_item_dict (AttrDict): Dictionary (possibly) containing `template`. - dim_name (Literal[nodes, techs]): - The name of the dimension we're working with, so that we can access the correct `_groups` definitions. - item_name (str): - The current position in the inheritance tree. - inheritance (list | None, optional): - A list of items that have been inherited (starting with the oldest). - If the first `dim_item_dict` does not contain `template`, this will remain as None. - Defaults to None. - - Raises: - KeyError: Must inherit from a named template item in `templates`. - - Returns: - tuple[AttrDict, list | None]: Definition dictionary with inherited data and a list of the inheritance tree climbed to get there. - """ - to_inherit = dim_item_dict.get("template", None) - dim_groups = AttrDict(self.model_definition.get("templates", {})) - if to_inherit is None: - if dim_name == "techs" and item_name in self.tech_data_from_tables: - _data_table_dict = deepcopy(self.tech_data_from_tables[item_name]) - _data_table_dict.union(dim_item_dict, allow_override=True) - dim_item_dict = _data_table_dict - updated_dim_item_dict = dim_item_dict - elif to_inherit not in dim_groups: - raise KeyError( - f"({dim_name}, {item_name}) | Cannot find `{to_inherit}` in template inheritance tree." - ) - else: - base_def_dict, inheritance = self._climb_template_tree( - dim_groups[to_inherit], dim_name, to_inherit, inheritance - ) - updated_dim_item_dict = deepcopy(base_def_dict) - updated_dim_item_dict.union(dim_item_dict, allow_override=True) - if inheritance is not None: - inheritance.append(to_inherit) - else: - inheritance = [to_inherit] - return updated_dim_item_dict, inheritance - def _deactivate_item(self, **item_ref): for dim_name, item_name in item_ref.items(): if item_name not in self.dataset.coords.get(dim_name, xr.DataArray()): diff --git a/src/calliope/util/tools.py b/src/calliope/util/tools.py index 51920d88..2c0e1a2c 100644 --- a/src/calliope/util/tools.py +++ b/src/calliope/util/tools.py @@ -2,11 +2,15 @@ # Licensed under the Apache 2.0 License (see LICENSE file). """Assorted helper tools.""" +from copy import deepcopy from pathlib import Path -from typing import Any, TypeVar +from typing import TYPE_CHECKING, Any, TypeVar from typing_extensions import ParamSpec +if TYPE_CHECKING: + from calliope import AttrDict + P = ParamSpec("P") T = TypeVar("T") @@ -47,3 +51,52 @@ def listify(var: Any) -> list: else: var = [var] return var + + +def climb_template_tree( + dim_item_dict: "AttrDict", + templates: "AttrDict", + item_name: str, + inheritance: list | None = None, +) -> tuple["AttrDict", list | None]: + """Follow the `template` references from model definition elements to `templates`. + + Model definition elements can inherit template entries (those in `templates`). + Template entries can also inherit each other, to create an inheritance chain. + + This function will be called recursively until a definition dictionary without `template` is reached. + + Args: + dim_item_dict (AttrDict): Dictionary (possibly) containing `template`. + templates (AttrDict): Dictionary of available templates. + item_name (str): + The current position in the inheritance tree. + inheritance (list | None, optional): + A list of items that have been inherited (starting with the oldest). + If the first `dim_item_dict` does not contain `template`, this will remain as None. + Defaults to None. + + Raises: + KeyError: Must inherit from a named template item in `templates`. + + Returns: + tuple[AttrDict, list | None]: Definition dictionary with inherited data and a list of the inheritance tree climbed to get there. + """ + to_inherit = dim_item_dict.get("template", None) + if to_inherit is None: + updated_dim_item_dict = dim_item_dict + elif to_inherit not in templates: + raise KeyError( + f"{item_name} | Cannot find `{to_inherit}` in template inheritance tree." + ) + else: + base_def_dict, inheritance = climb_template_tree( + templates[to_inherit], templates, to_inherit, inheritance + ) + updated_dim_item_dict = deepcopy(base_def_dict) + updated_dim_item_dict.union(dim_item_dict, allow_override=True) + if inheritance is not None: + inheritance.append(to_inherit) + else: + inheritance = [to_inherit] + return updated_dim_item_dict, inheritance diff --git a/tests/test_core_util.py b/tests/test_core_util.py index 8e9175ba..7e31b89b 100644 --- a/tests/test_core_util.py +++ b/tests/test_core_util.py @@ -13,6 +13,7 @@ from calliope.util import schema from calliope.util.generate_runs import generate_runs from calliope.util.logging import log_time +from calliope.util.tools import climb_template_tree from .common.util import check_error_or_warning @@ -465,3 +466,72 @@ def test_reset_schema(self): "^[^_^\\d][\\w]*$" ]["properties"] ) + + +class TestClimbTemplateTree: + @pytest.fixture + def templates(self) -> "calliope.AttrDict": + return calliope.AttrDict( + { + "foo_group": {"template": "bar_group", "my_param": 1}, + "bar_group": {"my_param": 2, "my_other_param": 2}, + "data_table_group": {"rows": ["foobar"]}, + } + ) + + @pytest.mark.parametrize( + ("starting_dict", "expected_dict", "expected_inheritance"), + [ + ({"my_param": 1}, {"my_param": 1}, None), + ( + {"template": "foo_group"}, + {"my_param": 1, "my_other_param": 2, "template": "foo_group"}, + ["bar_group", "foo_group"], + ), + ( + {"template": "bar_group"}, + {"my_param": 2, "my_other_param": 2, "template": "bar_group"}, + ["bar_group"], + ), + ( + {"template": "bar_group", "my_param": 3, "my_own_param": 1}, + { + "my_param": 3, + "my_other_param": 2, + "my_own_param": 1, + "template": "bar_group", + }, + ["bar_group"], + ), + ( + {"template": "data_table_group", "columns": "techs"}, + { + "columns": "techs", + "rows": ["foobar"], + "template": "data_table_group", + }, + ["data_table_group"], + ), + ], + ) + def test_climb_template_tree( + self, templates, starting_dict, expected_dict, expected_inheritance + ): + """Templates should be found and applied in order of 'ancestry' (newer dict keys replace older ones if they overlap).""" + + new_dict, inheritance = climb_template_tree( + calliope.AttrDict(starting_dict), templates, "A" + ) + assert new_dict == expected_dict + assert inheritance == expected_inheritance + + def test_climb_template_tree_missing_ancestor(self, templates): + """Referencing a template that doesn't exist in `templates` raises an error.""" + with pytest.raises(KeyError) as excinfo: + climb_template_tree( + calliope.AttrDict({"template": "not_there"}), templates, "A" + ) + + assert check_error_or_warning( + excinfo, "A | Cannot find `not_there` in template inheritance tree." + ) diff --git a/tests/test_preprocess_model_data.py b/tests/test_preprocess_model_data.py index f15393e2..48bc519c 100644 --- a/tests/test_preprocess_model_data.py +++ b/tests/test_preprocess_model_data.py @@ -6,9 +6,8 @@ import pytest import xarray as xr -from calliope import exceptions -from calliope.attrdict import AttrDict -from calliope.preprocess import data_tables, scenarios +from calliope import AttrDict, exceptions +from calliope.preprocess import scenarios from calliope.preprocess.model_data import ModelDataFactory from .common.util import build_test_model as build_model @@ -25,15 +24,6 @@ def model_def(): return model_def_override, model_def_path -@pytest.fixture -def data_source_list(model_def, init_config): - model_def_dict, model_def_path = model_def - return [ - data_tables.DataTable(init_config, source_name, source_dict, {}, model_def_path) - for source_name, source_dict in model_def_dict.pop("data_tables", {}).items() - ] - - @pytest.fixture def init_config(config_defaults, model_def): model_def_dict, _ = model_def @@ -530,67 +520,6 @@ def test_template_defs_techs_missing_base_def( "(foobar, bar), (techs, foo) | Reference to item not defined in base techs", ) - @pytest.mark.parametrize( - ("node_dict", "expected_dict", "expected_inheritance"), - [ - ({"my_param": 1}, {"my_param": 1}, None), - ( - {"template": "foo_group"}, - {"my_param": 1, "my_other_param": 2, "template": "foo_group"}, - ["bar_group", "foo_group"], - ), - ( - {"template": "bar_group"}, - {"my_param": 2, "my_other_param": 2, "template": "bar_group"}, - ["bar_group"], - ), - ( - {"template": "bar_group", "my_param": 3, "my_own_param": 1}, - { - "my_param": 3, - "my_other_param": 2, - "my_own_param": 1, - "template": "bar_group", - }, - ["bar_group"], - ), - ], - ) - def test_climb_template_tree( - self, - model_data_factory: ModelDataFactory, - node_dict, - expected_dict, - expected_inheritance, - ): - """Templates should be found and applied in order of 'ancestry' (newer dict keys replace older ones if they overlap).""" - group_dict = { - "foo_group": {"template": "bar_group", "my_param": 1}, - "bar_group": {"my_param": 2, "my_other_param": 2}, - } - model_data_factory.model_definition["templates"] = AttrDict(group_dict) - new_dict, inheritance = model_data_factory._climb_template_tree( - AttrDict(node_dict), "nodes", "A" - ) - assert new_dict == expected_dict - assert inheritance == expected_inheritance - - def test_climb_template_tree_missing_ancestor( - self, model_data_factory: ModelDataFactory - ): - """Referencing a template that doesn't exist in `templates` raises an error.""" - group_dict = { - "foo_group": {"template": "bar_group", "my_param": 1}, - "bar_group": {"my_param": 2, "my_other_param": 2}, - } - model_data_factory.model_definition["templates"] = AttrDict(group_dict) - with pytest.raises(KeyError) as excinfo: - model_data_factory._climb_template_tree( - AttrDict({"template": "not_there"}), "nodes", "A" - ) - - assert check_error_or_warning(excinfo, "(nodes, A) | Cannot find `not_there`") - def test_deactivate_single_dim(self, model_data_factory_w_params: ModelDataFactory): assert "a" in model_data_factory_w_params.dataset.nodes model_data_factory_w_params._deactivate_item(nodes="a")