From ed25596f4a397274c2a5979f09dc00d12472e157 Mon Sep 17 00:00:00 2001 From: Bryn Pickering <17178478+brynpickering@users.noreply.github.com> Date: Fri, 18 Oct 2024 16:18:26 +0100 Subject: [PATCH] Add templates for data tables (#695) --- CHANGELOG.md | 2 + docs/creating/data_tables.md | 70 ++++++++++++++++++++++-- docs/creating/templates.md | 43 ++++++++++++++- src/calliope/model.py | 18 +++--- src/calliope/preprocess/data_tables.py | 1 + src/calliope/preprocess/model_data.py | 65 +++------------------- src/calliope/util/tools.py | 58 +++++++++++++++++++- tests/test_core_util.py | 76 ++++++++++++++++++++++++++ tests/test_preprocess_model_data.py | 75 +------------------------ 9 files changed, 260 insertions(+), 148 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f334677..fecb4092 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ### User-facing changes +|new| Data tables can inherit options from `templates`, like `techs` and `nodes` (#676). + |new| dimension renaming functionality when loading from a data source, using the `rename_dims` option (#680). |changed| cost expressions in math, to split out investment costs into the capital cost (`cost_investment`), annualised capital cost (`cost_investment_annualised`), fixed operation costs (`cost_operation_fixed`) and variable operation costs (`cost_operation_variable`, previously `cost_var`) (#645). diff --git a/docs/creating/data_tables.md b/docs/creating/data_tables.md index dfd28b57..2319ed04 100644 --- a/docs/creating/data_tables.md +++ b/docs/creating/data_tables.md @@ -14,10 +14,11 @@ In brief it is: * **data**: path to file or reference name for an in-memory object. * **rows**: the dimension(s) in your table defined per row. * **columns**: the dimension(s) in your table defined per column. -* **select**: values within dimensions that you want to select from your tabular data, discarding the rest. -* **drop**: dimensions to drop from your rows/columns, e.g., a "comment" row. -* **add_dims**: dimensions to add to the table after loading it in, with the corresponding value(s) to assign to the dimension index. -* **rename_dims**: dimension names to map from those defined in the data table (e.g `time`) to those used in the Calliope model (e.g. `timesteps`). +* [**select**](#selecting-dimension-values-and-dropping-dimensions): values within dimensions that you want to select from your tabular data, discarding the rest. +* [**drop**](#selecting-dimension-values-and-dropping-dimensions): dimensions to drop from your rows/columns, e.g., a "comment" row. +* [**add_dims**](#adding-dimensions): dimensions to add to the table after loading it in, with the corresponding value(s) to assign to the dimension index. +* [**rename_dims**](#renaming-dimensions-on-load): dimension names to map from those defined in the data table (e.g `time`) to those used in the Calliope model (e.g. `timesteps`). +* [**template**](#using-a-template): Reference to a [template](templates.md) from which to inherit common configuration options. When we refer to "dimensions", we mean the sets over which data is indexed in the model: `nodes`, `techs`, `timesteps`, `carriers`, `costs`. In addition, when loading from file, there is the _required_ dimension `parameters`. @@ -417,7 +418,66 @@ Or to define the same timeseries source data for two technologies at different n parameters: source_use_max ``` -## Mapping dimension names +## Using a template + +Templates allow us to define common options that are inherited by several data tables. +They can be particularly useful if you are storing your data in many small tables. + +To assign the same input timeseries data for (tech1, node1) and (tech2, node2) using [`add_dims`](#adding-dimensions): + +=== "Without `template`" + + | | | + | ---------------: | :-- | + | 2005-01-01 00:00 | 100 | + | 2005-01-01 01:00 | 200 | + + ```yaml + data_tables: + tech_data_1: + data: data_tables/tech_data.csv + rows: timesteps + add_dims: + techs: tech1 + nodes: node1 + parameters: source_use_max + tech_data_2: + data: data_tables/tech_data.csv + rows: timesteps + add_dims: + techs: tech2 + nodes: node2 + parameters: source_use_max + ``` + +=== "With `template`" + + | | | + | ---------------: | :-- | + | 2005-01-01 00:00 | 100 | + | 2005-01-01 01:00 | 200 | + + ```yaml + templates: + common_data_options: + data: data_tables/tech_data.csv + rows: timesteps + add_dims: + parameters: source_use_max + data_tables: + tech_data_1: + template: common_data_options + add_dims: + techs: tech1 + nodes: node1 + tech_data_2: + template: common_data_options + add_dims: + techs: tech2 + nodes: node2 + ``` + +## Renaming dimensions on load Sometimes, data tables are prepared in a model-agnostic fashion, and it would require extra effort to follow Calliope's dimension naming conventions. To enable these tables to be loaded without Calliope complaining, we can rename dimensions when loading them using `rename_dims`. diff --git a/docs/creating/templates.md b/docs/creating/templates.md index d1d970dd..c723c8c8 100644 --- a/docs/creating/templates.md +++ b/docs/creating/templates.md @@ -2,9 +2,11 @@ # Inheriting from templates: `templates` For larger models, duplicate entries can start to crop up and become cumbersome. -To streamline data entry, technologies and nodes can inherit common data from a `template`. +To streamline data entry, technologies, nodes, and data tables can inherit common data from a `template`. -For example, if we want to set interest rate to `0.1` across all our technologies, we could define: +## Templates in technologies + +If we want to set interest rate to `0.1` across all our technologies, we could define: ```yaml templates: @@ -22,6 +24,8 @@ techs: ... ``` +## Templates in nodes + Similarly, if we want to allow the same technologies at all our nodes: ```yaml @@ -48,6 +52,37 @@ nodes: demand_power: ``` +## Templates in data tables + +Data tables can also store common options under the `templates` key, for example: + +```yaml +templates: + common_data_options: + rows: timesteps + columns: nodes + add_dims: + parameters: source_use_max +data_tables: + pv_data: + data: /path/to/pv_timeseries.csv + template: common_data_options + add_dims: + techs: pv + wind_data: + data: /path/to/wind_timeseries.csv + template: common_data_options + add_dims: + techs: wind + hydro_data: + data: /path/to/hydro_timeseries.csv + template: common_data_options + add_dims: + techs: hydro +``` + +## Inheritance chains + Inheritance chains can also be created. That is, templates can inherit from other templates. E.g.: @@ -78,7 +113,9 @@ techs: ... ``` -Finally, template properties can always be overridden by the inheriting component. +## Overriding template values + +Template properties can always be overridden by the inheriting component. This can be useful to streamline setting costs, e.g.: ```yaml diff --git a/src/calliope/model.py b/src/calliope/model.py index 42f67875..ee8c5a77 100644 --- a/src/calliope/model.py +++ b/src/calliope/model.py @@ -25,7 +25,7 @@ update_then_validate_config, validate_dict, ) -from calliope.util.tools import relative_path +from calliope.util.tools import climb_template_tree, relative_path if TYPE_CHECKING: from calliope.backend.backend_model import BackendModel @@ -180,15 +180,15 @@ def _init_from_model_def_dict( "scenario": scenario, "defaults": param_metadata["default"], } - - data_tables = [ - DataTable( - init_config, source_name, source_dict, data_table_dfs, self._def_path + templates = model_definition.get("templates", AttrDict()) + data_tables: list[DataTable] = [] + for table_name, table_dict in model_definition.pop("data_tables", {}).items(): + table_dict, _ = climb_template_tree(table_dict, templates, table_name) + data_tables.append( + DataTable( + init_config, table_name, table_dict, data_table_dfs, self._def_path + ) ) - for source_name, source_dict in model_definition.pop( - "data_tables", {} - ).items() - ] model_data_factory = ModelDataFactory( init_config, model_definition, data_tables, attributes, param_metadata diff --git a/src/calliope/preprocess/data_tables.py b/src/calliope/preprocess/data_tables.py index 717ef595..4a90fbf3 100644 --- a/src/calliope/preprocess/data_tables.py +++ b/src/calliope/preprocess/data_tables.py @@ -40,6 +40,7 @@ class DataTableDict(TypedDict): add_dims: NotRequired[dict[str, str | list[str]]] select: NotRequired[dict[str, str | bool | int]] drop: NotRequired[Hashable | list[Hashable]] + template: NotRequired[str] class DataTable: diff --git a/src/calliope/preprocess/model_data.py b/src/calliope/preprocess/model_data.py index 6de0aa3f..7c6d6cc3 100644 --- a/src/calliope/preprocess/model_data.py +++ b/src/calliope/preprocess/model_data.py @@ -17,7 +17,7 @@ from calliope.attrdict import AttrDict from calliope.preprocess import data_tables, time from calliope.util.schema import MODEL_SCHEMA, validate_dict -from calliope.util.tools import listify +from calliope.util.tools import climb_template_tree, listify LOGGER = logging.getLogger(__name__) @@ -555,10 +555,15 @@ def _inherit_defs( item_base_def = deepcopy(base_def[item_name]) item_base_def.union(item_def, allow_override=True) + if item_name in self.tech_data_from_tables: + _data_table_dict = deepcopy(self.tech_data_from_tables[item_name]) + _data_table_dict.union(item_base_def, allow_override=True) + item_base_def = _data_table_dict else: item_base_def = item_def - updated_item_def, inheritance = self._climb_template_tree( - item_base_def, dim_name, item_name + templates = self.model_definition.get("templates", AttrDict()) + updated_item_def, inheritance = climb_template_tree( + item_base_def, templates, item_name ) if not updated_item_def.get("active", True): @@ -576,60 +581,6 @@ def _inherit_defs( return updated_defs - def _climb_template_tree( - self, - dim_item_dict: AttrDict, - dim_name: Literal["nodes", "techs"], - item_name: str, - inheritance: list | None = None, - ) -> tuple[AttrDict, list | None]: - """Follow the `template` references from `nodes` / `techs` to `templates`. - - Abstract template definitions (those in `templates`) can inherit each other, but `nodes`/`techs` cannot. - - This function will be called recursively until a definition dictionary without `template` is reached. - - Args: - dim_item_dict (AttrDict): Dictionary (possibly) containing `template`. - dim_name (Literal[nodes, techs]): - The name of the dimension we're working with, so that we can access the correct `_groups` definitions. - item_name (str): - The current position in the inheritance tree. - inheritance (list | None, optional): - A list of items that have been inherited (starting with the oldest). - If the first `dim_item_dict` does not contain `template`, this will remain as None. - Defaults to None. - - Raises: - KeyError: Must inherit from a named template item in `templates`. - - Returns: - tuple[AttrDict, list | None]: Definition dictionary with inherited data and a list of the inheritance tree climbed to get there. - """ - to_inherit = dim_item_dict.get("template", None) - dim_groups = AttrDict(self.model_definition.get("templates", {})) - if to_inherit is None: - if dim_name == "techs" and item_name in self.tech_data_from_tables: - _data_table_dict = deepcopy(self.tech_data_from_tables[item_name]) - _data_table_dict.union(dim_item_dict, allow_override=True) - dim_item_dict = _data_table_dict - updated_dim_item_dict = dim_item_dict - elif to_inherit not in dim_groups: - raise KeyError( - f"({dim_name}, {item_name}) | Cannot find `{to_inherit}` in template inheritance tree." - ) - else: - base_def_dict, inheritance = self._climb_template_tree( - dim_groups[to_inherit], dim_name, to_inherit, inheritance - ) - updated_dim_item_dict = deepcopy(base_def_dict) - updated_dim_item_dict.union(dim_item_dict, allow_override=True) - if inheritance is not None: - inheritance.append(to_inherit) - else: - inheritance = [to_inherit] - return updated_dim_item_dict, inheritance - def _deactivate_item(self, **item_ref): for dim_name, item_name in item_ref.items(): if item_name not in self.dataset.coords.get(dim_name, xr.DataArray()): diff --git a/src/calliope/util/tools.py b/src/calliope/util/tools.py index 51920d88..dee2f6ca 100644 --- a/src/calliope/util/tools.py +++ b/src/calliope/util/tools.py @@ -2,11 +2,15 @@ # Licensed under the Apache 2.0 License (see LICENSE file). """Assorted helper tools.""" +from copy import deepcopy from pathlib import Path -from typing import Any, TypeVar +from typing import TYPE_CHECKING, Any, TypeVar from typing_extensions import ParamSpec +if TYPE_CHECKING: + from calliope import AttrDict + P = ParamSpec("P") T = TypeVar("T") @@ -47,3 +51,55 @@ def listify(var: Any) -> list: else: var = [var] return var + + +def climb_template_tree( + input_dict: "AttrDict", + templates: "AttrDict", + item_name: str | None = None, + inheritance: list | None = None, +) -> tuple["AttrDict", list | None]: + """Follow the `template` references from model definition elements to `templates`. + + Model definition elements can inherit template entries (those in `templates`). + Template entries can also inherit each other, to create an inheritance chain. + + This function will be called recursively until a definition dictionary without `template` is reached. + + Args: + input_dict (AttrDict): Dictionary (possibly) containing `template`. + templates (AttrDict): Dictionary of available templates. + item_name (str | None, optional): + The current position in the inheritance tree. + If given, used only for a more expressive KeyError. + Defaults to None. + inheritance (list | None, optional): + A list of items that have been inherited (starting with the oldest). + If the first `input_dict` does not contain `template`, this will remain as None. + Defaults to None. + + Raises: + KeyError: Must inherit from a named template item in `templates`. + + Returns: + tuple[AttrDict, list | None]: Definition dictionary with inherited data and a list of the inheritance tree climbed to get there. + """ + to_inherit = input_dict.get("template", None) + if to_inherit is None: + updated_input_dict = input_dict + elif to_inherit not in templates: + message = f"Cannot find `{to_inherit}` in template inheritance tree." + if item_name is not None: + message = f"{item_name} | {message}" + raise KeyError(message) + else: + base_def_dict, inheritance = climb_template_tree( + templates[to_inherit], templates, to_inherit, inheritance + ) + updated_input_dict = deepcopy(base_def_dict) + updated_input_dict.union(input_dict, allow_override=True) + if inheritance is not None: + inheritance.append(to_inherit) + else: + inheritance = [to_inherit] + return updated_input_dict, inheritance diff --git a/tests/test_core_util.py b/tests/test_core_util.py index 8e9175ba..32ea38e9 100644 --- a/tests/test_core_util.py +++ b/tests/test_core_util.py @@ -13,6 +13,7 @@ from calliope.util import schema from calliope.util.generate_runs import generate_runs from calliope.util.logging import log_time +from calliope.util.tools import climb_template_tree from .common.util import check_error_or_warning @@ -465,3 +466,78 @@ def test_reset_schema(self): "^[^_^\\d][\\w]*$" ]["properties"] ) + + +class TestClimbTemplateTree: + @pytest.fixture + def templates(self) -> "calliope.AttrDict": + return calliope.AttrDict( + { + "foo_group": {"template": "bar_group", "my_param": 1}, + "bar_group": {"my_param": 2, "my_other_param": 2}, + "data_table_group": {"rows": ["foobar"]}, + } + ) + + @pytest.mark.parametrize( + ("starting_dict", "expected_dict", "expected_inheritance"), + [ + ({"my_param": 1}, {"my_param": 1}, None), + ( + {"template": "foo_group"}, + {"my_param": 1, "my_other_param": 2, "template": "foo_group"}, + ["bar_group", "foo_group"], + ), + ( + {"template": "bar_group"}, + {"my_param": 2, "my_other_param": 2, "template": "bar_group"}, + ["bar_group"], + ), + ( + {"template": "bar_group", "my_param": 3, "my_own_param": 1}, + { + "my_param": 3, + "my_other_param": 2, + "my_own_param": 1, + "template": "bar_group", + }, + ["bar_group"], + ), + ( + {"template": "data_table_group", "columns": "techs"}, + { + "columns": "techs", + "rows": ["foobar"], + "template": "data_table_group", + }, + ["data_table_group"], + ), + ], + ) + def test_climb_template_tree( + self, templates, starting_dict, expected_dict, expected_inheritance + ): + """Templates should be found and applied in order of 'ancestry' (newer dict keys replace older ones if they overlap).""" + + new_dict, inheritance = climb_template_tree( + calliope.AttrDict(starting_dict), templates, "A" + ) + assert new_dict == expected_dict + assert inheritance == expected_inheritance + + @pytest.mark.parametrize( + ("item_name", "expected_message_prefix"), [("A", "A | "), (None, "")] + ) + def test_climb_template_tree_missing_ancestor( + self, templates, item_name, expected_message_prefix + ): + """Referencing a template that doesn't exist in `templates` raises an error.""" + with pytest.raises(KeyError) as excinfo: + climb_template_tree( + calliope.AttrDict({"template": "not_there"}), templates, item_name + ) + + assert check_error_or_warning( + excinfo, + f"{expected_message_prefix}Cannot find `not_there` in template inheritance tree.", + ) diff --git a/tests/test_preprocess_model_data.py b/tests/test_preprocess_model_data.py index f15393e2..48bc519c 100644 --- a/tests/test_preprocess_model_data.py +++ b/tests/test_preprocess_model_data.py @@ -6,9 +6,8 @@ import pytest import xarray as xr -from calliope import exceptions -from calliope.attrdict import AttrDict -from calliope.preprocess import data_tables, scenarios +from calliope import AttrDict, exceptions +from calliope.preprocess import scenarios from calliope.preprocess.model_data import ModelDataFactory from .common.util import build_test_model as build_model @@ -25,15 +24,6 @@ def model_def(): return model_def_override, model_def_path -@pytest.fixture -def data_source_list(model_def, init_config): - model_def_dict, model_def_path = model_def - return [ - data_tables.DataTable(init_config, source_name, source_dict, {}, model_def_path) - for source_name, source_dict in model_def_dict.pop("data_tables", {}).items() - ] - - @pytest.fixture def init_config(config_defaults, model_def): model_def_dict, _ = model_def @@ -530,67 +520,6 @@ def test_template_defs_techs_missing_base_def( "(foobar, bar), (techs, foo) | Reference to item not defined in base techs", ) - @pytest.mark.parametrize( - ("node_dict", "expected_dict", "expected_inheritance"), - [ - ({"my_param": 1}, {"my_param": 1}, None), - ( - {"template": "foo_group"}, - {"my_param": 1, "my_other_param": 2, "template": "foo_group"}, - ["bar_group", "foo_group"], - ), - ( - {"template": "bar_group"}, - {"my_param": 2, "my_other_param": 2, "template": "bar_group"}, - ["bar_group"], - ), - ( - {"template": "bar_group", "my_param": 3, "my_own_param": 1}, - { - "my_param": 3, - "my_other_param": 2, - "my_own_param": 1, - "template": "bar_group", - }, - ["bar_group"], - ), - ], - ) - def test_climb_template_tree( - self, - model_data_factory: ModelDataFactory, - node_dict, - expected_dict, - expected_inheritance, - ): - """Templates should be found and applied in order of 'ancestry' (newer dict keys replace older ones if they overlap).""" - group_dict = { - "foo_group": {"template": "bar_group", "my_param": 1}, - "bar_group": {"my_param": 2, "my_other_param": 2}, - } - model_data_factory.model_definition["templates"] = AttrDict(group_dict) - new_dict, inheritance = model_data_factory._climb_template_tree( - AttrDict(node_dict), "nodes", "A" - ) - assert new_dict == expected_dict - assert inheritance == expected_inheritance - - def test_climb_template_tree_missing_ancestor( - self, model_data_factory: ModelDataFactory - ): - """Referencing a template that doesn't exist in `templates` raises an error.""" - group_dict = { - "foo_group": {"template": "bar_group", "my_param": 1}, - "bar_group": {"my_param": 2, "my_other_param": 2}, - } - model_data_factory.model_definition["templates"] = AttrDict(group_dict) - with pytest.raises(KeyError) as excinfo: - model_data_factory._climb_template_tree( - AttrDict({"template": "not_there"}), "nodes", "A" - ) - - assert check_error_or_warning(excinfo, "(nodes, A) | Cannot find `not_there`") - def test_deactivate_single_dim(self, model_data_factory_w_params: ModelDataFactory): assert "a" in model_data_factory_w_params.dataset.nodes model_data_factory_w_params._deactivate_item(nodes="a")