Merge branch 'main' into update-definition-matrix

.github/.codecov.yml

@@ -3,4 +3,4 @@ coverage:
     patch: off
 
 comment:
-  layout: "diff, flags, files"
+  layout: "diff, flags, files"

.github/workflows/commit-ci.yml

@@ -40,5 +40,16 @@ jobs:
     - name: Install jupyter kernel
       run: python -m ipykernel install --user --name calliope
 
-    - name: run tests
+    - name: run tests without coverage
+      if: github.ref != 'refs/heads/main'
       run: pytest
+
+    - name: run tests with coverage
+      if: github.ref == 'refs/heads/main'
+      run: pytest --cov
+
+    - name: upload coverage report to Codecov
+      if: github.ref == 'refs/heads/main'
+      uses: codecov/codecov-action@v3
+      env:
+        directory: "./reports/coverage/"

.github/workflows/pr-ci.yml

@@ -69,11 +69,17 @@ jobs:
     - name: Install jupyter kernel
       run: python -m ipykernel install --user --name calliope
 
-    - name: run tests
+    - name: run tests with coverage
+      if: matrix.os == 'ubuntu-latest' && matrix.py3version == '11'
+      run: pytest --cov
+
+    - name: run tests without coverage
+      if: matrix.os != 'ubuntu-latest' || matrix.py3version != '11'
       run: pytest
 
     - name: Upload coverage reports to Codecov
       uses: codecov/codecov-action@v3
       if: matrix.os == 'ubuntu-latest' && matrix.py3version == '11'
-      env:
+      with:
+        fail_ci_if_error: true
         directory: "./reports/coverage/"

README.md

@@ -1,7 +1,7 @@
 [![Chat on Gitter](https://img.shields.io/gitter/room/calliope-project/calliope.svg?style=flat-square)](https://app.gitter.im/#/room/#calliope-project_calliope:gitter.im)
-[![Main branch build status](https://img.shields.io/azure-devops/build/calliope-project/371cbbaa-fa6b-4efb-9b23-c4283a8e33eb/1?style=flat-square)](https://dev.azure.com/calliope-project/calliope/_build?definitionId=1)
+[![Main branch build status](https://github.com/calliope-project/calliope/actions/workflows/commit-ci.yml/badge.svg?branch=main)](https://github.com/calliope-project/calliope/actions/workflows/commit-ci.yml)
 [![Documentation build status](https://img.shields.io/readthedocs/calliope.svg?style=flat-square)](https://readthedocs.org/projects/calliope/builds/)
-[![Test coverage](https://img.shields.io/codecov/c/github/calliope-project/calliope?style=flat-square&token=b4fd170f0e7b43679a8bf649719e1cea)](https://codecov.io/gh/calliope-project/calliope)
+[![Test coverage](https://codecov.io/gh/calliope-project/calliope/graph/badge.svg?token=UM542yaYrh)](https://codecov.io/gh/calliope-project/calliope)
 [![PyPI version](https://img.shields.io/pypi/v/calliope.svg?style=flat-square)](https://pypi.python.org/pypi/calliope)
 [![Anaconda.org/conda-forge version](https://img.shields.io/conda/vn/conda-forge/calliope.svg?style=flat-square&label=conda)](https://anaconda.org/conda-forge/calliope)
 [![JOSS DOI](https://img.shields.io/badge/JOSS-10.21105/joss.00825-green.svg?style=flat-square)](https://doi.org/10.21105/joss.00825)

doc/helpers/generate_math.py

@@ -54,7 +54,9 @@ def generate_base_math_model(model_config: dict) -> calliope.Model:
     Returns:
         calliope.Model: Base math model to use in generating custom math docs.
     """
-    model = calliope.Model(config=model_config, timeseries_dataframes=_ts_dfs())
+    model = calliope.Model(
+        model_definition=model_config, timeseries_dataframes=_ts_dfs()
+    )
     model.math_documentation.build()
     model.math_documentation.write(STATICPATH / "math.rst")
     return model
@@ -78,7 +80,9 @@ def generate_custom_math_model(
     model_config = calliope.AttrDict(model_config)
     model_config_updates = calliope.AttrDict(model_config_updates)
     model_config.union(model_config_updates)
-    model = calliope.Model(config=model_config, timeseries_dataframes=_ts_dfs())
+    model = calliope.Model(
+        model_definition=model_config, timeseries_dataframes=_ts_dfs()
+    )
     _keep_only_changes(base_model, model)
 
     model.math_documentation.write(STATICPATH / f"math_{name}.rst")
@@ -142,8 +146,6 @@ def generate_model_config() -> dict[str, dict]:
         }
 
     return {
-        "model": {},
-        "run": {"objective_options": {"cost_class": {"monetary": 1}}},
         "nodes": {
             "A": {"techs": {k: None for k in dummy_techs.keys()}, "available_area": 1}
         },
@@ -152,12 +154,9 @@ def generate_model_config() -> dict[str, dict]:
 
 
 def _add_data(name, default_val):
-    "If timeseries is allowed, we reference timeseries data. Some parameters need hardcoded values to be returned"
-    if name in POSSIBLE_TIMESERIES_DATA:
-        if name == "carrier_ratios":
-            return {"carrier_in.electricity": "df=ts"}
-        else:
-            return "df=ts"
+    "Some parameters need hardcoded values to be returned"
+    if name == "carrier_ratios":
+        return {"carrier_in.electricity": 1}
     elif name == "export_carrier":
         return "electricity"
     elif default_val is None or name == "interest_rate":
@@ -208,7 +207,7 @@ def _ts_dfs() -> dict[str, pd.DataFrame]:
         index=pd.date_range("2005-01-01 00:00", "2005-01-01 02:00", freq="H"),
         columns=["A"],
     )
-    return {"ts": ts, "ts_neg": -1 * ts}
+    return {"ts": ts}
 
 
 if __name__ == "__main__":
@@ -219,16 +218,18 @@ def _ts_dfs() -> dict[str, pd.DataFrame]:
         base_model,
         base_model_config,
         {
-            "model": {
-                "custom_math": ["storage_inter_cluster"],
-                "time": {
-                    "function": "apply_clustering",
-                    "function_options": {
-                        "clustering_func": "kmeans",
-                        "how": "mean",
-                        "k": 1,
+            "config": {
+                "init": {
+                    "custom_math": ["storage_inter_cluster"],
+                    "time": {
+                        "function": "apply_clustering",
+                        "function_options": {
+                            "clustering_func": "kmeans",
+                            "how": "mean",
+                            "k": 1,
+                        },
                     },
-                },
+                }
             },
         },
         "storage_inter_cluster",

doc/user/advanced_features.rst

@@ -136,15 +136,15 @@ None of the ``tech_groups`` appear in model results, they are only used to group
 Removing techs, locations and links
 -----------------------------------
 
-By specifying :yaml:`exists: false` in the model configuration, which can be done for example through overrides, model components can be removed for debugging or scenario analysis.
+By specifying :yaml:`active: false` in the model configuration, which can be done for example through overrides, model components can be removed for debugging or scenario analysis.
 
 This works for:
 
-* Techs: :yaml:`techs.tech_name.exists: false`
-* Locations: :yaml:`locations.location_name.exists: false`
-* Links: :yaml:`links.location1,location2.exists: false`
-* Techs at a specific location:  :yaml:`locations.location_name.techs.tech_name.exists: false`
-* Transmission techs at a specific location: :yaml:`links.location1,location2.techs.transmission_tech.exists: false`
+* Techs: :yaml:`techs.tech_name.active: false`
+* Locations: :yaml:`locations.location_name.active: false`
+* Links: :yaml:`links.location1,location2.active: false`
+* Techs at a specific location:  :yaml:`locations.location_name.techs.tech_name.active: false`
+* Transmission techs at a specific location: :yaml:`links.location1,location2.techs.transmission_tech.active: false`
 
 .. _operational_mode:
 

doc/user/building.rst

@@ -89,7 +89,7 @@ The ``init`` configuration items are accessed when you initialise your model (`c
 The ``build`` configuration items are accessed when you build your optimisation problem (`calliope.Model.build(...)`).
 The ``solve`` configuration items are accessed when you solve your optimisation problem (`calliope.Model.solve(...)`).
 
-At each of these stages you can override what you have put in your YAML file, or the default calliope will use when you have defined nothing, by providing additional keyword arguments on calling `calliope.Model` or its methods. E.g.,:
+At each of these stages you can override what you have put in your YAML file (or if not in your YAML file, the default that Calliope uses), by providing additional keyword arguments on calling `calliope.Model` or its methods. E.g.,:
 
 .. code-block:: python
 
@@ -129,8 +129,6 @@ In the case of Gurobi, for example, it is usually fastest to use the direct Pyth
 
 .. note:: The opposite is currently true for CPLEX, which runs faster with the default ``solver_io``.
 
-Further optional settings, including debug settings, can be specified in the run configuration.
-
 .. seealso::
 
     :ref:`config_reference_config`, :doc:`troubleshooting`, :ref:`solver_options`, :ref:`documentation on operate mode <operational_mode>`, :ref:`documentation on SPORES mode <spores_mode>`, :doc:`built-in examples <ref_example_models>`
@@ -146,7 +144,15 @@ This could be a single value:
 .. code-block:: yaml
 
     parameters:
-        my_param: { data: 10 }
+        my_param: 10
+
+or (equivalent):
+
+.. code-block:: yaml
+
+    parameters:
+        my_param:
+            data: 10
 
 which can then be accessed in the model inputs `model.inputs.my_param` and used in custom math as `my_param`.
 

doc/user/config_defaults.rst

@@ -7,26 +7,26 @@ Defaults are automatically applied in constraints whenever there is no user inpu
 
 .. _config_reference_config:
 
-Model initialisation configuration
-----------------------------------
+Model initialisation (``config.init``)
+--------------------------------------
 
 .. csv-table::
    :file: includes/config_init.csv
    :header: Setting,Default,Comments
    :widths: 10, 5, 15
    :stub-columns: 0
 
-Optimisation problem build configuration
-----------------------------------------
+Optimisation problem build (``config.build``)
+---------------------------------------------
 
 .. csv-table::
    :file: includes/config_build.csv
    :header: Setting,Default,Comments
    :widths: 10, 5, 15
    :stub-columns: 0
 
-Optimisation problem solve configuration
-----------------------------------------
+Optimisation problem solve (``config.solve``)
+---------------------------------------------
 
 .. csv-table::
    :file: includes/config_solve.csv

doc/user/ref_formulation.rst

@@ -10,7 +10,7 @@ All built-in Calliope math can be found in the calliope `math directory <https:/
 
 By default, the :ref:`base math <base_math>` is loaded from file.
 If you want to overwrite the base math with other built-in math, you can do so by referring to the file by its name (without the file extension) in :yaml:`model.custom_math`, e.g. :yaml:`model.custom_math: [storage_inter_cluster]`.
-When solving the model in a :ref:`run mode <config_reference_config>` other than `plan`, some built-in custom math will be applied automatically from a file of the same name (`spores` mode custom math is stored in `math/spores.yaml <https://github.com/calliope-project/calliope/blob/main/calliope/math/spores.yaml>`_).
+When solving the model in a :ref:`run mode <config_reference_config>` other than `plan`, some built-in custom math will be applied automatically from a file of the same name (e.g., `spores` mode custom math is stored in `math/spores.yaml <https://github.com/calliope-project/calliope/blob/main/calliope/math/spores.yaml>`_).
 The changes made by the built-in custom math are detailed in this page.
 
 .. note:: Custom math is applied in the order it appears in the :yaml:`model.custom_math` list. By default, any run mode custom math will be applied as the final step. If you want to apply your own custom math *after* the run mode custom math, you should add it explicitly to the :yaml:`model.custom_math` list, e.g., :yaml:`model.custom_math: [operate, my_custom_math.yaml]`.
@@ -64,4 +64,4 @@ SPORES mode custom math
 Below are the changes from the base math introduced by the built-in custom math file ``spores``.
 These changes are applied automatically if selecting the run mode ``spores``
 
-TODO: Add spores math
+TODO: Add spores math

doc/user/troubleshooting.rst

@@ -12,9 +12,9 @@ General strategies
 *
   **Analysing the optimisation problem without running the model**: If you are comfortable with navigating Pyomo objects, then you can inspect the Pyomo model backend after building it using :python:`model.build()`.
   Pyomo objects are then accessible within :python:`model.backend`.
-  For instance, the constraints limiting outflows can be viewed by calling :python:`model.get_constraint("flow_out_max")`.
-  A single Pyomo object can be then accessed by slicing the resulting array: :python:`model.get_constraint("flow_out_max").sel(techs=...)`.
-  You can also view the data in a more readable format by using setting the `as_backend_objs` option to false: :python:`constr = model.get_constraint("flow_out_max", as_backend_objs=False)`.
+  For instance, the constraints limiting outflows can be viewed by calling :python:`model.backend.get_constraint("flow_out_max")`.
+  A single Pyomo object can be then accessed by slicing the resulting array: :python:`model.backend.get_constraint("flow_out_max").sel(techs=...)`.
+  You can also view the data in a more readable format by using setting the `as_backend_objs` option to false: :python:`constr = model.backend.get_constraint("flow_out_max", as_backend_objs=False)`.
   This will allow you to inspect constraint upper bounds (`constr.ub`), lower bounds (`constr.lb`), and bodies as math strings (`constr.body`).
 
   Alternatively, if you are working from the command line or have little experience with Pyomo, you can generate an LP file.

pyproject.toml

@@ -5,8 +5,8 @@ minversion = "6.0"
 # `--strict-markers` - Raise error on unexpected pytest markers being used (add new markers to `markers` config)
 # `-nauto` - parallelise over as many threads as possible (uses pytest-xdist). If debugging (`--pdb`), this will default to one thread.
 # `--nbmake --nbmake-kernel=calliope` - test example notebooks using the "calliope" notebook kernel (uses nbmake)
-# `--cov --cov-report=xml --cov-config=pyproject.toml` - generate coverage report for tests (uses pytest-cov; call `--no-cov` in CLI to switch off; `--cov-config` include to avoid bug)
-addopts = "-rav --dist=loadscope --strict-markers -nauto --nbmake --nbmake-kernel=calliope --cov --cov-report=xml --cov-config=pyproject.toml"
+# `--cov-report=xml --cov-config=pyproject.toml` - coverage report config for when running in tests (uses pytest-cov; call `--cov` in CLI to switch coverage on; `--cov-config` include to avoid bug)
+addopts = "-rav --dist=loadscope --strict-markers -nauto --nbmake --nbmake-kernel=calliope --cov-report=xml --cov-config=pyproject.toml"
 # TODO: add testpath once notebooks are fixed: "doc/_static/notebooks"
 testpaths = ["tests"]
 

requirements/base.txt

@@ -6,7 +6,7 @@ jsonschema >= 4.17, < 4.19
 natsort >= 8, < 9
 netcdf4 >= 1.2, < 1.7
 numpy >= 1, < 2
-pandas >= 2, < 3
+pandas >= 2, < 2.1
 pyomo >= 6.5, < 7
 pyparsing >= 3.0, < 3.1
 ruamel.yaml >= 0.17, < 0.18

src/calliope/cli.py

@@ -182,7 +182,7 @@ def new(path, template, debug):
     with format_exceptions(debug):
         if template is None:
             template = "national_scale"
-        source_path = examples._PATHS[template]
+        source_path = examples.EXAMPLE_MODEL_DIR / template
         click.echo("Copying {} template to target directory: {}".format(template, path))
         shutil.copytree(source_path, path)
 

src/calliope/config/defaults.yaml

@@ -16,7 +16,7 @@ config:
     subset_time: null # Subset of timesteps as a two-element list giving the range, e.g. ['2005-01-01', '2005-01-05'], or a single string, e.g. '2005-01'
     time: null # Optional settings to adjust time resolution, see :ref:`time_clustering` for the available options
     timeseries_data_path: null # Path to time series data
-    timeseries_dateformat: "ISO8601" # Timestamp format of all time series data when read from file
+    timeseries_dateformat: "ISO8601" # Timestamp format of all time series data when read from file. "ISO8601" means "YYYY-mm-dd HH:MM:SS".
     custom_math: [] # List of references to files which contain custom mathematical formulations. If referring to an in-built Calliope custom math file (see documentation for available files), do not append the reference with ".yaml". If referring to your own custom math file, ensure the file type is given as a suffix (".yaml" or ".yml"). Relative paths will be assumed to be relative to the `config` file given when creating a calliope Model (`calliope.Model(config)`)
   build:
     backend: pyomo  # Module with which to build the optimisation problem
@@ -41,8 +41,12 @@ config:
     zero_threshold: 1e-10  # On postprocessing the optimisation results, values smaller than this threshold will be considered as optimisation artefacts and will be set to zero.
 
 parameters:
-  bigM:  # Large value used to define certain optimisation problems. See https://en.wikipedia.org/wiki/Big_M_method for more information. This value should be larger than the largest values that any decision variables can take, but should not be **too** large (i.e., do not set it greater than 3 orders of magnitude above the numeric range of the model). If too large, numerical problems may arise in the optimisation.
-    data: 1e9
+  # BigM is a large value used to define certain optimisation problems.
+  # See https://en.wikipedia.org/wiki/Big_M_method for more information.
+  # This value should be larger than the largest values that any decision variables can take,
+  # but should not be **too** large (i.e., do not set it greater than 3 orders of magnitude above the numeric range of the model).
+  # if too large, numerical problems may arise in the optimisation.
+  bigM: 1e9
   objective_cost_class: # Weightings for cost classes to apply in the objective function.
     data: 1
     index: [monetary]
@@ -438,20 +442,21 @@ techs:
         area_use: 0 # name: Cost of area use ¦ unit: m\ :sup:`-2` ¦
         source_cap: 0 # name: Cost of source flow capacity ¦ unit: kW :sup:`-1` ¦
         storage_cap: 0 # name: Cost of storage capacity ¦ unit: kWh :sup:`-1` ¦
-    exists: true
+        depreciation_rate: 1 # name: Depreciation rate for annualisation of investment costs ¦ unit: fraction ¦
+    active: true
 
 nodes:
   default_node:
     transmission_node: false # true only if no techs are defined (including complete omission of the `techs` key). Automatically added during processing if not defined by user.
     coordinates: {} # The node's x-y coordinates for distance calculations and plotting: {lat: ..., lon: ...} or {x: ..., y: ...}
     available_area: .inf # This node's available land area (required if constraining technology deployment by area).
     techs: null # A list of technologies, optionally with node-specific settings overriding the technology's global settings.
-    exists: true
+    active: true
 
 links:
   default_node_from,default_node_to:
     techs:
       default_tech:
         distance: null # Used for per_distance constraints, but automatically inferred from coordinates of nodes in a link if not given directly
 
-    exists: true
+    active: true

src/calliope/core/attrdict.py

@@ -177,7 +177,7 @@ def from_yaml(cls, f, resolve_imports=True):
             overrides definitions in the imported file.
 
         """
-        if isinstance(f, str) or isinstance(f, Path):
+        if isinstance(f, (str, Path)):
             with open(
                 f,
                 "r",