Clean and isolate model/backend attributes

CHANGELOG.md

@@ -50,6 +50,8 @@ This reduces memory footprint of parameter arrays.
 
 |changed| Moved from black formatting to the Ruff formatter (black-based, but faster).
 
+|fixed| Removed duplicated attribute: `model._timings` | `model._model_data.attrs["timestamps"]` -> `model.timestamps`
+
 ## 0.7.0.dev3 (2024-02-14)
 
 ### User-facing changes

docs/advanced/backend_choice.md

@@ -15,4 +15,4 @@ If you have access to a Gurobi license, this does not require any extra effort o
 1. Install the Gurobi Python library into your Calliope environment: `mamba install gurobi::gurobi`.
 1. Select the Gurobi backend in your YAML configuration (`!#yaml config.build.backend: gurobi`) or at build time if running in a Python script or interactively (`!#python model.build(backend="gurobi")`).
 
-You can still [interface with your optimisation problem](backend_interface.md), but some methods will raise an exception when the Gurobi Python API does not allow for something that the Pyomo API does.
+You can still [interface with your optimisation problem](backend_interface.md), but some methods will raise an exception when the Gurobi Python API does not allow for something that the Pyomo API does.

docs/examples/calliope_model_object.py

@@ -37,41 +37,32 @@
 print(m.info())
 
 # %% [markdown]
-# ## Model definition dictionary
-#
-# `m._model_def_dict` is a python dictionary that holds all the data from the model definition YAML files, restructured into one dictionary.
-#
-# The underscore before the method indicates that it defaults to being hidden (i.e. you wouldn't see it by trying a tab auto-complete and it isn't documented)
-
-# %%
-m._model_def_dict.keys()
-
-# %% [markdown]
-# `techs` hold only the information about a technology that is specific to that node
+# `techs` parameters can be accessed by selecting a specific technology.
+# Note how only the relevant carrier holds information for this parameter.
 
 # %%
-m._model_def_dict["techs"]["pv"]
+m.inputs.sel(techs="pv")["flow_cap_max"]
 
 # %% [markdown]
-# `nodes` hold only the information about a technology that is specific to that node
+# `nodes` parameters can be accessed in a similar way, and you can even combine both for specific searches:
 
 # %%
-m._model_def_dict["nodes"]["X2"]["techs"]["pv"]
+m.inputs.sel(techs="pv", nodes="X2")["flow_cap_max"]
 
 # %% [markdown]
 # ## Model data
 #
-# `m._model_data` is an xarray Dataset.
-# Like `_model_def_dict` it is a hidden prperty of the Model as you are expected to access the data via the public property `inputs`
+# `m._model_data` is an xarray Dataset containing the full Model data, and is a hidden property.
+# You are expected to access the data via the public property `inputs`
 
 # %%
-m.inputs
+m._model_data
 
 # %% [markdown]
 # Until we solve the model, `inputs` is the same as `_model_data`
 
 # %%
-m._model_data
+m.inputs
 
 # %% [markdown]
 # We can find the same PV `flow_cap_max` data as seen in `m._model_run`

docs/hooks/generate_math_docs.py

@@ -9,6 +9,7 @@
 from pathlib import Path
 
 import calliope
+from calliope.postprocess.math_documentation import MathDocumentation
 from mkdocs.structure.files import File
 
 logger = logging.getLogger("mkdocs")
@@ -42,7 +43,7 @@ def on_files(files: list, config: dict, **kwargs):
     """Process documentation for pre-defined calliope math files."""
     model_config = calliope.AttrDict.from_yaml(MODEL_PATH)
 
-    base_model = generate_base_math_model()
+    base_documentation = generate_base_math_documentation()
     write_file(
         "base.yaml",
         textwrap.dedent(
@@ -51,22 +52,24 @@ def on_files(files: list, config: dict, **kwargs):
         This math is _always_ applied but can be overridden with pre-defined additional math or [your own math][adding-your-own-math-to-a-model].
         """
         ),
-        base_model,
+        base_documentation,
         files,
         config,
     )
 
     for override in model_config["overrides"].keys():
-        custom_model = generate_custom_math_model(base_model, override)
+        custom_documentation = generate_custom_math_documentation(
+            base_documentation, override
+        )
         write_file(
             f"{override}.yaml",
             textwrap.dedent(
                 f"""
-            Pre-defined additional math to apply {custom_model.inputs.attrs['name']} math on top of the [base mathematical formulation][base-math].
+            Pre-defined additional math to apply {custom_documentation.name} math on top of the [base mathematical formulation][base-math].
             This math is _only_ applied if referenced in the `config.init.add_math` list as `{override}`.
             """
             ),
-            custom_model,
+            custom_documentation,
             files,
             config,
         )
@@ -77,7 +80,7 @@ def on_files(files: list, config: dict, **kwargs):
 def write_file(
     filename: str,
     description: str,
-    model: calliope.Model,
+    math_documentation: MathDocumentation,
     files: list[File],
     config: dict,
 ) -> None:
@@ -86,12 +89,10 @@ def write_file(
     Args:
         filename (str): name of produced `.md` file.
         description (str): first paragraph after title.
-        model (calliope.Model): calliope model with the given math.
+        math_documentation (MathDocumentation): calliope math documentation.
         files (list[File]): math files to parse.
         config (dict): documentation configuration.
     """
-    title = model.inputs.attrs["name"] + " math"
-
     output_file = (Path("math") / filename).with_suffix(".md")
     output_full_filepath = Path(TEMPDIR.name) / output_file
     output_full_filepath.parent.mkdir(exist_ok=True, parents=True)
@@ -122,7 +123,9 @@ def write_file(
 
     nav_reference["Pre-defined math"].append(output_file.as_posix())
 
-    math_doc = model.math_documentation.write(format="md", mkdocs_tabbed=True)
+    md_doc = math_documentation.write(format="md", mkdocs_tabbed=True)
+
+    title = math_documentation.name
     file_to_download = Path("..") / filename
     output_full_filepath.write_text(
         PREPEND_SNIPPET.format(
@@ -131,69 +134,69 @@ def write_file(
             math_type=title.lower(),
             filepath=file_to_download,
         )
-        + math_doc
+        + md_doc
     )
 
 
-def generate_base_math_model() -> calliope.Model:
-    """Generate model with documentation for the base math.
-
-    Args:
-        model_config (dict): Calliope model config.
+def generate_base_math_documentation() -> MathDocumentation:
+    """Generate model documentation for the base math.
 
     Returns:
-        calliope.Model: Base math model to use in generating math docs.
+        MathDocumentation: model math documentation with latex backend.
     """
     model = calliope.Model(model_definition=MODEL_PATH)
-    model.math_documentation.build()
-    return model
+    return MathDocumentation(model)
 
 
-def generate_custom_math_model(
-    base_model: calliope.Model, override: str
-) -> calliope.Model:
-    """Generate model with documentation for a pre-defined math file.
+def generate_custom_math_documentation(
+    base_documentation: MathDocumentation, override: str
+) -> MathDocumentation:
+    """Generate model documentation for a pre-defined math file.
 
     Only the changes made relative to the base math will be shown.
 
     Args:
-        base_model (calliope.Model): Calliope model with only the base math applied.
+        base_documentation (MathDocumentation): model documentation with only the base math applied.
         override (str): Name of override to load from the list available in the model config.
+
+    Returns:
+        MathDocumentation: model math documentation with latex backend.
     """
     model = calliope.Model(model_definition=MODEL_PATH, scenario=override)
 
     full_del = []
     expr_del = []
     for component_group, component_group_dict in model.math.items():
         for name, component_dict in component_group_dict.items():
-            if name in base_model.math[component_group]:
+            if name in base_documentation.math[component_group]:
                 if not component_dict.get("active", True):
                     expr_del.append(name)
                     component_dict["description"] = "|REMOVED|"
                     component_dict["active"] = True
-                elif base_model.math[component_group].get(name, {}) != component_dict:
+                elif (
+                    base_documentation.math[component_group].get(name, {})
+                    != component_dict
+                ):
                     _add_to_description(component_dict, "|UPDATED|")
                 else:
                     full_del.append(name)
             else:
                 _add_to_description(component_dict, "|NEW|")
 
-    model.math_documentation.build()
+    math_documentation = MathDocumentation(model)
     for key in expr_del:
-        model.math_documentation._instance._dataset[key].attrs["math_string"] = ""
+        math_documentation.backend._dataset[key].attrs["math_string"] = ""
     for key in full_del:
-        del model.math_documentation._instance._dataset[key]
-    for var in model.math_documentation._instance._dataset.values():
+        del math_documentation.backend._dataset[key]
+    for var in math_documentation.backend._dataset.values():
         var.attrs["references"] = var.attrs["references"].intersection(
-            model.math_documentation._instance._dataset.keys()
+            math_documentation.backend._dataset.keys()
         )
         var.attrs["references"] = var.attrs["references"].difference(expr_del)
 
-    logger.info(
-        model.math_documentation._instance._dataset["carrier_in"].attrs["references"]
-    )
+    logger.info(math_documentation.backend._dataset["carrier_in"].attrs["references"])
 
-    return model
+    return math_documentation
 
 
 def _add_to_description(component_dict: dict, update_string: str) -> None:

mkdocs.yml

@@ -62,6 +62,7 @@ plugins:
             - https://docs.python.org/3/objects.inv
             - https://pandas.pydata.org/docs/objects.inv
             - https://docs.xarray.dev/en/stable/objects.inv
+            - https://pyomo.readthedocs.io/en/stable/objects.inv
 markdown_extensions:
   - admonition
   - attr_list

src/calliope/attrdict.py

@@ -229,12 +229,11 @@ def set_key(self, key, value):
                 # want to overwrite so stop and warn the user
                 else:
                     raise KeyError("Cannot set nested key on non-dict key.")
+        elif key in self and isinstance(value, AttrDict):
+            for k, v in value.items():
+                self[key].set_key(k, v)
         else:
-            if key in self and isinstance(value, AttrDict):
-                for k, v in value.items():
-                    self[key].set_key(k, v)
-            else:
-                self[key] = value
+            self[key] = value
 
     def get_key(self, key, default=_MISSING):
         """Looks up the given ``key``.
@@ -258,12 +257,10 @@ def get_key(self, key, default=_MISSING):
                     return default
             else:
                 value = self[key].get_key(remainder)
+        elif default != _MISSING:
+            return self.get(key, default)
         else:
-            # Single, non-nested key of form "foo"
-            if default != _MISSING:
-                return self.get(key, default)
-            else:
-                return self[key]
+            return self[key]
         return value
 
     def del_key(self, key):
@@ -374,7 +371,6 @@ def union(
         other: Self | dict,
         allow_override: bool = False,
         allow_replacement: bool = False,
-        allow_subdict_override_with_none: bool = False,
     ):
         """In-place merge with another AttrDict.
 
@@ -384,9 +380,6 @@ def union(
                 already defined keys. Defaults to False.
             allow_replacement (bool, optional): allow "_REPLACE_" key to replace an
                 entire sub-dict. Defaults to False.
-            allow_subdict_override_with_none (bool, optional): if False, keys in the
-                form `this.that: None` in `other` will be ignored if subdicts exist in
-                self like `this.that.foo: 1`, rather than wiping them. Defaults to False.
 
         Raises:
             KeyError: `other` has an already defined key and `allow_override == False`

src/calliope/backend/__init__.py

@@ -4,24 +4,28 @@
 
 import xarray as xr
 
+from calliope.backend.backend_model import BackendSetup
 from calliope.backend.gurobi_backend_model import GurobiBackendModel
-from calliope.backend.latex_backend_model import MathDocumentation
+from calliope.backend.latex_backend_model import (
+    ALLOWED_MATH_FILE_FORMATS,
+    LatexBackendModel,
+)
 from calliope.backend.parsing import ParsedBackendComponent
 from calliope.backend.pyomo_backend_model import PyomoBackendModel
 from calliope.exceptions import BackendError
 
-MODEL_BACKENDS = ("pyomo",)
+MODEL_BACKENDS = ("pyomo", "gurobi")
 
 if TYPE_CHECKING:
     from calliope.backend.backend_model import BackendModel
 
 
-def get_model_backend(name: str, data: xr.Dataset, **kwargs) -> "BackendModel":
+def get_model_backend(name: str, setup: BackendSetup, **kwargs) -> "BackendModel":
     """Assign a backend using the given configuration.
 
     Args:
         name (str): name of the backend to use.
-        data (Dataset): model data for the backend.
+        setup (BackendSetup): standard backend inputs.
         **kwargs: backend keyword arguments corresponding to model.config.build.
 
     Raises:
@@ -32,8 +36,8 @@ def get_model_backend(name: str, data: xr.Dataset, **kwargs) -> "BackendModel":
     """
     match name:
         case "pyomo":
-            return PyomoBackendModel(data, **kwargs)
+            return PyomoBackendModel(setup, **kwargs)
         case "gurobi":
-            return GurobiBackendModel(data, **kwargs)
+            return GurobiBackendModel(setup, **kwargs)
         case _:
             raise BackendError(f"Incorrect backend '{name}' requested.")