From dcd9699b3952f370f58a6c8b9f7f80e88a847204 Mon Sep 17 00:00:00 2001
From: Bryn Pickering <17178478+brynpickering@users.noreply.github.com>
Date: Wed, 20 Nov 2024 16:25:57 +0000
Subject: [PATCH] Add `broadcast_param_data` config option and default it to
 False. (#715)

---
 CHANGELOG.md                                  |  3 +
 docs/creating/config.md                       |  4 ++
 docs/creating/parameters.md                   | 26 +++++++++
 docs/examples/national_scale/index.md         | 55 +++++++++++++------
 docs/examples/urban_scale/index.md            | 23 ++------
 docs/user_defined_math/syntax.md              |  2 +-
 src/calliope/config/config_schema.yaml        |  7 +++
 .../national_scale/data_tables/costs.csv      |  6 ++
 .../example_models/national_scale/model.yaml  | 16 +++++-
 .../national_scale/model_config/techs.yaml    | 51 +----------------
 .../example_models/urban_scale/model.yaml     |  5 ++
 .../urban_scale/model_config/techs.yaml       | 14 -----
 src/calliope/preprocess/model_data.py         |  7 +++
 tests/common/test_model/model.yaml            |  2 +
 tests/test_preprocess_model_data.py           | 13 +++++
 15 files changed, 132 insertions(+), 102 deletions(-)
 create mode 100644 src/calliope/example_models/national_scale/data_tables/costs.csv

diff --git a/CHANGELOG.md b/CHANGELOG.md
index e6b1c295..b58be7ca 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,9 @@
 
 ### User-facing changes
 
+|changed| Single data entries defined in YAML indexed parameters will not be automatically broadcast along indexed dimensions.
+To achieve the same functionality as in `<v0.7.dev4`, the user must set the new `init` configuration option `broadcast_param_data` to True (#615).
+
 |changed| Helper functions are now documented on their own page within the "Defining your own math" section of the documentation (#698).
 
 |new| `where(array, condition)` math helper function to apply a where array _inside_ an expression, to enable extending component dimensions on-the-fly, and applying filtering to different components within the expression (#604, #679).
diff --git a/docs/creating/config.md b/docs/creating/config.md
index aff62163..70b60f9d 100644
--- a/docs/creating/config.md
+++ b/docs/creating/config.md
@@ -41,6 +41,10 @@ To test your model pipeline, `config.init.time_subset` is a good way to limit yo
     Various capabilities are available to adjust the temporal resolution of a model on-the-fly, both by resampling or using externally-provided clustering.
     See our [time adjustment page](../advanced/time.md) for more details.
 
+!!! info "See also"
+    The full set of available configuration options is documented in the [configuration schema][model-configuration-schema].
+    This provides you with a description of each configuration option and the default which will be used if you do not provide a value.
+
 ## Deep-dive into some key configuration options
 
 ### `config.build.backend`
diff --git a/docs/creating/parameters.md b/docs/creating/parameters.md
index ff6906e2..20961937 100644
--- a/docs/creating/parameters.md
+++ b/docs/creating/parameters.md
@@ -53,3 +53,29 @@ Which will add the new dimension `my_new_dim` to your model: `model.inputs.my_ne
 
 !!! warning
     The `parameter` section should not be used for large datasets (e.g., indexing over the time dimension) as it will have a high memory overhead on loading the data.
+
+## Broadcasting data along indexed dimensions
+
+If you want to set the same data for all index items, you can set the `init` [configuration option](config.md) `broadcast_param_data` to True and then use a single value in `data`:
+
+=== "Without broadcasting"
+
+    ```yaml
+    my_indexed_param:
+      data: [1, 1, 1, 1]
+      index: [my_index_val1, my_index_val2, my_index_val3, my_index_val4]
+      dims: my_new_dim
+    ```
+
+=== "With broadcasting"
+
+    ```yaml
+    my_indexed_param:
+      data: 1  # All index items will take on this value
+      index: [my_index_val1, my_index_val2, my_index_val3, my_index_val4]
+      dims: my_new_dim
+    ```
+
+!!! warning
+    The danger of broadcasting is that you maybe update `index` as a scenario override without realising that the data will be broadcast over this new index.
+    E.g., if you start with `!#yaml {data: 1, index: monetary, dims: costs}` and update it with `!#yaml {index: [monetary, emissions]}` then the `data` value of `1` will be set for both `monetary` and `emissions` index values.
diff --git a/docs/examples/national_scale/index.md b/docs/examples/national_scale/index.md
index 25a1b46d..dc4bfe28 100644
--- a/docs/examples/national_scale/index.md
+++ b/docs/examples/national_scale/index.md
@@ -1,3 +1,10 @@
+---
+costs:
+    file: "src/calliope/example_models/national_scale/data_tables/costs.csv"
+    header: [0, 1]
+    index_col: 0
+---
+
 # National Scale Example Model
 
 This example consists of two possible power supply technologies,
@@ -25,6 +32,29 @@ It does not contain much data, but the scaffolding with which to construct and r
 
 ## Model definition
 
+### Referencing tabular data
+
+As of Calliope v0.7.0 it is possible to load tabular data completely separately from the YAML model definition.
+To do this we reference data tables under the `data_tables` key:
+
+```yaml
+--8<-- "src/calliope/example_models/national_scale/model.yaml:data-tables"
+```
+
+In the Calliope national scale example model, we load both timeseries and cost data from file.
+As an example, the data in the cost CSV file looks like this:
+
+{{ read_csv(page.meta.costs.file, header=page.meta.costs.header, index_col=page.meta.costs.index_col) }}
+
+You'll notice that in each row there is reference to a technology, and in each column to a cost parameter and a comment on the units being used.
+Therefore, we reference `techs` in our data table _rows_, and `parameters` and `comment` in our data table _columns_.
+The `comment` information is metadata that we don't need in our Calliope model object, so we _drop_ it on loading the table.
+Since all the data refers to the one cost class `monetary`, we don't add that information in the CSV file, but instead add it on as a dimension when loading the file.
+Where there is no data for that combination of technology and cost parameter, the value is Not-a-Number (NaN) and this combination will be ignored on loading the table.
+
+!!! info
+    You can read more about loading data from file in [our dedicated tutorial][loading-tabular-data].
+
 ### Indexed parameters
 
 Before we dive into the technologies and nodes in the model, we have defined some parameters that are independent of both of these:
@@ -110,23 +140,6 @@ The costs are more numerous as well, and include monetary costs for all relevant
 * carrier conversion capacity
 * variable operational and maintenance costs
 
-### Interlude: inheriting from templates
-
-You will notice that the above technologies _inherit_ `cost_dim_setter`.
-Templates allow us to avoid excessive repetition in our model definition.
-In this case, `cost_dim_setter` defines the dimension and index of costs, allowing us to keep our definition of technology costs to only defining `data`.
-By defining `data`, the technologies override the `null` setting applied by `cost_dim_setter`.
-We also use it to set the `interest_rate` for all technologies, which will be used to annualise any investment costs each technology defines.
-
-Technologies and nodes can inherit from anything defined in `templates`.
-items in `templates` can also inherit from each other, so you can create inheritance chains.
-
-`cost_dim_setter` looks like this:
-
-```yaml
---8<-- "src/calliope/example_models/national_scale/model_config/techs.yaml:cost-dim-setter"
-```
-
 ### Storage technologies
 
 The second location allows a limited amount of battery storage to be deployed to better balance the system.
@@ -184,8 +197,16 @@ Transmission technologies look different to other technologies, as they link the
 `free_transmission` allows local power transmission from any of the csp facilities to the nearest location.
 As the name suggests, it applies no cost or efficiency losses to this transmission.
 
+### Interlude: inheriting from templates
+
 We can see that those technologies which rely on `free_transmission` inherit a lot of this information from elsewhere in the model definition.
 `free_transmission` is defined in `templates`, which makes it inheritable.
+[Templates](../../creating/templates.md) allow us to avoid excessive repetition in our model definition.
+
+Technologies and nodes can inherit from anything defined in `templates`.
+items in `templates` can also inherit from each other, so you can create inheritance chains.
+
+The `free_transmission` template looks like this:
 
 ```yaml
 --8<-- "src/calliope/example_models/national_scale/model_config/techs.yaml:free-transmission"
diff --git a/docs/examples/urban_scale/index.md b/docs/examples/urban_scale/index.md
index 499c9932..241385ac 100644
--- a/docs/examples/urban_scale/index.md
+++ b/docs/examples/urban_scale/index.md
@@ -47,6 +47,8 @@ The import section in our file looks like this:
 --8<-- "src/calliope/example_models/urban_scale/model.yaml:import"
 ```
 
+## Model definition
+
 ### Referencing tabular data
 
 As of Calliope v0.7.0 it is possible to load tabular data completely separately from the YAML model definition.
@@ -56,7 +58,7 @@ To do this we reference data tables under the `data_tables` key:
 --8<-- "src/calliope/example_models/urban_scale/model.yaml:data-tables"
 ```
 
-In the Calliope example models, we only load timeseries data from file, including for [energy demand](#demand-technologies), [electricity export price](#revenue-by-export) and [solar PV resource availability](#supply-technologies).
+In the Calliope urban scale example model, we only load timeseries data from file, including for [energy demand](#demand-technologies), [electricity export price](#revenue-by-export) and [solar PV resource availability](#supply-technologies).
 These are large tables of data that do not work well in YAML files!
 As an example, the data in the energy demand CSV file looks like this:
 
@@ -69,8 +71,6 @@ Since all the data refers to the one parameter `sink_use_equals`, we don't add t
 !!! info
     You can read more about loading data from file in [our dedicated tutorial][loading-tabular-data].
 
-## Model definition
-
 ### Indexed parameters
 
 Before we dive into the technologies and nodes in the model, we have defined some parameters that are independent of both of these:
@@ -127,21 +127,6 @@ The definition of this technology in the example model's configuration looks as
 --8<-- "src/calliope/example_models/urban_scale/model_config/techs.yaml:pv"
 ```
 
-### Interlude: inheriting from templates
-
-You will notice that the above technologies _inherit_ `interest_rate_setter`.
-Templates allow us to avoid excessive repetition in our model definition.
-In this case, `interest_rate_setter` defines an interest rate that will be used to annualise any investment costs the technology defines.
-
-Technologies / nodes can inherit from anything defined in `templates`.
-items in `templates` can also inherit from each other, so you can create inheritance chains.
-
-`interest_rate_setter` looks like this:
-
-```yaml
---8<-- "src/calliope/example_models/urban_scale/model_config/techs.yaml:interest-rate-setter"
-```
-
 ### Conversion technologies
 
 The example model defines two conversion technologies.
@@ -241,7 +226,7 @@ Gas is made available in each node without consideration of transmission.
 --8<-- "src/calliope/example_models/urban_scale/model_config/techs.yaml:transmission"
 ```
 
-To avoid excessive duplication in model definition, our transmission technologies inherit most of the their parameters from _templates_:
+To avoid excessive duplication in model definition, our transmission technologies inherit most of the their parameters from [templates](../../creating/templates.md):
 
 ```yaml
 --8<-- "src/calliope/example_models/urban_scale/model_config/techs.yaml:transmission-templates"
diff --git a/docs/user_defined_math/syntax.md b/docs/user_defined_math/syntax.md
index afdf91e6..955c4ac6 100644
--- a/docs/user_defined_math/syntax.md
+++ b/docs/user_defined_math/syntax.md
@@ -210,7 +210,7 @@ If you define a lookup parameter "lookup_techs" as:
 ```yaml
 parameters:
   lookup_techs:
-    data: True
+    data: [True, True]
     index: [tech_1, tech_2]
     dims: [techs]
 ```
diff --git a/src/calliope/config/config_schema.yaml b/src/calliope/config/config_schema.yaml
index b9ebe627..96aedfe0 100644
--- a/src/calliope/config/config_schema.yaml
+++ b/src/calliope/config/config_schema.yaml
@@ -58,6 +58,13 @@ properties:
               Unit of transmission link `distance` (m - metres, km - kilometres).
               Automatically derived distances from lat/lon coordinates will be given in this unit.
             enum: [m, km]
+          broadcast_param_data:
+            type: boolean
+            default: false
+            description:
+              If True, single data entries in YAML indexed parameters will be broadcast across all index items.
+              If False, the number of data entries in an indexed parameter needs to match the number of index items.
+              Defaults to False to mitigate unexpected broadcasting when applying overrides.
 
       build:
         type: object
diff --git a/src/calliope/example_models/national_scale/data_tables/costs.csv b/src/calliope/example_models/national_scale/data_tables/costs.csv
new file mode 100644
index 00000000..6815233d
--- /dev/null
+++ b/src/calliope/example_models/national_scale/data_tables/costs.csv
@@ -0,0 +1,6 @@
+parameters,cost_flow_cap,cost_storage_cap,cost_area_use,cost_source_cap,cost_flow_in,cost_flow_out
+comment,USD per kW,USD per kWh storage capacity,USD per m2,USD per kW,USD per kWh,USD per kWh
+ccgt,750,,,,0.02,
+csp,1000,50,200,200,,0.002
+battery,,200,,,,
+region1_to_region2,200,,,,,0.002
diff --git a/src/calliope/example_models/national_scale/model.yaml b/src/calliope/example_models/national_scale/model.yaml
index 654a7194..153fdea7 100644
--- a/src/calliope/example_models/national_scale/model.yaml
+++ b/src/calliope/example_models/national_scale/model.yaml
@@ -13,6 +13,7 @@ config:
     # What version of Calliope this model is intended for
     calliope_version: 0.7.0
     time_subset: ["2005-01-01", "2005-01-05"] # Subset of timesteps
+    broadcast_param_data: true  # allow single indexed parameter data entries to be broadcast across all index items, if there are multiple entries.
 
   build:
     ensure_feasibility: true # Switches on the "unmet demand" constraint
@@ -27,15 +28,28 @@ config:
 parameters:
   objective_cost_weights:
     data: 1
-    index: [monetary]
+    index: monetary
     dims: costs
   # `bigM` sets the scale of unmet demand, which cannot be too high, otherwise the optimisation will not converge
   bigM: 1e6
+  cost_interest_rate:
+    data: 0.10
+    index: monetary
+    dims: costs
 # --8<-- [end:parameters]
 
+# --8<-- [start:data-tables]
 data_tables:
   time_varying_parameters:
     data: data_tables/time_varying_params.csv
     rows: timesteps
     columns: [comment, nodes, techs, parameters]
     drop: comment
+  cost_parameters:
+    data: data_tables/costs.csv
+    rows: techs
+    columns: [parameters, comment]
+    drop: comment
+    add_dims:
+      costs: monetary
+# --8<-- [end:data-tables]
diff --git a/src/calliope/example_models/national_scale/model_config/techs.yaml b/src/calliope/example_models/national_scale/model_config/techs.yaml
index d0b481e4..6a0b7b68 100644
--- a/src/calliope/example_models/national_scale/model_config/techs.yaml
+++ b/src/calliope/example_models/national_scale/model_config/techs.yaml
@@ -4,40 +4,8 @@
 
 # Note: --8<--start:'' and --8<--end:'' is used in tutorial documentation only
 
-# --8<-- [start:cost-dim-setter]
-templates:
-  cost_dim_setter:
-    cost_flow_cap:
-      data: null
-      index: monetary
-      dims: costs
-    cost_flow_in:
-      data: null
-      index: monetary
-      dims: costs
-    cost_flow_out:
-      data: null
-      index: monetary
-      dims: costs
-    cost_storage_cap:
-      data: null
-      index: monetary
-      dims: costs
-    cost_area_use:
-      data: null
-      index: monetary
-      dims: costs
-    cost_source_cap:
-      data: null
-      index: monetary
-      dims: costs
-    cost_interest_rate:
-      data: 0.10
-      index: monetary
-      dims: costs
-# --8<-- [end:cost-dim-setter]
-
 # --8<-- [start:free-transmission]
+templates:
   free_transmission:
     name: "Local power transmission"
     color: "#6783E3"
@@ -56,16 +24,12 @@ techs:
     name: "Combined cycle gas turbine"
     color: "#E37A72"
     base_tech: supply
-    template: cost_dim_setter
     carrier_out: power
     flow_out_eff: 0.5
     flow_cap_max: 40000 # kW
     flow_cap_max_systemwide: 100000 # kW
     flow_ramping: 0.8
     lifetime: 25
-
-    cost_flow_cap.data: 750 # USD per kW
-    cost_flow_in.data: 0.02 # USD per kWh
   # --8<-- [end:ccgt]
 
   # --8<-- [start:csp]
@@ -73,7 +37,6 @@ techs:
     name: "Concentrating solar power"
     color: "#F9CF22"
     base_tech: supply
-    template: cost_dim_setter
     carrier_out: power
     source_unit: per_area
     include_storage: True
@@ -85,12 +48,6 @@ techs:
     area_use_max: .inf
     flow_cap_max: 10000
     lifetime: 25
-
-    cost_storage_cap.data: 50
-    cost_area_use.data: 200
-    cost_source_cap.data: 200
-    cost_flow_cap.data: 1000
-    cost_flow_out.data: 0.002
   # --8<-- [end:csp]
 
   ##
@@ -101,7 +58,6 @@ techs:
     name: "Battery storage"
     color: "#3B61E3"
     base_tech: storage
-    template: cost_dim_setter
     carrier_in: power
     carrier_out: power
     flow_cap_max: 1000 # kW
@@ -112,8 +68,6 @@ techs:
     flow_in_eff: 0.95
     storage_loss: 0 # No loss over time assumed
     lifetime: 25
-
-    cost_storage_cap.data: 200 # USD per kWh storage capacity
   # --8<-- [end:battery]
 
   ##
@@ -139,13 +93,10 @@ techs:
     name: "AC power transmission"
     color: "#8465A9"
     base_tech: transmission
-    template: cost_dim_setter
     carrier_in: power
     carrier_out: power
     flow_out_eff: 0.85
     lifetime: 25
-    cost_flow_cap.data: 200
-    cost_flow_out.data: 0.002
     flow_cap_max: 10000
 
   region1_to_region1_1:
diff --git a/src/calliope/example_models/urban_scale/model.yaml b/src/calliope/example_models/urban_scale/model.yaml
index e56c1302..9583945e 100644
--- a/src/calliope/example_models/urban_scale/model.yaml
+++ b/src/calliope/example_models/urban_scale/model.yaml
@@ -13,6 +13,7 @@ config:
     calliope_version: 0.7.0
     # Time series data path - can either be a path relative to this file, or an absolute path
     time_subset: ["2005-07-01", "2005-07-02"] # Subset of timesteps
+    broadcast_param_data: true  # allow single indexed parameter data entries to be broadcast across all index items, if there are multiple entries.
 
   build:
     mode: plan # Choices: plan, operate
@@ -31,6 +32,10 @@ parameters:
     dims: costs
   # `bigM` sets the scale of unmet demand, which cannot be too high, otherwise the optimisation will not converge
   bigM: 1e6
+  cost_interest_rate:
+    data: 0.10
+    index: monetary
+    dims: costs
 # --8<-- [end:parameters]
 
 # --8<-- [start:data-tables]
diff --git a/src/calliope/example_models/urban_scale/model_config/techs.yaml b/src/calliope/example_models/urban_scale/model_config/techs.yaml
index 50e29c9b..8b1478db 100644
--- a/src/calliope/example_models/urban_scale/model_config/techs.yaml
+++ b/src/calliope/example_models/urban_scale/model_config/techs.yaml
@@ -4,20 +4,12 @@
 
 # --8<-- [start:--8<-- [end:Note: ']' and ']' is used in tutorial documentation only
 
-# --8<-- [start:interest-rate-setter]
 templates:
-  interest_rate_setter:
-    cost_interest_rate:
-      data: 0.10
-      index: monetary
-      dims: costs
-# --8<-- [end:interest-rate-setter]
 # --8<-- [start:transmission-templates]
   power_lines:
     name: "Electrical power distribution"
     color: "#6783E3"
     base_tech: transmission
-    template: interest_rate_setter
     carrier_in: electricity
     carrier_out: electricity
     flow_cap_max: 2000
@@ -32,7 +24,6 @@ templates:
     name: "District heat distribution"
     color: "#823739"
     base_tech: transmission
-    template: interest_rate_setter
     carrier_in: heat
     carrier_out: heat
     flow_cap_max: 2000
@@ -50,7 +41,6 @@ techs:
     name: "National grid import"
     color: "#C5ABE3"
     base_tech: supply
-    template: interest_rate_setter
     carrier_out: electricity
     source_use_max: .inf
     flow_cap_max: 2000
@@ -68,7 +58,6 @@ techs:
     name: "Natural gas import"
     color: "#C98AAD"
     base_tech: supply
-    template: interest_rate_setter
     carrier_out: gas
     source_use_max: .inf
     flow_cap_max: 2000
@@ -90,7 +79,6 @@ techs:
     color: "#F9D956"
     base_tech: supply
     carrier_out: electricity
-    template: interest_rate_setter
     carrier_export: electricity
     source_unit: per_area
     area_use_per_flow_cap: 7 # 7m2 of panels needed to fit 1kWp of panels
@@ -111,7 +99,6 @@ techs:
     name: "Natural gas boiler"
     color: "#8E2999"
     base_tech: conversion
-    template: interest_rate_setter
     carrier_in: gas
     carrier_out: heat
     flow_cap_max:
@@ -132,7 +119,6 @@ techs:
     name: "Combined heat and power"
     color: "#E4AB97"
     base_tech: conversion
-    template: interest_rate_setter
     carrier_in: gas
     carrier_out: [electricity, heat]
     carrier_export: electricity
diff --git a/src/calliope/preprocess/model_data.py b/src/calliope/preprocess/model_data.py
index 7c6d6cc3..2fa0c3fe 100644
--- a/src/calliope/preprocess/model_data.py
+++ b/src/calliope/preprocess/model_data.py
@@ -486,6 +486,13 @@ def _prepare_param_dict(
             data = param_data["data"]
             index_items = [listify(idx) for idx in listify(param_data["index"])]
             dims = listify(param_data["dims"])
+            broadcast_param_data = self.config.broadcast_param_data
+            if not broadcast_param_data and len(listify(data)) != len(index_items):
+                raise exceptions.ModelError(
+                    f"{param_name} | Length mismatch between data ({data}) and index ({index_items}) for parameter definition. "
+                    "Check lengths of arrays or set `config.init.broadcast_param_data` to True "
+                    "to allow single data entries to be broadcast across all parameter index items."
+                )
         elif param_name in self.LOOKUP_PARAMS.keys():
             data = True
             index_items = [[i] for i in listify(param_data)]
diff --git a/tests/common/test_model/model.yaml b/tests/common/test_model/model.yaml
index 9eef09c5..1afe91c0 100644
--- a/tests/common/test_model/model.yaml
+++ b/tests/common/test_model/model.yaml
@@ -15,6 +15,8 @@ config:
   init:
     name: Test model
     time_subset: ["2005-01-01", "2005-01-02"]
+    broadcast_param_data: true
+
   build:
     mode: plan
   solve:
diff --git a/tests/test_preprocess_model_data.py b/tests/test_preprocess_model_data.py
index 48bc519c..c6036087 100644
--- a/tests/test_preprocess_model_data.py
+++ b/tests/test_preprocess_model_data.py
@@ -431,6 +431,19 @@ def test_prepare_param_dict_not_lookup(self, model_data_factory: ModelDataFactor
             "foo | Cannot pass parameter data as a list unless the parameter is one of the pre-defined lookup arrays",
         )
 
+    @pytest.mark.parametrize("param_data", [1, [1], [1, 2, 3]])
+    def test_prepare_param_dict_no_broadcast_allowed(
+        self, model_data_factory, param_data
+    ):
+        model_data_factory.config.broadcast_param_data = False
+        param_dict = {"data": param_data, "index": [["foo"], ["bar"]], "dims": "foobar"}
+        with pytest.raises(exceptions.ModelError) as excinfo:  # noqa: PT011, false positive
+            model_data_factory._prepare_param_dict("foo", param_dict)
+        assert check_error_or_warning(
+            excinfo,
+            f"foo | Length mismatch between data ({param_data}) and index ([['foo'], ['bar']]) for parameter definition",
+        )
+
     def test_template_defs_inactive(
         self, my_caplog, model_data_factory: ModelDataFactory
     ):