Fallback for component splitting (#558)

* Fallback for component splitting

* Merge branch 'main' into bug/component-forecast-padding

Signed-off-by: Clara De Smet <clara.de.smet@alliander.com>

---------

Signed-off-by: Clara De Smet <clara.de.smet@alliander.com>

openstef/pipeline/create_component_forecast.py

@@ -108,7 +108,7 @@ def create_components_forecast_pipeline(
 
     # Make component forecasts
     try:
-        input_data = create_input(pj, input_data, weather_data)
+        dazls_input_data = create_input(pj, input_data, weather_data)
 
         # Save and load the model as .sav file (or as .z file)
         # For the code contact: korte.termijn.prognoses@alliander.com
@@ -119,13 +119,13 @@ def create_components_forecast_pipeline(
 
         # Use the predict function of Dazls model
         # As input data we use the input_data function which takes into consideration what we want as an input for the forecast and what Dazls can accept as an input
-        forecasts = dazls_model.predict(x=input_data)
+        forecasts = dazls_model.predict(x=dazls_input_data)
 
         # Set the columns for the output forecast dataframe
         forecasts = pd.DataFrame(
             forecasts,
             columns=["forecast_wind_on_shore", "forecast_solar"],
-            index=input_data.index,
+            index=dazls_input_data.index,
         )
 
         # Make post-processed forecasts for solar and wind power
@@ -140,18 +140,25 @@ def create_components_forecast_pipeline(
 
         # Make forecast for the component: "forecast_other"
         forecasts["forecast_other"] = (
-            input_data["total_load"]
+            dazls_input_data["total_load"]
             - forecasts["forecast_solar"]
             - forecasts["forecast_wind_on_shore"]
         )
+
+        # Make sure the forecasts have the same form as the input data. Pad with 0 if necessary
+        forecasts = forecasts.reindex(index=input_data.index, fill_value=0)
     except Exception as e:
-        # In case something goes wrong we fall back on aan empty dataframe
+        # In case something goes wrong we fall back on an a zero-filled dataframe
         logger.warning(
             f"Could not make component forecasts: {e}, falling back on series of"
             " zeros!",
             exc_info=e,
         )
-        forecasts = pd.DataFrame()
+        forecasts = pd.DataFrame(
+            data=0,
+            index=input_data.index,
+            columns=["forecast_wind_on_shore", "forecast_solar", "forecast_other"],
+        )
 
     # Prepare for output
     # Add more prediction properties to the forecast ("pid","customer","description","type","algtype)

test/unit/pipeline/test_create_component_forecast.py

@@ -80,6 +80,61 @@ def test_component_forecast_pipeline_happy_flow(self):
             ],
         )
 
+    def test_component_forecast_pipeline_weather_data_ends_early(self):
+        # Arrange
+        data = TestData.load("reference_sets/307-test-data.csv")
+        weather = data[["radiation", "windspeed_100m"]]
+        forecast_input = TestData.load("forecastdf_test_add_corrections.csv")
+        forecast_input["stdev"] = 0
+
+        # Shift example data to match current time interval as code expects data
+        # available relative to the current time.
+        utc_now = (
+            pd.Series(datetime.utcnow().replace(tzinfo=timezone.utc))
+            .min()
+            .round("15T")
+            .to_pydatetime()
+        )
+        most_recent_date = forecast_input.index.max().ceil("15T").to_pydatetime()
+        delta = utc_now - most_recent_date + timedelta(3)
+
+        forecast_input.index = forecast_input.index.shift(delta, freq=1)
+        most_recent_date = weather.index.max().ceil("15T").to_pydatetime()
+        delta = utc_now - most_recent_date + timedelta(3)
+        weather.index = weather.index.shift(delta, freq=1)
+
+        # Drop the last couple of rows of weather data to simulate missing data
+        weather.drop(weather.tail(30).index, inplace=True)
+
+        # Act
+        component_forecast = create_components_forecast_pipeline(
+            self.PJ, forecast_input, weather
+        )
+
+        # Assert
+        self.assertEqual(len(component_forecast), 193)
+        self.assertTrue(
+            component_forecast.tail(30)[
+                ["forecast_wind_on_shore", "forecast_solar", "forecast_other"]
+            ]
+            .eq(0)
+            .all()
+            .all()
+        )
+        self.assertEqual(
+            component_forecast.columns.to_list(),
+            [
+                "forecast_wind_on_shore",
+                "forecast_solar",
+                "forecast_other",
+                "pid",
+                "customer",
+                "description",
+                "type",
+                "algtype",
+            ],
+        )
+
     def test_component_forecast_pipeline_not_all_weather_data_available(self):
         # Test happy flow
         data = TestData.load("reference_sets/307-test-data.csv")
@@ -108,6 +163,22 @@ def test_component_forecast_pipeline_not_all_weather_data_available(self):
         # Check if the output matches expectations
         self.assertEqual(
             component_forecast.columns.to_list(),
-            ["pid", "customer", "description", "type", "algtype"],
+            [
+                "forecast_wind_on_shore",
+                "forecast_solar",
+                "forecast_other",
+                "pid",
+                "customer",
+                "description",
+                "type",
+                "algtype",
+            ],
+        )
+        self.assertTrue(
+            component_forecast[
+                ["forecast_wind_on_shore", "forecast_solar", "forecast_other"]
+            ]
+            .eq(0)
+            .all()
+            .all()
         )
-        self.assertEqual(len(component_forecast), 0)