Merge pull request #153 from pysat/combine_bugfix

Combine bugfix

CHANGELOG.md

@@ -3,10 +3,16 @@ Change Log
 All notable changes to this project will be documented in this file.
 This project adheres to [Semantic Versioning](https://semver.org/).
 
-[X.X.X] - 20XX-XX-XX
+[0.2.1] - 2024-10-XX
 --------------------
+* Enhancements
+  * Added a utility function for evaluating fill values of different types
 * Maintenance
   * Updated Ops tests to new lower limit of Python 3.9 and removed 3.6 support
+* Bugs
+  * Fixed error in mock downloading F10.7 prelim files
+  * Fixed combine_kp to consider desired time limits and fill values when
+    loading the standard dataset
 
 [0.2.0] - 2024-08-30
 --------------------

pysatSpaceWeather/instruments/methods/f107.py

@@ -17,6 +17,7 @@
 import pysat
 
 import pysatSpaceWeather as pysat_sw
+from pysatSpaceWeather.instruments.methods.general import is_fill_val
 
 
 def acknowledgements(tag):
@@ -162,7 +163,6 @@ def combine_f107(standard_inst, forecast_inst, start=None, stop=None):
 
     # Cycle through the desired time range
     itime = dt.datetime(start.year, start.month, start.day)
-
     while itime < stop and inst_flag is not None:
         # Load and save the standard data for as many times as possible
         if inst_flag == 'standard':
@@ -193,8 +193,13 @@ def combine_f107(standard_inst, forecast_inst, start=None, stop=None):
                     fill_val = f107_inst.meta['f107'][
                         f107_inst.meta.labels.fill_val]
 
-                good_vals = standard_inst['f107'][good_times] != fill_val
+                good_vals = np.array([not is_fill_val(val, fill_val) for val
+                                      in standard_inst['f107'][good_times]])
                 new_times = list(standard_inst.index[good_times][good_vals])
+            else:
+                new_times = []
+
+            if len(new_times) > 0:
                 f107_times.extend(new_times)
                 new_vals = list(standard_inst['f107'][good_times][good_vals])
                 f107_values.extend(new_vals)
@@ -237,12 +242,13 @@ def combine_f107(standard_inst, forecast_inst, start=None, stop=None):
                     # Get the good times and values
                     good_times = ((forecast_inst.index >= itime)
                                   & (forecast_inst.index < stop))
-                    good_vals = forecast_inst['f107'][good_times] != fill_val
+                    good_vals = np.array([
+                        not is_fill_val(val, fill_val) for val
+                        in forecast_inst['f107'][good_times]])
+                    new_times = list(forecast_inst.index[good_times][good_vals])
 
                 # Save desired data and cycle time
-                if len(good_vals) > 0:
-                    new_times = list(
-                        forecast_inst.index[good_times][good_vals])
+                if len(new_times) > 0:
                     f107_times.extend(new_times)
                     new_vals = list(
                         forecast_inst['f107'][good_times][good_vals])
@@ -267,8 +273,6 @@ def combine_f107(standard_inst, forecast_inst, start=None, stop=None):
     if len(f107_times) == 0:
         f107_times = date_range
 
-    date_range = pds.date_range(start=start, end=end_date, freq=freq)
-
     if date_range[0] < f107_times[0]:
         # Extend the time and value arrays from their beginning with fill
         # values

pysatSpaceWeather/instruments/methods/general.py

@@ -17,6 +17,38 @@
 import pysat
 
 
+def is_fill_val(data, fill_val):
+    """Evaluate whether or not a value is a fill value.
+
+    Parameters
+    ----------
+    data : int, float, or str
+        Data value
+    fill_val : int, float, or str
+        Fill value
+
+    Returns
+    -------
+    is_fill : bool
+        True if the data is equal to the fill value, False if it is not.
+
+    """
+
+    try:
+        # NaN and finite evaluation will fail for non-numeric types
+        if np.isnan(fill_val):
+            is_fill = np.isnan(data)
+        elif np.isfinite(fill_val):
+            is_fill = data == fill_val
+        else:
+            is_fill = ~np.isfinite(data)
+    except TypeError:
+        # Use equality for string and similar types
+        is_fill = data == fill_val
+
+    return is_fill
+
+
 def preprocess(inst):
     """Preprocess the meta data by replacing the file fill values with NaN.
 

pysatSpaceWeather/instruments/methods/kp_ap.py

@@ -16,6 +16,7 @@
 import pysat
 
 import pysatSpaceWeather as pysat_sw
+from pysatSpaceWeather.instruments.methods import general
 from pysatSpaceWeather.instruments.methods import gfz
 from pysatSpaceWeather.instruments.methods import swpc
 
@@ -600,7 +601,9 @@ def combine_kp(standard_inst=None, recent_inst=None, forecast_inst=None,
     while itime < stop and inst_flag is not None:
         # Load and save the standard data for as many times as possible
         if inst_flag == 'standard':
-            standard_inst.load(date=itime)
+            # Test to see if data loading is needed
+            if not np.any(standard_inst.index == itime):
+                standard_inst.load(date=itime)
 
             if notes.find("standard") < 0:
                 notes += " the {:} source ({:} to ".format(inst_flag,
@@ -610,9 +613,23 @@ def combine_kp(standard_inst=None, recent_inst=None, forecast_inst=None,
                 inst_flag = 'forecast' if recent_inst is None else 'recent'
                 notes += "{:})".format(itime.date())
             else:
-                kp_times.extend(list(standard_inst.index))
-                kp_values.extend(list(standard_inst['Kp']))
-                itime = kp_times[-1] + pds.DateOffset(hours=3)
+                local_fill_val = standard_inst.meta[
+                    'Kp', standard_inst.meta.labels.fill_val]
+                good_times = ((standard_inst.index >= itime)
+                              & (standard_inst.index < stop))
+                good_vals = np.array([
+                    not general.is_fill_val(val, local_fill_val)
+                    for val in standard_inst['Kp'][good_times]])
+                new_times = list(standard_inst.index[good_times][good_vals])
+
+                if len(new_times) > 0:
+                    kp_times.extend(new_times)
+                    kp_values.extend(list(
+                        standard_inst['Kp'][good_times][good_vals]))
+                    itime = kp_times[-1] + pds.DateOffset(hours=3)
+                else:
+                    inst_flag = 'forecast' if recent_inst is None else 'recent'
+                    notes += "{:})".format(itime.date())
 
         # Load and save the recent data for as many times as possible
         if inst_flag == 'recent':
@@ -637,18 +654,20 @@ def combine_kp(standard_inst=None, recent_inst=None, forecast_inst=None,
 
                 # Determine which times to save
                 if recent_inst.empty:
-                    good_vals = []
+                    new_times = []
                 else:
                     local_fill_val = recent_inst.meta[
                         'Kp', recent_inst.meta.labels.fill_val]
                     good_times = ((recent_inst.index >= itime)
                                   & (recent_inst.index < stop))
-                    good_vals = recent_inst['Kp'][good_times] != local_fill_val
+                    good_vals = np.array([
+                        not general.is_fill_val(val, local_fill_val)
+                        for val in recent_inst['Kp'][good_times]])
+                    new_times = list(recent_inst.index[good_times][good_vals])
 
                 # Save output data and cycle time
-                if len(good_vals):
-                    kp_times.extend(list(
-                        recent_inst.index[good_times][good_vals]))
+                if len(new_times) > 0:
+                    kp_times.extend(new_times)
                     kp_values.extend(list(
                         recent_inst['Kp'][good_times][good_vals]))
                     itime = kp_times[-1] + pds.DateOffset(hours=3)
@@ -683,17 +702,21 @@ def combine_kp(standard_inst=None, recent_inst=None, forecast_inst=None,
                         'Kp', forecast_inst.meta.labels.fill_val]
                     good_times = ((forecast_inst.index >= itime)
                                   & (forecast_inst.index < stop))
-                    good_vals = forecast_inst['Kp'][
-                        good_times] != local_fill_val
+                    good_vals = np.array([
+                        not general.is_fill_val(val, local_fill_val)
+                        for val in forecast_inst['Kp'][good_times]])
 
                     # Save desired data
                     new_times = list(forecast_inst.index[good_times][good_vals])
-                    kp_times.extend(new_times)
-                    new_vals = list(forecast_inst['Kp'][good_times][good_vals])
-                    kp_values.extend(new_vals)
 
-                    # Cycle time
-                    itime = kp_times[-1] + pds.DateOffset(hours=3)
+                    if len(new_times) > 0:
+                        kp_times.extend(new_times)
+                        new_vals = list(forecast_inst['Kp'][good_times][
+                            good_vals])
+                        kp_values.extend(new_vals)
+
+                        # Cycle time
+                        itime = kp_times[-1] + pds.DateOffset(hours=3)
             notes += "{:})".format(itime.date())
 
             inst_flag = None

pysatSpaceWeather/instruments/methods/swpc.py

@@ -240,9 +240,9 @@ def old_indices_dsd_download(name, date_array, data_path, local_files, today,
                 else:
                     # Set the saved filename
                     saved_fname = os.path.join(mock_download_dir, local_fname)
-                    downloaded = True
 
                     if os.path.isfile(saved_fname):
+                        downloaded = True
                         rewritten = True
                     else:
                         pysat.logger.info("".join([saved_fname, "is missing, ",
@@ -273,6 +273,7 @@ def old_indices_dsd_download(name, date_array, data_path, local_files, today,
     # Close connection after downloading all dates
     if mock_download_dir is None:
         ftp.close()
+
     return
 
 

pysatSpaceWeather/tests/test_methods_general.py

@@ -9,6 +9,7 @@
 """Integration and unit test suite for ACE methods."""
 
 import numpy as np
+import pytest
 
 import pysat
 
@@ -22,24 +23,45 @@ def setup_method(self):
         """Create a clean testing setup."""
         self.testInst = pysat.Instrument('pysat', 'testing')
         self.testInst.load(date=self.testInst.inst_module._test_dates[''][''])
+        self.var = self.testInst.variables[0]
         return
 
     def teardown_method(self):
         """Clean up previous testing setup."""
-        del self.testInst
+        del self.testInst, self.var
         return
 
     def test_preprocess(self):
         """Test the preprocessing routine updates all fill values to be NaN."""
 
         # Make sure at least one fill value is not already NaN
-        var = self.testInst.variables[0]
-        self.testInst.meta[var] = {self.testInst.meta.labels.fill_val: 0.0}
+        self.testInst.meta[self.var] = {self.testInst.meta.labels.fill_val: 0.0}
 
         # Update the meta data using the general preprocess routine
         general.preprocess(self.testInst)
 
         # Test the output
         assert np.isnan(
-            self.testInst.meta[var, self.testInst.meta.labels.fill_val])
+            self.testInst.meta[self.var, self.testInst.meta.labels.fill_val])
+        return
+
+    @pytest.mark.parametrize("fill_val", [-1.0, -1, np.nan, np.inf, ''])
+    def test_is_fill(self, fill_val):
+        """Test the successful evaluation of fill values.
+
+        Parameters
+        ----------
+        fill_val : float, int, or str
+            Fill value to use as a comparison
+
+        """
+        # Set the data value to not be a fill value
+        if fill_val != '':
+            self.var = -47
+
+        # Evaluate the variable is False
+        assert not general.is_fill_val(self.var, fill_val)
+
+        # Evaluate the fill value is a fill value
+        assert general.is_fill_val(fill_val, fill_val)
         return