diff --git a/CHANGELOG.md b/CHANGELOG.md index 63a6d39..cf7b3f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,10 +3,16 @@ Change Log All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](https://semver.org/). -[X.X.X] - 20XX-XX-XX +[0.2.1] - 2024-10-XX -------------------- +* Enhancements + * Added a utility function for evaluating fill values of different types * Maintenance * Updated Ops tests to new lower limit of Python 3.9 and removed 3.6 support +* Bugs + * Fixed error in mock downloading F10.7 prelim files + * Fixed combine_kp to consider desired time limits and fill values when + loading the standard dataset [0.2.0] - 2024-08-30 -------------------- diff --git a/pysatSpaceWeather/instruments/methods/f107.py b/pysatSpaceWeather/instruments/methods/f107.py index b32a5f4..648da5c 100644 --- a/pysatSpaceWeather/instruments/methods/f107.py +++ b/pysatSpaceWeather/instruments/methods/f107.py @@ -17,6 +17,7 @@ import pysat import pysatSpaceWeather as pysat_sw +from pysatSpaceWeather.instruments.methods.general import is_fill_val def acknowledgements(tag): @@ -162,7 +163,6 @@ def combine_f107(standard_inst, forecast_inst, start=None, stop=None): # Cycle through the desired time range itime = dt.datetime(start.year, start.month, start.day) - while itime < stop and inst_flag is not None: # Load and save the standard data for as many times as possible if inst_flag == 'standard': @@ -193,8 +193,13 @@ def combine_f107(standard_inst, forecast_inst, start=None, stop=None): fill_val = f107_inst.meta['f107'][ f107_inst.meta.labels.fill_val] - good_vals = standard_inst['f107'][good_times] != fill_val + good_vals = np.array([not is_fill_val(val, fill_val) for val + in standard_inst['f107'][good_times]]) new_times = list(standard_inst.index[good_times][good_vals]) + else: + new_times = [] + + if len(new_times) > 0: f107_times.extend(new_times) new_vals = list(standard_inst['f107'][good_times][good_vals]) f107_values.extend(new_vals) @@ -237,12 +242,13 @@ def combine_f107(standard_inst, forecast_inst, start=None, stop=None): # Get the good times and values good_times = ((forecast_inst.index >= itime) & (forecast_inst.index < stop)) - good_vals = forecast_inst['f107'][good_times] != fill_val + good_vals = np.array([ + not is_fill_val(val, fill_val) for val + in forecast_inst['f107'][good_times]]) + new_times = list(forecast_inst.index[good_times][good_vals]) # Save desired data and cycle time - if len(good_vals) > 0: - new_times = list( - forecast_inst.index[good_times][good_vals]) + if len(new_times) > 0: f107_times.extend(new_times) new_vals = list( forecast_inst['f107'][good_times][good_vals]) @@ -267,8 +273,6 @@ def combine_f107(standard_inst, forecast_inst, start=None, stop=None): if len(f107_times) == 0: f107_times = date_range - date_range = pds.date_range(start=start, end=end_date, freq=freq) - if date_range[0] < f107_times[0]: # Extend the time and value arrays from their beginning with fill # values diff --git a/pysatSpaceWeather/instruments/methods/general.py b/pysatSpaceWeather/instruments/methods/general.py index eccccbe..673d96c 100644 --- a/pysatSpaceWeather/instruments/methods/general.py +++ b/pysatSpaceWeather/instruments/methods/general.py @@ -17,6 +17,38 @@ import pysat +def is_fill_val(data, fill_val): + """Evaluate whether or not a value is a fill value. + + Parameters + ---------- + data : int, float, or str + Data value + fill_val : int, float, or str + Fill value + + Returns + ------- + is_fill : bool + True if the data is equal to the fill value, False if it is not. + + """ + + try: + # NaN and finite evaluation will fail for non-numeric types + if np.isnan(fill_val): + is_fill = np.isnan(data) + elif np.isfinite(fill_val): + is_fill = data == fill_val + else: + is_fill = ~np.isfinite(data) + except TypeError: + # Use equality for string and similar types + is_fill = data == fill_val + + return is_fill + + def preprocess(inst): """Preprocess the meta data by replacing the file fill values with NaN. diff --git a/pysatSpaceWeather/instruments/methods/kp_ap.py b/pysatSpaceWeather/instruments/methods/kp_ap.py index 24489be..c28bdb0 100644 --- a/pysatSpaceWeather/instruments/methods/kp_ap.py +++ b/pysatSpaceWeather/instruments/methods/kp_ap.py @@ -16,6 +16,7 @@ import pysat import pysatSpaceWeather as pysat_sw +from pysatSpaceWeather.instruments.methods import general from pysatSpaceWeather.instruments.methods import gfz from pysatSpaceWeather.instruments.methods import swpc @@ -600,7 +601,9 @@ def combine_kp(standard_inst=None, recent_inst=None, forecast_inst=None, while itime < stop and inst_flag is not None: # Load and save the standard data for as many times as possible if inst_flag == 'standard': - standard_inst.load(date=itime) + # Test to see if data loading is needed + if not np.any(standard_inst.index == itime): + standard_inst.load(date=itime) if notes.find("standard") < 0: notes += " the {:} source ({:} to ".format(inst_flag, @@ -610,9 +613,23 @@ def combine_kp(standard_inst=None, recent_inst=None, forecast_inst=None, inst_flag = 'forecast' if recent_inst is None else 'recent' notes += "{:})".format(itime.date()) else: - kp_times.extend(list(standard_inst.index)) - kp_values.extend(list(standard_inst['Kp'])) - itime = kp_times[-1] + pds.DateOffset(hours=3) + local_fill_val = standard_inst.meta[ + 'Kp', standard_inst.meta.labels.fill_val] + good_times = ((standard_inst.index >= itime) + & (standard_inst.index < stop)) + good_vals = np.array([ + not general.is_fill_val(val, local_fill_val) + for val in standard_inst['Kp'][good_times]]) + new_times = list(standard_inst.index[good_times][good_vals]) + + if len(new_times) > 0: + kp_times.extend(new_times) + kp_values.extend(list( + standard_inst['Kp'][good_times][good_vals])) + itime = kp_times[-1] + pds.DateOffset(hours=3) + else: + inst_flag = 'forecast' if recent_inst is None else 'recent' + notes += "{:})".format(itime.date()) # Load and save the recent data for as many times as possible if inst_flag == 'recent': @@ -637,18 +654,20 @@ def combine_kp(standard_inst=None, recent_inst=None, forecast_inst=None, # Determine which times to save if recent_inst.empty: - good_vals = [] + new_times = [] else: local_fill_val = recent_inst.meta[ 'Kp', recent_inst.meta.labels.fill_val] good_times = ((recent_inst.index >= itime) & (recent_inst.index < stop)) - good_vals = recent_inst['Kp'][good_times] != local_fill_val + good_vals = np.array([ + not general.is_fill_val(val, local_fill_val) + for val in recent_inst['Kp'][good_times]]) + new_times = list(recent_inst.index[good_times][good_vals]) # Save output data and cycle time - if len(good_vals): - kp_times.extend(list( - recent_inst.index[good_times][good_vals])) + if len(new_times) > 0: + kp_times.extend(new_times) kp_values.extend(list( recent_inst['Kp'][good_times][good_vals])) itime = kp_times[-1] + pds.DateOffset(hours=3) @@ -683,17 +702,21 @@ def combine_kp(standard_inst=None, recent_inst=None, forecast_inst=None, 'Kp', forecast_inst.meta.labels.fill_val] good_times = ((forecast_inst.index >= itime) & (forecast_inst.index < stop)) - good_vals = forecast_inst['Kp'][ - good_times] != local_fill_val + good_vals = np.array([ + not general.is_fill_val(val, local_fill_val) + for val in forecast_inst['Kp'][good_times]]) # Save desired data new_times = list(forecast_inst.index[good_times][good_vals]) - kp_times.extend(new_times) - new_vals = list(forecast_inst['Kp'][good_times][good_vals]) - kp_values.extend(new_vals) - # Cycle time - itime = kp_times[-1] + pds.DateOffset(hours=3) + if len(new_times) > 0: + kp_times.extend(new_times) + new_vals = list(forecast_inst['Kp'][good_times][ + good_vals]) + kp_values.extend(new_vals) + + # Cycle time + itime = kp_times[-1] + pds.DateOffset(hours=3) notes += "{:})".format(itime.date()) inst_flag = None diff --git a/pysatSpaceWeather/instruments/methods/swpc.py b/pysatSpaceWeather/instruments/methods/swpc.py index ca70e22..9e6431c 100644 --- a/pysatSpaceWeather/instruments/methods/swpc.py +++ b/pysatSpaceWeather/instruments/methods/swpc.py @@ -240,9 +240,9 @@ def old_indices_dsd_download(name, date_array, data_path, local_files, today, else: # Set the saved filename saved_fname = os.path.join(mock_download_dir, local_fname) - downloaded = True if os.path.isfile(saved_fname): + downloaded = True rewritten = True else: pysat.logger.info("".join([saved_fname, "is missing, ", @@ -273,6 +273,7 @@ def old_indices_dsd_download(name, date_array, data_path, local_files, today, # Close connection after downloading all dates if mock_download_dir is None: ftp.close() + return diff --git a/pysatSpaceWeather/tests/test_methods_general.py b/pysatSpaceWeather/tests/test_methods_general.py index eab60b2..bac18e2 100644 --- a/pysatSpaceWeather/tests/test_methods_general.py +++ b/pysatSpaceWeather/tests/test_methods_general.py @@ -9,6 +9,7 @@ """Integration and unit test suite for ACE methods.""" import numpy as np +import pytest import pysat @@ -22,24 +23,45 @@ def setup_method(self): """Create a clean testing setup.""" self.testInst = pysat.Instrument('pysat', 'testing') self.testInst.load(date=self.testInst.inst_module._test_dates['']['']) + self.var = self.testInst.variables[0] return def teardown_method(self): """Clean up previous testing setup.""" - del self.testInst + del self.testInst, self.var return def test_preprocess(self): """Test the preprocessing routine updates all fill values to be NaN.""" # Make sure at least one fill value is not already NaN - var = self.testInst.variables[0] - self.testInst.meta[var] = {self.testInst.meta.labels.fill_val: 0.0} + self.testInst.meta[self.var] = {self.testInst.meta.labels.fill_val: 0.0} # Update the meta data using the general preprocess routine general.preprocess(self.testInst) # Test the output assert np.isnan( - self.testInst.meta[var, self.testInst.meta.labels.fill_val]) + self.testInst.meta[self.var, self.testInst.meta.labels.fill_val]) + return + + @pytest.mark.parametrize("fill_val", [-1.0, -1, np.nan, np.inf, '']) + def test_is_fill(self, fill_val): + """Test the successful evaluation of fill values. + + Parameters + ---------- + fill_val : float, int, or str + Fill value to use as a comparison + + """ + # Set the data value to not be a fill value + if fill_val != '': + self.var = -47 + + # Evaluate the variable is False + assert not general.is_fill_val(self.var, fill_val) + + # Evaluate the fill value is a fill value + assert general.is_fill_val(fill_val, fill_val) return