From 4477ee10e8de16d2b57b16ce919d4d456b62fcdb Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Tue, 1 Oct 2024 18:12:50 -0400 Subject: [PATCH 01/11] BUG: update f107 fill value treatment Ensure fill value evaluation works for NaN. --- pysatSpaceWeather/instruments/methods/f107.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/pysatSpaceWeather/instruments/methods/f107.py b/pysatSpaceWeather/instruments/methods/f107.py index b32a5f4..6bef664 100644 --- a/pysatSpaceWeather/instruments/methods/f107.py +++ b/pysatSpaceWeather/instruments/methods/f107.py @@ -17,6 +17,7 @@ import pysat import pysatSpaceWeather as pysat_sw +from pysatSpaceWeather.instruments.methods.general import is_fill_val def acknowledgements(tag): @@ -162,7 +163,6 @@ def combine_f107(standard_inst, forecast_inst, start=None, stop=None): # Cycle through the desired time range itime = dt.datetime(start.year, start.month, start.day) - while itime < stop and inst_flag is not None: # Load and save the standard data for as many times as possible if inst_flag == 'standard': @@ -193,8 +193,13 @@ def combine_f107(standard_inst, forecast_inst, start=None, stop=None): fill_val = f107_inst.meta['f107'][ f107_inst.meta.labels.fill_val] - good_vals = standard_inst['f107'][good_times] != fill_val + good_vals = np.array([~is_fill_val(val, fill_val) for val + in standard_inst['f107'][good_times]]) new_times = list(standard_inst.index[good_times][good_vals]) + else: + new_times = [] + + if len(new_times) > 0: f107_times.extend(new_times) new_vals = list(standard_inst['f107'][good_times][good_vals]) f107_values.extend(new_vals) @@ -237,7 +242,9 @@ def combine_f107(standard_inst, forecast_inst, start=None, stop=None): # Get the good times and values good_times = ((forecast_inst.index >= itime) & (forecast_inst.index < stop)) - good_vals = forecast_inst['f107'][good_times] != fill_val + good_vals = np.array([ + ~is_fill_val(val, fill_val) for val + in forecast_inst['f107'][good_times]]) # Save desired data and cycle time if len(good_vals) > 0: From ede7999a1bbab87adda6a106384ae0dd35aa4c7d Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Tue, 1 Oct 2024 18:13:25 -0400 Subject: [PATCH 02/11] ENH: added a fill value evaluation function Added a function to evaluate fill values for normal and special values (e.g., NaN and Inf). --- .../instruments/methods/general.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/pysatSpaceWeather/instruments/methods/general.py b/pysatSpaceWeather/instruments/methods/general.py index eccccbe..90dd0fb 100644 --- a/pysatSpaceWeather/instruments/methods/general.py +++ b/pysatSpaceWeather/instruments/methods/general.py @@ -17,6 +17,33 @@ import pysat +def is_fill_val(data, fill_val): + """Evaluate whether or not a value is a fill value. + + Parameters + ---------- + data : int, float, or str + Data value + fill_val : int, float, or str + Fill value + + Returns + ------- + is_fill : bool + True if the data is equal to the fill value, False if it is not. + + """ + + if np.isnan(fill_val): + is_fill = np.isnan(data) + elif np.isfinite(fill_val): + is_fill = data == fill_val + else: + is_fill = ~np.isfinite(data) + + return is_fill + + def preprocess(inst): """Preprocess the meta data by replacing the file fill values with NaN. From 680095f8f1dfa8841aca55786f77b88684d39050 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Tue, 1 Oct 2024 18:14:03 -0400 Subject: [PATCH 03/11] BUG: fix mock download file cycling Ensure mock downloads test for both versions of potential F10.7 prelim file names. --- pysatSpaceWeather/instruments/methods/swpc.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pysatSpaceWeather/instruments/methods/swpc.py b/pysatSpaceWeather/instruments/methods/swpc.py index ca70e22..9e6431c 100644 --- a/pysatSpaceWeather/instruments/methods/swpc.py +++ b/pysatSpaceWeather/instruments/methods/swpc.py @@ -240,9 +240,9 @@ def old_indices_dsd_download(name, date_array, data_path, local_files, today, else: # Set the saved filename saved_fname = os.path.join(mock_download_dir, local_fname) - downloaded = True if os.path.isfile(saved_fname): + downloaded = True rewritten = True else: pysat.logger.info("".join([saved_fname, "is missing, ", @@ -273,6 +273,7 @@ def old_indices_dsd_download(name, date_array, data_path, local_files, today, # Close connection after downloading all dates if mock_download_dir is None: ftp.close() + return From a85380997d42e6825229e0ff1bce742aa7d46bb1 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Tue, 1 Oct 2024 18:14:33 -0400 Subject: [PATCH 04/11] BUG: fixing combine_kp bugs Partial fix for combine_kp bugs. --- .../instruments/methods/kp_ap.py | 57 +++++++++++++------ 1 file changed, 41 insertions(+), 16 deletions(-) diff --git a/pysatSpaceWeather/instruments/methods/kp_ap.py b/pysatSpaceWeather/instruments/methods/kp_ap.py index 24489be..817c893 100644 --- a/pysatSpaceWeather/instruments/methods/kp_ap.py +++ b/pysatSpaceWeather/instruments/methods/kp_ap.py @@ -16,6 +16,7 @@ import pysat import pysatSpaceWeather as pysat_sw +from pysatSpaceWeather.instruments.methods import general from pysatSpaceWeather.instruments.methods import gfz from pysatSpaceWeather.instruments.methods import swpc @@ -600,7 +601,9 @@ def combine_kp(standard_inst=None, recent_inst=None, forecast_inst=None, while itime < stop and inst_flag is not None: # Load and save the standard data for as many times as possible if inst_flag == 'standard': - standard_inst.load(date=itime) + # Test to see if data loading is needed + if not np.any(standard_inst.index == itime): + standard_inst.load(date=itime) if notes.find("standard") < 0: notes += " the {:} source ({:} to ".format(inst_flag, @@ -610,9 +613,23 @@ def combine_kp(standard_inst=None, recent_inst=None, forecast_inst=None, inst_flag = 'forecast' if recent_inst is None else 'recent' notes += "{:})".format(itime.date()) else: - kp_times.extend(list(standard_inst.index)) - kp_values.extend(list(standard_inst['Kp'])) - itime = kp_times[-1] + pds.DateOffset(hours=3) + local_fill_val = standard_inst.meta[ + 'Kp', standard_inst.meta.labels.fill_val] + good_times = ((standard_inst.index >= itime) + & (standard_inst.index < stop)) + good_vals = np.array([ + ~general.is_fill_val(val, local_fill_val) + for val in standard_inst['Kp'][good_times]]) + new_times = list(standard_inst.index[good_times][good_vals]) + + if len(new_times) > 0: + kp_times.extend(new_times) + kp_values.extend(list( + standard_inst['Kp'][good_times][good_vals])) + itime = kp_times[-1] + pds.DateOffset(hours=3) + else: + inst_flag = 'forecast' if recent_inst is None else 'recent' + notes += "{:})".format(itime.date()) # Load and save the recent data for as many times as possible if inst_flag == 'recent': @@ -637,18 +654,20 @@ def combine_kp(standard_inst=None, recent_inst=None, forecast_inst=None, # Determine which times to save if recent_inst.empty: - good_vals = [] + new_times = [] else: local_fill_val = recent_inst.meta[ 'Kp', recent_inst.meta.labels.fill_val] good_times = ((recent_inst.index >= itime) & (recent_inst.index < stop)) - good_vals = recent_inst['Kp'][good_times] != local_fill_val + good_vals = np.array([ + ~general.is_fill_val(val, local_fill_val) + for val in recent_inst['Kp'][good_times]]) + new_times = list(recent_inst.index[good_times][good_vals]) # Save output data and cycle time - if len(good_vals): - kp_times.extend(list( - recent_inst.index[good_times][good_vals])) + if len(new_times) > 0: + kp_times.extend(new_times) kp_values.extend(list( recent_inst['Kp'][good_times][good_vals])) itime = kp_times[-1] + pds.DateOffset(hours=3) @@ -683,17 +702,21 @@ def combine_kp(standard_inst=None, recent_inst=None, forecast_inst=None, 'Kp', forecast_inst.meta.labels.fill_val] good_times = ((forecast_inst.index >= itime) & (forecast_inst.index < stop)) - good_vals = forecast_inst['Kp'][ - good_times] != local_fill_val + good_vals = np.array([ + ~general.is_fill_val(val, local_fill_val) + for val in forecast_inst['Kp'][good_times]]) # Save desired data new_times = list(forecast_inst.index[good_times][good_vals]) - kp_times.extend(new_times) - new_vals = list(forecast_inst['Kp'][good_times][good_vals]) - kp_values.extend(new_vals) - # Cycle time - itime = kp_times[-1] + pds.DateOffset(hours=3) + if len(new_times) > 0: + kp_times.extend(new_times) + new_vals = list(forecast_inst['Kp'][good_times][ + good_vals]) + kp_values.extend(new_vals) + + # Cycle time + itime = kp_times[-1] + pds.DateOffset(hours=3) notes += "{:})".format(itime.date()) inst_flag = None @@ -709,6 +732,8 @@ def combine_kp(standard_inst=None, recent_inst=None, forecast_inst=None, if len(kp_times) == 0: kp_times = date_range + raise RuntimeError(start, end_date, stop, freq, date_range) + if date_range[0] < kp_times[0]: # Extend the time and value arrays from their beginning with fill # values From f6acd82b7574207985b4f423969281a455568d7f Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Wed, 2 Oct 2024 11:49:57 -0400 Subject: [PATCH 05/11] BUG: fixed boolean swap Fixed boolean True/False swap. --- pysatSpaceWeather/instruments/methods/f107.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pysatSpaceWeather/instruments/methods/f107.py b/pysatSpaceWeather/instruments/methods/f107.py index 6bef664..4e6b368 100644 --- a/pysatSpaceWeather/instruments/methods/f107.py +++ b/pysatSpaceWeather/instruments/methods/f107.py @@ -193,7 +193,7 @@ def combine_f107(standard_inst, forecast_inst, start=None, stop=None): fill_val = f107_inst.meta['f107'][ f107_inst.meta.labels.fill_val] - good_vals = np.array([~is_fill_val(val, fill_val) for val + good_vals = np.array([not is_fill_val(val, fill_val) for val in standard_inst['f107'][good_times]]) new_times = list(standard_inst.index[good_times][good_vals]) else: @@ -243,7 +243,7 @@ def combine_f107(standard_inst, forecast_inst, start=None, stop=None): good_times = ((forecast_inst.index >= itime) & (forecast_inst.index < stop)) good_vals = np.array([ - ~is_fill_val(val, fill_val) for val + not is_fill_val(val, fill_val) for val in forecast_inst['f107'][good_times]]) # Save desired data and cycle time From 9775d6b9e436ff7d8898f393661151f098816e0e Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Wed, 2 Oct 2024 11:50:22 -0400 Subject: [PATCH 06/11] BUG: fixed combine_kp Fixed the boolean swap logic and removed debug statements. --- pysatSpaceWeather/instruments/methods/kp_ap.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/pysatSpaceWeather/instruments/methods/kp_ap.py b/pysatSpaceWeather/instruments/methods/kp_ap.py index 817c893..2838664 100644 --- a/pysatSpaceWeather/instruments/methods/kp_ap.py +++ b/pysatSpaceWeather/instruments/methods/kp_ap.py @@ -618,7 +618,7 @@ def combine_kp(standard_inst=None, recent_inst=None, forecast_inst=None, good_times = ((standard_inst.index >= itime) & (standard_inst.index < stop)) good_vals = np.array([ - ~general.is_fill_val(val, local_fill_val) + not general.is_fill_val(val, local_fill_val) for val in standard_inst['Kp'][good_times]]) new_times = list(standard_inst.index[good_times][good_vals]) @@ -661,7 +661,7 @@ def combine_kp(standard_inst=None, recent_inst=None, forecast_inst=None, good_times = ((recent_inst.index >= itime) & (recent_inst.index < stop)) good_vals = np.array([ - ~general.is_fill_val(val, local_fill_val) + not general.is_fill_val(val, local_fill_val) for val in recent_inst['Kp'][good_times]]) new_times = list(recent_inst.index[good_times][good_vals]) @@ -703,7 +703,7 @@ def combine_kp(standard_inst=None, recent_inst=None, forecast_inst=None, good_times = ((forecast_inst.index >= itime) & (forecast_inst.index < stop)) good_vals = np.array([ - ~general.is_fill_val(val, local_fill_val) + not general.is_fill_val(val, local_fill_val) for val in forecast_inst['Kp'][good_times]]) # Save desired data @@ -732,8 +732,6 @@ def combine_kp(standard_inst=None, recent_inst=None, forecast_inst=None, if len(kp_times) == 0: kp_times = date_range - raise RuntimeError(start, end_date, stop, freq, date_range) - if date_range[0] < kp_times[0]: # Extend the time and value arrays from their beginning with fill # values From a6f83754af1167b0c75bd204eb78594a8dbb6bc5 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Wed, 2 Oct 2024 12:10:37 -0400 Subject: [PATCH 07/11] BUG: fixed string implementation Fixed the `is_fill_val` implementation for string types. --- pysatSpaceWeather/instruments/methods/general.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/pysatSpaceWeather/instruments/methods/general.py b/pysatSpaceWeather/instruments/methods/general.py index 90dd0fb..673d96c 100644 --- a/pysatSpaceWeather/instruments/methods/general.py +++ b/pysatSpaceWeather/instruments/methods/general.py @@ -34,12 +34,17 @@ def is_fill_val(data, fill_val): """ - if np.isnan(fill_val): - is_fill = np.isnan(data) - elif np.isfinite(fill_val): + try: + # NaN and finite evaluation will fail for non-numeric types + if np.isnan(fill_val): + is_fill = np.isnan(data) + elif np.isfinite(fill_val): + is_fill = data == fill_val + else: + is_fill = ~np.isfinite(data) + except TypeError: + # Use equality for string and similar types is_fill = data == fill_val - else: - is_fill = ~np.isfinite(data) return is_fill From 3e00d8ba5a60117c7cd7370d98cc69fccd872380 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Wed, 2 Oct 2024 12:10:57 -0400 Subject: [PATCH 08/11] STY: fixed indentation Fixed under-indented line. --- pysatSpaceWeather/instruments/methods/kp_ap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatSpaceWeather/instruments/methods/kp_ap.py b/pysatSpaceWeather/instruments/methods/kp_ap.py index 2838664..c28bdb0 100644 --- a/pysatSpaceWeather/instruments/methods/kp_ap.py +++ b/pysatSpaceWeather/instruments/methods/kp_ap.py @@ -614,7 +614,7 @@ def combine_kp(standard_inst=None, recent_inst=None, forecast_inst=None, notes += "{:})".format(itime.date()) else: local_fill_val = standard_inst.meta[ - 'Kp', standard_inst.meta.labels.fill_val] + 'Kp', standard_inst.meta.labels.fill_val] good_times = ((standard_inst.index >= itime) & (standard_inst.index < stop)) good_vals = np.array([ From 3044dc8e8e4ba93c732ad2690407f116730f6ee8 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Wed, 2 Oct 2024 12:11:14 -0400 Subject: [PATCH 09/11] TST: added unit test Added unit test for `is_fill_val`. --- .../tests/test_methods_general.py | 30 ++++++++++++++++--- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/pysatSpaceWeather/tests/test_methods_general.py b/pysatSpaceWeather/tests/test_methods_general.py index eab60b2..bac18e2 100644 --- a/pysatSpaceWeather/tests/test_methods_general.py +++ b/pysatSpaceWeather/tests/test_methods_general.py @@ -9,6 +9,7 @@ """Integration and unit test suite for ACE methods.""" import numpy as np +import pytest import pysat @@ -22,24 +23,45 @@ def setup_method(self): """Create a clean testing setup.""" self.testInst = pysat.Instrument('pysat', 'testing') self.testInst.load(date=self.testInst.inst_module._test_dates['']['']) + self.var = self.testInst.variables[0] return def teardown_method(self): """Clean up previous testing setup.""" - del self.testInst + del self.testInst, self.var return def test_preprocess(self): """Test the preprocessing routine updates all fill values to be NaN.""" # Make sure at least one fill value is not already NaN - var = self.testInst.variables[0] - self.testInst.meta[var] = {self.testInst.meta.labels.fill_val: 0.0} + self.testInst.meta[self.var] = {self.testInst.meta.labels.fill_val: 0.0} # Update the meta data using the general preprocess routine general.preprocess(self.testInst) # Test the output assert np.isnan( - self.testInst.meta[var, self.testInst.meta.labels.fill_val]) + self.testInst.meta[self.var, self.testInst.meta.labels.fill_val]) + return + + @pytest.mark.parametrize("fill_val", [-1.0, -1, np.nan, np.inf, '']) + def test_is_fill(self, fill_val): + """Test the successful evaluation of fill values. + + Parameters + ---------- + fill_val : float, int, or str + Fill value to use as a comparison + + """ + # Set the data value to not be a fill value + if fill_val != '': + self.var = -47 + + # Evaluate the variable is False + assert not general.is_fill_val(self.var, fill_val) + + # Evaluate the fill value is a fill value + assert general.is_fill_val(fill_val, fill_val) return From 2f164d6b264634a0a1ec43d3726a1175c7255488 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Wed, 2 Oct 2024 12:11:32 -0400 Subject: [PATCH 10/11] DOC: updated changelog Added summary of changes to the changelog. --- CHANGELOG.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 63a6d39..cf7b3f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,10 +3,16 @@ Change Log All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](https://semver.org/). -[X.X.X] - 20XX-XX-XX +[0.2.1] - 2024-10-XX -------------------- +* Enhancements + * Added a utility function for evaluating fill values of different types * Maintenance * Updated Ops tests to new lower limit of Python 3.9 and removed 3.6 support +* Bugs + * Fixed error in mock downloading F10.7 prelim files + * Fixed combine_kp to consider desired time limits and fill values when + loading the standard dataset [0.2.0] - 2024-08-30 -------------------- From 13b0f9fed84902f34b86f8fd24eb56c3a01a5fe3 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Wed, 2 Oct 2024 13:24:42 -0400 Subject: [PATCH 11/11] BUG: fixed time evaluation Moved up the time evaluation for the forecast F10.7 data in the combine function. Also removed a duplicate definition of a date range variable. --- pysatSpaceWeather/instruments/methods/f107.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/pysatSpaceWeather/instruments/methods/f107.py b/pysatSpaceWeather/instruments/methods/f107.py index 4e6b368..648da5c 100644 --- a/pysatSpaceWeather/instruments/methods/f107.py +++ b/pysatSpaceWeather/instruments/methods/f107.py @@ -245,11 +245,10 @@ def combine_f107(standard_inst, forecast_inst, start=None, stop=None): good_vals = np.array([ not is_fill_val(val, fill_val) for val in forecast_inst['f107'][good_times]]) + new_times = list(forecast_inst.index[good_times][good_vals]) # Save desired data and cycle time - if len(good_vals) > 0: - new_times = list( - forecast_inst.index[good_times][good_vals]) + if len(new_times) > 0: f107_times.extend(new_times) new_vals = list( forecast_inst['f107'][good_times][good_vals]) @@ -274,8 +273,6 @@ def combine_f107(standard_inst, forecast_inst, start=None, stop=None): if len(f107_times) == 0: f107_times = date_range - date_range = pds.date_range(start=start, end=end_date, freq=freq) - if date_range[0] < f107_times[0]: # Extend the time and value arrays from their beginning with fill # values