Merge pull request #70 from zmoon/mm-fix

Fixes for MM data proc examples

.github/workflows/ci.yml

@@ -35,6 +35,11 @@ jobs:
       - name: Test with pytest
         run: pytest -n auto -v
 
+      - name: Test with pytspack installed
+        run: |
+          pip install https://github.com/noaa-oar-arl/pytspack/archive/master.zip --no-deps
+          pytest -n auto -v -k with_pytspack
+
   docs:
     name: Check docs build
     runs-on: ubuntu-latest

monetio/obs/aeronet.py

@@ -41,7 +41,7 @@ def add_local(
     # TODO: actually detect the specific inv type
 
     a.new_aod_values = interp_to_aod_values
-    if a.new_aod_values and not a.prod.startswith("AOD"):
+    if a.new_aod_values is not None and not a.prod.startswith("AOD"):
         print("`interp_to_aod_values` will be ignored")
 
     a.url = fname
@@ -52,7 +52,7 @@ def add_local(
 
     # TODO: DRY wrt. class?
     if freq is not None:
-        a.df = a.df.groupby("siteid").resample(freq).mean().reset_index()
+        a.df = a.df.set_index("time").groupby("siteid").resample(freq).mean().reset_index()
 
     if detect_dust:
         a.dust_detect()
@@ -475,7 +475,7 @@ def add_data(
         else:
             self.lunar = 0  # no lunar
         self.new_aod_values = interp_to_aod_values
-        if self.new_aod_values and not self.prod.startswith("AOD"):
+        if self.new_aod_values is not None and not self.prod.startswith("AOD"):
             print("`interp_to_aod_values` will be ignored")
 
         self.build_url()
@@ -488,7 +488,9 @@ def add_data(
             ) from e
 
         if freq is not None:
-            self.df = self.df.groupby("siteid").resample(freq).mean().reset_index()
+            self.df = (
+                self.df.set_index("time").groupby("siteid").resample(freq).mean().reset_index()
+            )
 
         if detect_dust:
             self.dust_detect()
@@ -514,9 +516,11 @@ def _tspack_aod_interp(row, new_wv=[440.0, 470.0, 550.0, 670.0, 870.0, 1020.0, 1
 
             try:
                 import pytspack
-            except ImportError:
-                print("You must install pytspack before using this function")
-                raise
+            except ImportError as e:
+                raise RuntimeError(
+                    "You must install pytspack before using this function.\n"
+                    "See https://github.com/noaa-oar-arl/pytspack/"
+                ) from e
 
             new_wv = np.asarray(new_wv)
 

monetio/obs/aqs.py

@@ -1,5 +1,6 @@
 import inspect
 import os
+import warnings
 
 import pandas as pd
 from dask.diagnostics import ProgressBar
@@ -648,6 +649,18 @@ def get_species(self, df, voc=False):
                 df.loc[con, "variable"] = "RH"
             if i == 62103:
                 df.loc[con, "variable"] = "DP"
+
+        # For any remaining, use parameter_name but warn
+        con = df.variable == ""
+        if con.sum() > 0:
+            _tbl = (
+                df[con][["parameter_name", "parameter_code"]]
+                .drop_duplicates("parameter_name")
+                .to_string(index=False)
+            )
+            warnings.warn(f"Short names not available for these variables:\n{_tbl}")
+        df.loc[con, "variable"] = df.parameter_name
+
         return df
 
     @staticmethod

tests/test_aeronet.py

@@ -1,12 +1,20 @@
 from pathlib import Path
 
+import numpy as np
 import pandas as pd
 import pytest
 
 from monetio import aeronet
 
 DATA = Path(__file__).parent / "data"
 
+try:
+    import pytspack  # noqa: F401
+except ImportError:
+    has_pytspack = False
+else:
+    has_pytspack = True
+
 
 def test_build_url_required_param_checks():
     # Default (nothing set; `dates`, `prod``, `daily` required)
@@ -152,3 +160,43 @@ def test_add_data_lunar():
     dates = pd.date_range("2022/01/20", "2022/01/21")
     df = aeronet.add_data(dates, lunar=True, siteid="Tucson")
     assert df.index.size > 0
+
+
+def test_serial_freq():
+    # For MM data proc example
+    dates = pd.date_range(start="2019-09-01", end="2019-09-2", freq="H")
+    df = aeronet.add_data(dates, freq="2H", n_procs=1)
+    assert (
+        pd.DatetimeIndex(sorted(df.time.unique()))
+        == pd.date_range("2019-09-01", freq="2H", periods=12)
+    ).all()
+
+
+@pytest.mark.skipif(has_pytspack, reason="has pytspack")
+def test_interp_without_pytspack():
+    # For MM data proc example
+    dates = pd.date_range(start="2019-09-01", end="2019-09-2", freq="H")
+    standard_wavelengths = np.array([0.34, 0.44, 0.55, 0.66, 0.86, 1.63, 11.1]) * 1000
+    with pytest.raises(RuntimeError, match="You must install pytspack"):
+        aeronet.add_data(dates, n_procs=1, interp_to_aod_values=standard_wavelengths)
+
+
+@pytest.mark.skipif(not has_pytspack, reason="no pytspack")
+def test_interp_with_pytspack():
+    # For MM data proc example
+    dates = pd.date_range(start="2019-09-01", end="2019-09-2", freq="H")
+    standard_wavelengths = np.array([0.34, 0.44, 0.55, 0.66, 0.86, 1.63, 11.1]) * 1000
+    df = aeronet.add_data(dates, n_procs=1, interp_to_aod_values=standard_wavelengths)
+    # Note: default wls for this period:
+    #
+    # wls = sorted(df.columns[df.columns.str.startswith("aod")].str.slice(4, -2).astype(int).tolist())
+    #
+    # [340, 380, 400, 412, 440,
+    #  443, 490, 500, 510, 532,
+    #  551, 555, 560, 620, 667,
+    #  675, 681, 709, 779, 865,
+    #  870, 1020, 1640]
+    #
+    # Note: Some of the ones we want already are in there (340 and 440 nm)
+    # TODO: add `_old` to the old ones or `_new` to the new ones? Or remove the old ones?
+    assert {f"aod_{int(wl)}nm" for wl in standard_wavelengths}.issubset(df.columns)

tests/test_aqs.py

@@ -0,0 +1,14 @@
+import pandas as pd
+import pytest
+
+from monetio import aqs
+
+
+def test_aqs():
+    # For MM data proc example
+    dates = pd.date_range(start="2019-08-01", end="2019-08-31", freq="H")
+    # Note: will retrieve full year
+    network = "NCORE"  # CSN NCORE CASTNET
+    with pytest.warns(UserWarning, match="Short names not available for these variables"):
+        df = aqs.add_data(dates, param=["PM10SPEC"], network=network, wide_fmt=False, daily=True)
+    assert (df.variable == "").sum() == 0