counts2phys_byfloats.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "708a63af-6dc5-4513-aee3-f54021dda62f",
   "metadata": {},
   "source": [
    "# Script to convert BGC-Argo profiles Ed and Lu data in counts into physics units.\n",
    "NB : This code is adapted to a priliminary version where profiles files are located in aux/coriolis/\n",
    "and calibration files are from Edouar database."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a4e39473",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.executable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3e8370f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import packages\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import xarray as xr\n",
    "import os\n",
    "import glob\n",
    "import cmocean\n",
    "#import matplotlib\n",
    "\n",
    "import sys\n",
    "# add path to the folder which contains Toolbox_RAMSES.py script\n",
    "sys.path.append('/Users/charlotte.begouen/Documents/Hyperspectral_floats_Herve')\n",
    "import Toolbox_RAMSES as tools\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cd41a47d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define filepathes\n",
    "root = '/Users/charlotte.begouen/Documents/Hyperspectral_floats_Herve/'\n",
    "# path for metadate folder\n",
    "meta_dir = root + 'float_profile/'\n",
    "# path for profiles folder\n",
    "profile_dir = root + 'float_profile/'\n",
    "# path to save profiles in physics units\n",
    "profile_dir_txt = root + 'Outputs/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a0b10d05",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Choose float WMO that you want to convert\n",
    "n_float = '6990503' "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d9f6f844-972e-4a5b-bcec-896838a6d26e",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# files names\n",
    "float_dir = profile_dir+n_float+'/'\n",
    "meta_name = n_float+'_meta_aux.nc'\n",
    "meta = xr.open_dataset(float_dir+meta_name)\n",
    "\n",
    "# find calibration file names associated to this float sensors :\n",
    "tab = pd.read_table('WMOvsNSerie.txt') \n",
    "n_series = tab[ tab.WMO==int(n_float) ]\n",
    "\n",
    "# open calibration files (retrieved from Edouard)\n",
    "path_cal = '/Users/charlotte.begouen/Documents/Hyperspectral_floats_Herve/'\n",
    "path_cal_Ed = path_cal+n_series.N_Serie[ n_series.EdLu=='Ed' ].iloc[0]+'/'\n",
    "path_cal_Lu = path_cal+n_series.N_Serie[ n_series.EdLu=='Lu' ].iloc[0]+'/'\n",
    "\n",
    "file_Ed = glob.glob('*AllCal*',root_dir=path_cal_Ed)\n",
    "file_Lu = glob.glob('*AllCal*',root_dir=path_cal_Lu)\n",
    "\n",
    "# # open bgc-argo netcdf file\n",
    "# files_sorted = sorted(os.listdir(float_dir))\n",
    "# file = xr.open_dataset(float_dir+files_sorted[5])\n",
    "# file"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f2d95f90",
   "metadata": {},
   "source": [
    "## Create a csv file with data in physics units"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "47225231",
   "metadata": {
    "scrolled": true,
    "ExecuteTime": {
     "end_time": "2024-01-22T19:51:00.667129Z",
     "start_time": "2024-01-22T19:51:00.619082Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ".DS_Store\n",
      "R6990503_002_aux.nc\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "found the following matches with the input file in xarray's IO backends: ['netcdf4', 'h5netcdf', 'scipy']. But their dependencies may not be installed, see:\nhttps://docs.xarray.dev/en/stable/user-guide/io.html \nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html",
     "output_type": "error",
     "traceback": [
      "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[0;31mValueError\u001B[0m                                Traceback (most recent call last)",
      "Cell \u001B[0;32mIn[29], line 12\u001B[0m\n\u001B[1;32m      9\u001B[0m     \u001B[38;5;28;01mcontinue\u001B[39;00m\n\u001B[1;32m     11\u001B[0m \u001B[38;5;66;03m# open netcdf\u001B[39;00m\n\u001B[0;32m---> 12\u001B[0m file \u001B[38;5;241m=\u001B[39m \u001B[43mxr\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mopen_dataset\u001B[49m\u001B[43m(\u001B[49m\u001B[43mfloat_dir\u001B[49m\u001B[38;5;241;43m+\u001B[39;49m\u001B[38;5;124;43m'\u001B[39;49m\u001B[38;5;124;43mprofiles\u001B[39;49m\u001B[38;5;124;43m'\u001B[39;49m\u001B[38;5;241;43m+\u001B[39;49m\u001B[43mfile_name\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m     14\u001B[0m \u001B[38;5;66;03m# Find index in STATION_PARAMETERS values where Radiance and Irradiance are to know what is the N_PROF of Ed and Lu.\u001B[39;00m\n\u001B[1;32m     15\u001B[0m Ed_n_prof \u001B[38;5;241m=\u001B[39m np\u001B[38;5;241m.\u001B[39mwhere(file\u001B[38;5;241m.\u001B[39mSTATION_PARAMETERS\u001B[38;5;241m.\u001B[39mvalues\u001B[38;5;241m==\u001B[39m\u001B[38;5;124mb\u001B[39m\u001B[38;5;124m'\u001B[39m\u001B[38;5;124mRAW_DOWNWELLING_IRRADIANCE                                      \u001B[39m\u001B[38;5;124m'\u001B[39m)[\u001B[38;5;241m0\u001B[39m][\u001B[38;5;241m0\u001B[39m]\n",
      "File \u001B[0;32m~/opt/anaconda3/envs/HyperArgo/lib/python3.11/site-packages/xarray/backends/api.py:553\u001B[0m, in \u001B[0;36mopen_dataset\u001B[0;34m(filename_or_obj, engine, chunks, cache, decode_cf, mask_and_scale, decode_times, decode_timedelta, use_cftime, concat_characters, decode_coords, drop_variables, inline_array, chunked_array_type, from_array_kwargs, backend_kwargs, **kwargs)\u001B[0m\n\u001B[1;32m    550\u001B[0m     kwargs\u001B[38;5;241m.\u001B[39mupdate(backend_kwargs)\n\u001B[1;32m    552\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m engine \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n\u001B[0;32m--> 553\u001B[0m     engine \u001B[38;5;241m=\u001B[39m \u001B[43mplugins\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mguess_engine\u001B[49m\u001B[43m(\u001B[49m\u001B[43mfilename_or_obj\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    555\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m from_array_kwargs \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n\u001B[1;32m    556\u001B[0m     from_array_kwargs \u001B[38;5;241m=\u001B[39m {}\n",
      "File \u001B[0;32m~/opt/anaconda3/envs/HyperArgo/lib/python3.11/site-packages/xarray/backends/plugins.py:197\u001B[0m, in \u001B[0;36mguess_engine\u001B[0;34m(store_spec)\u001B[0m\n\u001B[1;32m    189\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[1;32m    190\u001B[0m     error_msg \u001B[38;5;241m=\u001B[39m (\n\u001B[1;32m    191\u001B[0m         \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mfound the following matches with the input file in xarray\u001B[39m\u001B[38;5;124m'\u001B[39m\u001B[38;5;124ms IO \u001B[39m\u001B[38;5;124m\"\u001B[39m\n\u001B[1;32m    192\u001B[0m         \u001B[38;5;124mf\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mbackends: \u001B[39m\u001B[38;5;132;01m{\u001B[39;00mcompatible_engines\u001B[38;5;132;01m}\u001B[39;00m\u001B[38;5;124m. But their dependencies may not be installed, see:\u001B[39m\u001B[38;5;130;01m\\n\u001B[39;00m\u001B[38;5;124m\"\u001B[39m\n\u001B[1;32m    193\u001B[0m         \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mhttps://docs.xarray.dev/en/stable/user-guide/io.html \u001B[39m\u001B[38;5;130;01m\\n\u001B[39;00m\u001B[38;5;124m\"\u001B[39m\n\u001B[1;32m    194\u001B[0m         \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html\u001B[39m\u001B[38;5;124m\"\u001B[39m\n\u001B[1;32m    195\u001B[0m     )\n\u001B[0;32m--> 197\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;167;01mValueError\u001B[39;00m(error_msg)\n",
      "\u001B[0;31mValueError\u001B[0m: found the following matches with the input file in xarray's IO backends: ['netcdf4', 'h5netcdf', 'scipy']. But their dependencies may not be installed, see:\nhttps://docs.xarray.dev/en/stable/user-guide/io.html \nhttps://docs.xarray.dev/en/stable/getting-started-guide/installing.html"
     ]
    }
   ],
   "source": [
    "Ed_physic, Lu_physic = pd.DataFrame(), pd.DataFrame()\n",
    "\n",
    "# list of the float's profiles sorted in alphabetic order\n",
    "files_name = sorted(os.listdir(float_dir+'profiles'))\n",
    "\n",
    "for file_name in files_name :\n",
    "    print(file_name)\n",
    "    if file_name == '.DS_Store':\n",
    "        continue\n",
    "\n",
    "    # open netcdf\n",
    "    file = xr.open_dataset(float_dir+'profiles'+file_name)\n",
    "\n",
    "    # Find index in STATION_PARAMETERS values where Radiance and Irradiance are to know what is the N_PROF of Ed and Lu.\n",
    "    Ed_n_prof = np.where(file.STATION_PARAMETERS.values==b'RAW_DOWNWELLING_IRRADIANCE                                      ')[0][0]\n",
    "    Lu_n_prof = np.where(file.STATION_PARAMETERS.values==b'RAW_UPWELLING_RADIANCE                                          ')[0][0]\n",
    "\n",
    "    # format ramses data\n",
    "    # careful : some float have different pixel binning or pixel stop parameters. \n",
    "    #You need to uncomment parameter when it is not the commun config.\n",
    "    Ed_physic_profile, Lu_physic_profile = tools.format_ramses(float_dir+file_name,meta_dir+meta_name,path_cal_Ed+file_Ed[0],\n",
    "                                                               path_cal_Lu+file_Lu[0], Ed_n_prof, Lu_n_prof)#, PixelBinning=1)#, PixelStop=144)\n",
    "    \n",
    "    # format into a complete dataframe\n",
    "    profile_Ed = pd.DataFrame({'CRUISE': [file_name[1:8]]*Ed_physic_profile.shape[0],\n",
    "                               'CYCLE': [int(file_name[-10:-7])]*Ed_physic_profile.shape[0],\n",
    "                               'WMO': [file_name[1:-7]]*Ed_physic_profile.shape[0],\n",
    "                               'TIME': [file.JULD.sel(N_PROF=Ed_n_prof).values]*Ed_physic_profile.shape[0],\n",
    "                               'lon': [file.lonGITUDE.sel(N_PROF=Ed_n_prof).values]*Ed_physic_profile.shape[0],\n",
    "                               'lat': [file.latITUDE.sel(N_PROF=Ed_n_prof).values]*Ed_physic_profile.shape[0],\n",
    "                               'PRES_FLOAT' : file.PRES.sel(N_PROF=Ed_n_prof).values[0:Ed_physic_profile.shape[0]] })\n",
    "    \n",
    "    profile_Lu = pd.DataFrame({'CRUISE': [file_name[1:8]]*Lu_physic_profile.shape[0],\n",
    "                               'CYCLE': [int(file_name[-10:-7])]*Lu_physic_profile.shape[0],\n",
    "                               'WMO': [file_name[1:-7]]*Lu_physic_profile.shape[0],\n",
    "                                'TIME': [file.JULD.sel(N_PROF=Lu_n_prof).values]*Lu_physic_profile.shape[0],\n",
    "                               'lon': [file.lonGITUDE.sel(N_PROF=Lu_n_prof).values]*Lu_physic_profile.shape[0],\n",
    "                               'lat': [file.latITUDE.sel(N_PROF=Lu_n_prof).values]*Lu_physic_profile.shape[0],\n",
    "                               'PRES_FLOAT' : file.PRES.sel(N_PROF=Lu_n_prof).values[0:Lu_physic_profile.shape[0]] })\n",
    "    \n",
    "    Ed_profile = pd.concat([profile_Ed,Ed_physic_profile],axis=1)\n",
    "    Lu_profile = pd.concat([profile_Lu,Lu_physic_profile],axis=1)\n",
    "    \n",
    "    # add to global table of the float\n",
    "    Ed_physic = pd.concat([Ed_physic,Ed_profile])\n",
    "    Lu_physic = pd.concat([Lu_physic,Lu_profile])\n",
    "    \n",
    "       \n",
    "# save csv file\n",
    "Ed_physic.to_csv(profile_dir_txt+file_name[1:8]+'_Ed.csv', index=False)\n",
    "Lu_physic.to_csv(profile_dir_txt+file_name[1:8]+'_Lu.csv', index=False)\n",
    "print('All files have been converted into a physics units table')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3ba5a212",
   "metadata": {},
   "source": [
    "## Plot data to check"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1bb2e506",
   "metadata": {},
   "outputs": [],
   "source": [
    "import cmocean\n",
    "import matplotlib.colors as mcolors\n",
    "import matplotlib.cm as cm\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "95d6209f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# to save fig\n",
    "to_save = '/home/lou/Documents/These/phd_axe1/Calibration_RAMSES/Outputs/raw_EdLu_210823/Figures/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6478abd9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import files just created\n",
    "Ed_physic = pd.read_csv(profile_dir_txt+'6990514_Ed.csv')\n",
    "Lu_physic = pd.read_csv(profile_dir_txt+'6990514_Lu.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bd158180",
   "metadata": {},
   "outputs": [],
   "source": [
    "# choose a profile\n",
    "Ed_physic_profile = Ed_physic[ Ed_physic.CYCLE==2 ]\n",
    "Lu_physic_profile = Lu_physic[ Lu_physic.CYCLE==2 ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "598b2e45",
   "metadata": {},
   "outputs": [],
   "source": [
    "# setup the normalization and the colormap\n",
    "nValues=Ed_physic_profile.Post_Pres\n",
    "normalize = mcolors.Normalize(vmin=nValues.min(), vmax=nValues.max())\n",
    "colormap = plt.get_cmap(cmocean.cm.haline, len(Ed_physic_profile.Post_Pres)).reversed()\n",
    "\n",
    "# wavelength\n",
    "wavelength = pd.to_numeric(Ed_physic_profile.columns[8:]).to_list()\n",
    "\n",
    "# plot figure\n",
    "plt.figure()\n",
    "for i in range(len(Ed_physic_profile.Post_Pres)):\n",
    "    plt.plot(wavelength, Ed_physic_profile.iloc[i,8:], label=Ed_physic_profile.Post_Pres.iloc[i], c=colormap(i), linewidth=0.5)\n",
    "\n",
    "# setup axis\n",
    "plt.xlim(300,800)\n",
    "plt.xlabel('Wavelength (nm)')\n",
    "plt.ylabel('Ed ($W.nm^{-1}.m^{-2}$)')\n",
    "plt.yscale('log')\n",
    "plt.title('{} \\n {} \\n Lon:{} , Lat:{}'.format(Ed_physic_profile.WMO.iloc[0],Ed_physic_profile.date.iloc[0],round(Ed_physic_profile.lon.iloc[0],2),round(Ed_physic_profile.lat.iloc[0],2)))\n",
    "\n",
    "\n",
    "# setup the colorbar\n",
    "scalarmappaple = cm.ScalarMappable(norm=normalize, cmap=colormap )\n",
    "scalarmappaple.set_array(nValues)\n",
    "plt.colorbar(scalarmappaple, label='depth (db)')\n",
    "\n",
    "\n",
    "plt.savefig(to_save+'Edvswave_'+Lu_physic_profile.WMO.iloc[0]+'.png', dpi=300, bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6bffc4b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# setup the normalization and the colormap\n",
    "nValues=Lu_physic_profile.Post_Pres\n",
    "normalize = mcolors.Normalize(vmin=nValues.min(), vmax=nValues.max())\n",
    "colormap = plt.get_cmap(cmocean.cm.haline, len(Lu_physic_profile.Post_Pres)).reversed()\n",
    "\n",
    "# wavelength\n",
    "wavelength = pd.to_numeric(Ed_physic_profile.columns[8:]).to_list()\n",
    "\n",
    "# plot figure\n",
    "plt.figure()\n",
    "for i in range(len(Lu_physic_profile.Post_Pres)):\n",
    "    plt.plot(wavelength, Lu_physic_profile.iloc[i,8:], label=Lu_physic_profile.Post_Pres.iloc[i], c=colormap(i), linewidth=0.5)\n",
    "\n",
    "# setup axis\n",
    "plt.xlim(300,800)\n",
    "plt.xlabel('Wavelength (nm)')\n",
    "plt.ylabel('Lu ($W.nm^{-1}.m^{-2}.sr^{-1}$)')\n",
    "plt.yscale('log')\n",
    "plt.title('{} \\n {} \\n Lon:{} , Lat:{}'.format(Lu_physic_profile.WMO.iloc[0],Lu_physic_profile.date.iloc[0],round(Lu_physic_profile.lon.iloc[0],2),round(Lu_physic_profile.lat.iloc[0],2)))\n",
    "\n",
    "# setup the colorbar\n",
    "scalarmappaple = cm.ScalarMappable(norm=normalize, cmap=colormap )\n",
    "scalarmappaple.set_array(nValues)\n",
    "plt.colorbar(scalarmappaple, label='depth (db)')\n",
    "\n",
    "plt.savefig(to_save+'Luvswave_'+Lu_physic_profile.WMO.iloc[0]+'.png', dpi=300, bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "90efe4d0",
   "metadata": {},
   "source": [
    "### Test plot en 3D (non fructueux pour l'instant)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "db28fbe6",
   "metadata": {},
   "outputs": [],
   "source": [
    "from mpl_toolkits.mplot3d import Axes3D\n",
    "\n",
    "# setup the normalization and the colormap\n",
    "nValues=Lu_physic.Post_Pres\n",
    "normalize = mcolors.Normalize(vmin=nValues.min(), vmax=nValues.max())\n",
    "colormap = plt.get_cmap(cmocean.cm.haline, len(Lu_physic_profile.Post_Pres)).reversed()\n",
    "\n",
    "# make data\n",
    "wavelength = pd.to_numeric(Ed_physic.columns[7:]).to_list()\n",
    "time = Ed_physic.date.to_list()\n",
    "depth = Ed_physic.Post_Pres.to_list()\n",
    "\n",
    "X, Y, Z = np.meshgrid(wavelength, time, depth)\n",
    "\n",
    "# Création de la figure et de l'axe 3D\n",
    "fig = plt.figure()\n",
    "ax = fig.add_subplot(111, projection='3d')\n",
    "\n",
    "# Tracé du plot 3D\n",
    "scatter = ax.plot(X.flatten(), Y.flatten(), Z.flatten(),\n",
    "                     c=Ed_physic_profile.iloc[:,7:].flatten(), cmap=cmocean.cm.thermal, linewidth=0.5)\n",
    "\n",
    "# for i in range(len(Lu_physic_profile.Post_Pres)):\n",
    "#     ax.plot_surface(wavelength, time, Lu_physic_profile.iloc[i,7:], label=Lu_physic_profile.Post_Pres.iloc[i], c=colormap(i), linewidth=0.5)\n",
    "\n",
    "# setup axis\n",
    "# plt.xlim(300,800)\n",
    "# plt.xlabel('Wavelength (nm)')\n",
    "# plt.ylabel('Ed ($W.nm^{-1}.m^{-2}.sr^{-1}$)')\n",
    "# plt.yscale('log')\n",
    "\n",
    "# Configuration de la colorbar\n",
    "cbar = fig.colorbar(scatter, ax=ax)\n",
    "cbar.set_label('Intensité de la lumière')\n",
    "\n",
    "# Affichage de la figure\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a5455dfc",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (Spyder)",
   "language": "python3",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}