Skip to content

Commit

Permalink
Few minor fixes plus CDA masters analysis notebook
Browse files Browse the repository at this point in the history 
Signed-off-by: Alexis Jeandet <alexis.jeandet@member.fsf.org>
  • Loading branch information
@jeandet
jeandet committed Aug 27, 2022
1 parent c2bc8d5 commit 7727114
Show file tree
Hide file tree
Showing 3 changed files with 357 additions and 13 deletions.
338 changes: 338 additions & 0 deletions notebooks/CDAWeb_masters.ipynb
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,338 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"collapsed": true,
"pycharm": {
"name": "#%%\n"
}
},
"outputs": [],
"source": [
"import os\n",
"\n",
"os.environ[\"PYISTP_CDFLIB\"] = 'pycdfpp'\n",
"#os.environ[\"PYISTP_CDFLIB\"]='spacepy'\n",
"import pyistp\n",
"from pyistp import _impl as pyistp_impl\n",
"import requests\n",
"from glob import glob\n",
"import zstd\n",
"import tempfile\n",
"import tarfile\n",
"import io\n",
"import logging\n",
"from tqdm import tqdm\n",
"\n",
"def clean_logs():\n",
" lines=open('output/result.log').readlines()\n",
" index = 1\n",
" while index < len(lines):\n",
" if lines[index].startswith('Loading') and lines[index-1].startswith('Loading'):\n",
" lines.pop(index-1)\n",
" else:\n",
" index+=1\n",
" open('output/result_clean.log','w').writelines(lines)\n",
"\n",
"def compare_master_and_sample(master, sample_url_part, variable_name):\n",
" pyistp_impl.log.handlers.clear()\n",
" pyistp_impl.log.addHandler(logging.StreamHandler())\n",
" master = pyistp.load(buffer=requests.get(f\"https://cdaweb.gsfc.nasa.gov/pub/software/cdawlib/0MASTERS/{master}\").content)\n",
" sample_file = pyistp.load(buffer=requests.get(f\"https://cdaweb.gsfc.nasa.gov/pub/data/{sample_url_part}\").content)\n",
" print(\"master:\", flush=True)\n",
" v=master.data_variable(variable_name)\n",
" print(\"sample_file:\", flush=True)\n",
" v=sample_file.data_variable(variable_name)"
]
},
{
"cell_type": "markdown",
"source": [
"# Prepare output folder"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%% md\n"
}
}
},
{
"cell_type": "code",
"execution_count": 2,
"outputs": [],
"source": [
"if not os.path.exists('output'):\n",
" os.mkdir('output')\n",
"\n",
"if os.path.exists('output/result.log'):\n",
" os.remove('output/result.log')\n",
"\n",
"pyistp_impl.log.handlers.clear()\n",
"pyistp_impl.log.addHandler(logging.FileHandler(\"output/result.log\"))\n",
"pyistp_impl.log.setLevel(logging.DEBUG)"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n"
}
}
},
{
"cell_type": "markdown",
"source": [
"# Download and a load all masters as ISTP files"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%% md\n"
}
}
},
{
"cell_type": "code",
"execution_count": 3,
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████| 3324/3324 [00:21<00:00, 152.98it/s]\n"
]
}
],
"source": [
"\n",
"with tempfile.TemporaryDirectory() as tmp_dir:\n",
" tar = tarfile.open(fileobj=io.BytesIO(\n",
" zstd.decompress(requests.get('https://hephaistos.lpp.polytechnique.fr/data/jeandet/master.tar.zst').content)))\n",
" tar.extractall(tmp_dir)\n",
" del tar\n",
" cdf_masters = glob(f\"{tmp_dir}/*.cdf\")\n",
" for master in tqdm(cdf_masters):\n",
" istp_file = pyistp.load(master)\n",
" for vname in istp_file.data_variables():\n",
" v = istp_file.data_variable(vname)\n",
"\n",
"clean_logs()"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n"
}
}
},
{
"cell_type": "markdown",
"source": [
"# let's manually check few files"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%% md\n"
}
}
},
{
"cell_type": "markdown",
"source": [
"## c3_cp_efw_l3_e3d_inert\n",
"Both master and sample file have the same issue"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%% md\n"
}
}
},
{
"cell_type": "code",
"execution_count": 9,
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"master:\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Non compliant ISTP file: variable delta_Ez_ISR2__C3_CP_EFW_L3_E3D_INERT has LABL_PTR_1 attribute which points to variable dEz which does not exist\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"sample_file:\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Non compliant ISTP file: variable delta_Ez_ISR2__C3_CP_EFW_L3_E3D_INERT has LABL_PTR_1 attribute which points to variable dEz which does not exist\n"
]
}
],
"source": [
"compare_master_and_sample(master=\"c3_cp_efw_l3_e3d_inert_00000000_v01.cdf\",\n",
" sample_url_part=\"cluster/c3/efw/efield_3dvect_spinreso_l3_inertialframe/2003/c3_cp_efw_l3_e3d_inert_20030120_v20130803.cdf\",\n",
" variable_name=\"delta_Ez_ISR2__C3_CP_EFW_L3_E3D_INERT\")"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n"
}
}
},
{
"cell_type": "markdown",
"source": [
"## mms4_fpi_fast_l2_dis-momsaux\n",
"Both master and sample file have the same issue"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%% md\n"
}
}
},
{
"cell_type": "code",
"execution_count": 10,
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"master:\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Non compliant ISTP file: mms4_dis_compressionloss_fast was marked as data variable but it has 0 support variable\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"sample_file:\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Non compliant ISTP file: mms4_dis_compressionloss_fast was marked as data variable but it has 0 support variable\n"
]
}
],
"source": [
"compare_master_and_sample(master=\"mms4_fpi_fast_l2_dis-momsaux_00000000_v01.cdf\",\n",
" sample_url_part=\"mms/mms4/fpi/fast/l2/dis-momsaux/2021/10/mms4_fpi_fast_l2_dis-momsaux_20211003140000_v3.4.0.cdf\",\n",
" variable_name=\"mms4_dis_compressionloss_fast\")"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n"
}
}
},
{
"cell_type": "markdown",
"source": [
"## mvn_swi_l2_finearc3d\n",
"Both master and sample file have the same issue"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%% md\n"
}
}
},
{
"cell_type": "code",
"execution_count": 11,
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"master:\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Non compliant ISTP file: eindex was marked as data variable but it has 0 support variable\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"sample_file:\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Non compliant ISTP file: eindex was marked as data variable but it has 0 support variable\n"
]
}
],
"source": [
"compare_master_and_sample(master=\"mvn_swi_l2_finearc3d_00000000_v01.cdf\",\n",
" sample_url_part=\"maven/swia/l2/finearc3d/2020/11/mvn_swi_l2_finearc3d_20201125_v02_r00.cdf\",\n",
" variable_name=\"eindex\")"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n"
}
}
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.6"
}
},
"nbformat": 4,
"nbformat_minor": 0
}
Loading

0 comments on commit 7727114

Please sign in to comment.