update documentation

ecmwf · Aug 5, 2024 · 32f91d8 · 32f91d8
1 parent 7b3fb1b
commit 32f91d8
Show file tree

Hide file tree

Showing 11 changed files with 88 additions and 197 deletions.
diff --git a/docs/building/sources.rst b/docs/building/sources.rst
@@ -20,11 +20,14 @@ The following `sources` are currently available:
 .. toctree::
    :maxdepth: 1
 
-   sources/mars
+   sources/accumulations
+   sources/forcings
    sources/grib
+   sources/hindcasts
+   sources/mars
    sources/netcdf
-   sources/xarray
    sources/opendap
-   sources/forcings
-   sources/accumulations
    sources/recentre
+   sources/xarray-kerchunk
+   sources/xarray-zarr
+   sources/zenodo
diff --git a/docs/building/sources/hindcasts.yaml b/docs/building/sources/hindcasts.yaml
diff --git a/docs/building/sources/xarray-kerchunk.py b/docs/building/sources/xarray-kerchunk.py
@@ -0,0 +1,30 @@
+import json
+
+import fsspec
+import tqdm
+from kerchunk.combine import MultiZarrToZarr
+from kerchunk.hdf import SingleHdf5ToZarr
+
+fs = fsspec.filesystem("s3", anon=True)
+
+pattern = "s3://nsf-ncar-era5/e5.oper.an.pl/202403/e5.oper.an.pl.*.ll025sc.2024????00_2024????23.nc"
+
+
+jsons = []
+
+for file in tqdm.tqdm(fs.glob(pattern)):
+    with fs.open(file, "rb", anon=True) as f:
+        h5chunks = SingleHdf5ToZarr(f, file)
+        jsons.append(h5chunks.translate())
+
+
+mzz = MultiZarrToZarr(
+    jsons,
+    remote_protocol="s3",
+    remote_options={"anon": True},
+    concat_dims=["time"],
+    identical_dims=["latitude", "longitude"],
+)
+
+with open("combined.json", "w") as f:
+    json.dump(mzz.translate(), f)
diff --git a/docs/building/sources/xarray-kerchunk.rst b/docs/building/sources/xarray-kerchunk.rst
@@ -0,0 +1,17 @@
+#################
+ xarray-kerchunk
+#################
+
+.. literalinclude:: xarray-kerchunk.yaml
+   :language: yaml
+
+The code below is inspired by the `kerchunk tutorial`_, and makes use of
+a subset of the `ERA5 dataset available on AWS`_. You may need to
+install the relevant packages before running the code below.
+
+.. literalinclude:: xarray-kerchunk.py
+   :language: python
+
+.. _era5 dataset available on aws: https://registry.opendata.aws/ecmwf-era5/
+
+.. _kerchunk tutorial: https://fsspec.github.io/kerchunk/tutorial.html
diff --git a/docs/building/sources/xarray-kerchunk.yaml b/docs/building/sources/xarray-kerchunk.yaml
@@ -0,0 +1,10 @@
+dates:
+  start: 2024-03-24T00:00:00
+  end: 2024-03-24T03:00:00
+  frequency: 1h
+
+input:
+  xarray-kerchunk:
+    json: combined.json
+    param: T
+    level: [1000, 50]
diff --git a/docs/building/sources/xarray-zarr.rst b/docs/building/sources/xarray-zarr.rst
@@ -0,0 +1,6 @@
+#############
+ xarray-zarr
+#############
+
+.. literalinclude:: xarray-zarr.yaml
+   :language: yaml
diff --git a/docs/building/sources/xarray.yaml → docs/building/sources/xarray-zarr.yaml b/docs/building/sources/xarray.yaml → docs/building/sources/xarray-zarr.yaml
@@ -1,3 +1,3 @@
 input:
-  xarray:
+  xarray-zarr:
     url: https://...
diff --git a/docs/building/sources/xarray.rst b/docs/building/sources/xarray.rst
diff --git a/docs/building/sources/zenodo.rst b/docs/building/sources/zenodo.rst
@@ -0,0 +1,8 @@
+########
+ zenodo
+########
+
+https://zenodo.org/records/6470725
+
+.. literalinclude:: zenodo.yaml
+   :language: yaml
diff --git a/docs/building/sources/zenodo.yaml b/docs/building/sources/zenodo.yaml
@@ -0,0 +1,9 @@
+dates:
+  start: 2021-06-28
+  end: 2021-06-29
+
+input:
+  zenodo:
+    record_id: 6470725
+    file_key: "{param}_INCA_202106280700.nc"
+    param: [RR,PN, PT]
diff --git a/docs/test.ipynb b/docs/test.ipynb