Merge branch 'main' into standardising_atlas_file_downloading

brainglobe · Sep 4, 2024 · a0d6378 · a0d6378
2 parents 14007f8 + 414b0fa
commit a0d6378
Show file tree

Hide file tree

Showing 11 changed files with 2,012 additions and 1,771 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -16,7 +16,7 @@ repos:
           - id: requirements-txt-fixer
           - id: trailing-whitespace
     - repo: https://github.com/astral-sh/ruff-pre-commit
-      rev: v0.5.6
+      rev: v0.6.3
       hooks:
         - id: ruff
     - repo: https://github.com/psf/black

diff --git a/README.md b/README.md
@@ -23,20 +23,23 @@ The brainglobe atlas API (brainglobe-atlasapi) provides a common interface for p
 
 A number of atlases are in development, but those available currently are:
 
-- [Allen Mouse Brain Atlas](https://doi.org/10.1016/j.cell.2020.04.007) at 10, 25, 50 and 100 micron resolutions
-- [Allen Human Brain Atlas](https://www.brain-map.org) at 100 micron resolution
-- [Max Planck Zebrafish Brain Atlas](http://fishatlas.neuro.mpg.de) at 1 micron resolution
-- [Enhanced and Unified Mouse Brain Atlas](https://kimlab.io/brain-map/atlas/) at 10, 25, 50 and 100 micron resolutions
-- [Smoothed version of the Kim et al. mouse reference atlas](https://doi.org/10.1016/j.celrep.2014.12.014) at 10, 25, 50 and 100 micron resolutions
-- [Gubra's LSFM mouse brain atlas](https://doi.org/10.1007/s12021-020-09490-8) at 20 micron resolution
-- [3D version of the Allen mouse spinal cord atlas](https://doi.org/10.1101/2021.05.06.443008) at 20 x 10 x 10 micron resolution
-- [AZBA: A 3D Adult Zebrafish Brain Atlas](https://doi.org/10.1101/2021.05.04.442625) at 4 micron resolution
-- [Waxholm Space atlas of the Sprague Dawley rat brain](https://doi.org/10.1038/s41592-023-02034-3) at 39 micron resolution
-- [3D Edge-Aware Refined Atlases Derived from the Allen Developing Mouse Brain Atlases](https://doi.org/10.7554/eLife.61408) (E13, E15, E18, P4, P14, P28 & P56)
-- [Princeton Mouse Brain Atlas](https://brainmaps.princeton.edu/2020/09/princeton-mouse-brain-atlas-links) at 20 micron resolution
-- [Kim Lab Developmental CCF (P56)](https://data.mendeley.com/datasets/2svx788ddf/1) at 10 micron resolution with 8 reference images - STP, LSFM (iDISCO) and MRI (a0, adc, dwo, fa, MTR, T2)
-- [Blind Mexican Cavefish Brain Atlas](https://doi.org/10.7554/eLife.80777) at 2 micron resolution
-- [BlueBrain Barrel Cortex Atlas](https://doi.org/10.1162/imag_a_00209) at 10 and 25 micron resolution
+| Atlas Name | Resolution | Ages | Reference Images
+| --- |  --- | --- | --- |
+| [Allen Mouse Brain Atlas](https://doi.org/10.1016/j.cell.2020.04.007) | 10, 25, 50, and 100 micron | P56 | STPT
+| [Allen Human Brain Atlas](https://www.brain-map.org) | 500 micron | Adult | MRI
+| [Max Planck Zebrafish Brain Atlas](http://fishatlas.neuro.mpg.de) | 1 micron | 6-dpf | FISH
+| [Enhanced and Unified Mouse Brain Atlas](https://kimlab.io/brain-map/atlas/) | 10, 25, 50, and 100 micron | P56 | STPT
+| [Smoothed version of the Kim et al. mouse reference atlas](https://doi.org/10.1016/j.celrep.2014.12.014) | 10, 25, 50 and 100 micron | P56 | STPT
+| [Gubra's LSFM mouse brain atlas](https://doi.org/10.1007/s12021-020-09490-8) | 20 micron | 8 to 10 weeks post natal | LSFM
+| [3D version of the Allen mouse spinal cord atlas](https://doi.org/10.1101/2021.05.06.443008) | 20 x 10 x 10 micron | Adult | Nissl
+| [AZBA: A 3D Adult Zebrafish Brain Atlas](https://doi.org/10.1101/2021.05.04.442625) | 4 micron | 15-16 weeks post natal | LSFM
+| [Waxholm Space atlas of the Sprague Dawley rat brain](https://doi.org/10.1038/s41592-023-02034-3) | 39 micron | P80  | MRI
+| [3D Edge-Aware Refined Atlases Derived from the Allen Developing Mouse Brain Atlases](https://doi.org/10.7554/eLife.61408) | 16, 16.75, and 25 micron | E13, E15, E18, P4, P14, P28 & P56 | Nissl
+| [Princeton Mouse Brain Atlas](https://brainmaps.princeton.edu/2020/09/princeton-mouse-brain-atlas-links) | 20 micron | >P56 (older animals included) | LSFM
+| [Kim Lab Developmental CCF](https://data.mendeley.com/datasets/2svx788ddf/1) | 10 micron | P56  | STP, LSFM (iDISCO) and MRI (a0, adc, dwo, fa, MTR, T2)
+| [Blind Mexican Cavefish Brain Atlas](https://doi.org/10.7554/eLife.80777) | 2 micron | 1 year | IHC
+| [BlueBrain Barrel Cortex Atlas](https://doi.org/10.1162/imag_a_00209) | 10 and 25 micron | P56 | STPT
+| [UNAM Axolotl Brain Atlas](https://doi.org/10.1038/s41598-021-89357-3) | 40 micron | ~ 3 months post hatching | MRI
 
 ## Installation
 

diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/axolotl_atlas.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/axolotl_atlas.py
@@ -10,6 +10,9 @@
 import pooch
 from brainglobe_utils.IO.image import load_nii
 from rich.progress import track
+from skimage.filters.rank import modal
+from skimage.measure import label, regionprops
+from skimage.morphology import ball
 
 from brainglobe_atlasapi import utils
 from brainglobe_atlasapi.atlas_generation.mesh_utils import (
@@ -24,11 +27,10 @@
 def create_atlas(working_dir, resolution):
     ATLAS_NAME = "unam_axolotl"
     SPECIES = "Ambystoma mexicanum"
-    ATLAS_LINK = "https://www.nature.com/articles/s41598-021-89357-3"
+    ATLAS_LINK = "https://zenodo.org/records/4595016"
     CITATION = (
         "Lazcano, I. et al. 2021, https://doi.org/10.1038/s41598-021-89357-3"
     )
-    ATLAS_FILE_URL = "https://zenodo.org/records/4595016"
     ORIENTATION = "lpi"
     ROOT_ID = 999
     ATLAS_PACKAGER = "Saima Abdus, David Perez-Suarez, Alessandro Felder"
@@ -57,7 +59,7 @@ def create_atlas(working_dir, resolution):
     }
     for filename, hash in list_files.items():
         pooch.retrieve(
-            url=f"{ATLAS_FILE_URL}/files/{filename}",
+            url=f"{ATLAS_LINK}/files/{filename}",
             known_hash=hash,
             path=atlas_path,
             progressbar=True,
@@ -76,6 +78,21 @@ def create_atlas(working_dir, resolution):
     reference_image = (reference_image - dmin) * dscale
     reference_image = reference_image.astype(np.uint16)
 
+    # mask: put 1 where there is an annotation
+    annotation_mask = np.zeros(annotation_image.shape)
+    annotation_mask[annotation_image > 0] = 1
+
+    # find the connected regions in the mask
+    labeled_image = label(annotation_mask)
+    regions = regionprops(labeled_image)
+
+    # find the pixels belonging to the largest region
+    largest_region = max(regions, key=lambda region: region.area)
+    largest_mask = labeled_image == largest_region.label
+
+    # keep only annotations in the largest connected region
+    annotation_image = annotation_image * largest_mask
+
     hierarchy = []
 
     # create dictionaries # create dictionary from data read from the CSV file
@@ -165,13 +182,19 @@ def create_structure_id_path(main_structure):
         node.data = Region(is_label)
 
     # Mesh creation parameters
-    closing_n_iters = 5
-    decimate_fraction = 0.1
+    closing_n_iters = 10
+    decimate_fraction = 0.6
     smooth = True
 
     meshes_dir_path = working_dir / "meshes"
     meshes_dir_path.mkdir(exist_ok=True)
 
+    # pass a smoothed version of the annotations for meshing
+    smoothed_annotations = annotation_image.copy()
+    smoothed_annotations = modal(
+        smoothed_annotations.astype(np.uint8), ball(5)
+    )
+
     # Measure duration of mesh creation
     start = time.time()
 
@@ -188,7 +211,7 @@ def create_structure_id_path(main_structure):
                 node,
                 tree,
                 labels,
-                annotation_image,
+                smoothed_annotations,
                 ROOT_ID,
                 closing_n_iters,
                 decimate_fraction,

diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/whs_sd_rat.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/whs_sd_rat.py
@@ -1,15 +1,16 @@
-__version__ = "0"
+__version__ = "2"
 __atlas__ = "whs_sd_rat"
 
+
 import json
 import multiprocessing as mp
 import time
 from pathlib import Path
 
-import imio
 import numpy as np
 import pooch
 import xmltodict
+from brainglobe_utils.IO.image import load_any
 from rich.progress import track
 
 from brainglobe_atlasapi import utils
@@ -201,23 +202,22 @@ def create_atlas(working_dir):
     ATLAS_NAME = "whs_sd_rat"
     SPECIES = "Rattus norvegicus"
     ATLAS_LINK = "https://www.nitrc.org/projects/whs-sd-atlas"
-    CITATION = (
-        "Papp et al 2014, https://doi.org/10.1016/j.neuroimage.2014.04.001"
-    )
+    CITATION = "Kleven et al 2023, https://doi.org/10.1038/s41592-023-02034-3"
     ORIENTATION = "lpi"
     RESOLUTION = (39, 39, 39)
     ROOT_ID = 10000
-    ATLAS_FILE_URL = "https://www.nitrc.org/frs/download.php/12263/MBAT_WHS_SD_rat_atlas_v4_pack.zip"
+    REFERENCE_URL = "https://www.nitrc.org/frs/download.php/12263/MBAT_WHS_SD_rat_atlas_v4_pack.zip"
+    ANNOTATION_URL = "https://www.nitrc.org/frs/download.php/13400/MBAT_WHS_SD_rat_atlas_v4.01.zip//?i_agree=1&download_now=1"
     ATLAS_PACKAGER = (
-        "Ben Kantor, Tel Aviv University, Israel, benkantor@mail.tau.ac.il"
+        "Harry Carey, University of Oslo, Norway, harry.carey@medisin.uio.no"
     )
 
     assert len(ORIENTATION) == 3, (
         "Orientation is not 3 characters, Got" + ORIENTATION
     )
     assert len(RESOLUTION) == 3, "Resolution is not correct, Got " + RESOLUTION
     assert (
-        ATLAS_FILE_URL
+        REFERENCE_URL
     ), "No download link provided for atlas in ATLAS_FILE_URL"
 
     # Generated atlas path:
@@ -228,22 +228,30 @@ def create_atlas(working_dir):
     download_dir_path.mkdir(exist_ok=True)
 
     # Download atlas files from link provided
-    print("Downloading atlas from link: ", ATLAS_FILE_URL)
+
     atlas_files_dir = download_atlas_files(
         download_dir_path, ATLAS_FILE_URL, __atlas__
+
     )
     atlas_files_dir = atlas_files_dir / "MBAT_WHS_SD_rat_atlas_v4_pack/Data"
 
+    annotation_files_dir = download_atlas_files(
+        download_dir_path, ANNOTATION_URL, ATLAS_NAME + "_annotation"
+    )
+    annotation_files_dir = (
+        annotation_files_dir / "MBAT_WHS_SD_rat_atlas_v4.01/Data"
+    )
+
     # Parse structure metadata
     structures = parse_structures(
-        atlas_files_dir / "WHS_SD_rat_atlas_v4_labels.ilf"
+        annotation_files_dir / "WHS_SD_rat_atlas_v4.01_labels.ilf"
     )
 
     # Load files
-    annotation_stack = imio.load_any(
-        atlas_files_dir / "WHS_SD_rat_atlas_v4.nii.gz", as_numpy=True
+    annotation_stack = load_any(
+        annotation_files_dir / "WHS_SD_rat_atlas_v4.01.nii.gz", as_numpy=True
     ).astype(np.int64)
-    reference_stack = imio.load_any(
+    reference_stack = load_any(
         atlas_files_dir / "WHS_SD_rat_T2star_v1.01.nii.gz", as_numpy=True
     )
 

diff --git a/brainglobe_atlasapi/bg_atlas.py b/brainglobe_atlasapi/bg_atlas.py
@@ -8,7 +8,11 @@
 from rich.console import Console
 
 from brainglobe_atlasapi import config, core, descriptors, utils
-from brainglobe_atlasapi.utils import _rich_atlas_metadata
+from brainglobe_atlasapi.utils import (
+    _rich_atlas_metadata,
+    check_gin_status,
+    check_internet_connection,
+)
 
 COMPRESSED_FILENAME = "atlas.tar.gz"
 
@@ -80,13 +84,13 @@ def __init__(
         # Look for this atlas in local brainglobe folder:
         if self.local_full_name is None:
             if self.remote_version is None:
-                raise ValueError(f"{atlas_name} is not a valid atlas name!")
+                check_internet_connection(raise_error=True)
+                check_gin_status(raise_error=True)
 
-            rprint(
-                f"[magenta2]brainglobe_atlasapi: {self.atlas_name} "
-                "not found locally. Downloading...[magenta2]"
-            )
-            self.download_extract_file()
+                # If internet and GIN are up, then the atlas name was invalid
+                raise ValueError(f"{atlas_name} is not a valid atlas name!")
+            else:
+                self.download_extract_file()
 
         # Instantiate after eventual download:
         super().__init__(self.brainglobe_dir / self.local_full_name)
@@ -113,11 +117,11 @@ def remote_version(self):
         """
         remote_url = self._remote_url_base.format("last_versions.conf")
 
-        # Grasp remote version if a connection is available:
         try:
+            # Grasp remote version
             versions_conf = utils.conf_from_url(remote_url)
         except requests.ConnectionError:
-            return
+            return None
 
         try:
             return _version_tuple_from_str(
@@ -161,7 +165,8 @@ def remote_url(self):
 
     def download_extract_file(self):
         """Download and extract atlas from remote url."""
-        utils.check_internet_connection()
+        check_internet_connection()
+        check_gin_status()
 
         # Get path to folder where data will be saved
         destination_path = self.interm_download_dir / COMPRESSED_FILENAME

diff --git a/brainglobe_atlasapi/list_atlases.py b/brainglobe_atlasapi/list_atlases.py
@@ -51,12 +51,20 @@ def get_local_atlas_version(atlas_name):
 
 
 def get_all_atlases_lastversions():
-    """Read from URL all available last versions"""
-    available_atlases = utils.conf_from_url(
-        descriptors.remote_url_base.format("last_versions.conf")
-    )
-    available_atlases = dict(available_atlases["atlases"])
-    return available_atlases
+    """Read from URL or local cache all available last versions"""
+    cache_path = config.get_brainglobe_dir() / "last_versions.conf"
+
+    if utils.check_internet_connection(
+        raise_error=False
+    ) and utils.check_gin_status(raise_error=False):
+        available_atlases = utils.conf_from_url(
+            descriptors.remote_url_base.format("last_versions.conf")
+        )
+    else:
+        print("Cannot fetch latest atlas versions from the server.")
+        available_atlases = utils.conf_from_file(cache_path)
+
+    return dict(available_atlases["atlases"])
 
 
 def get_atlases_lastversions():