Merge pull request #618 from bioimage-io/improve_api

Improve API

README.md

@@ -104,10 +104,25 @@ TODO: link to settings in dev docs
 
 Made with [contrib.rocks](https://contrib.rocks).
 
+## 🛈 Versioining scheme
+
+To keep the bioimageio.spec Python package version in sync with the (model) description format version, bioimageio.spec is versioned as MAJOR.MINRO.PATCH.LIB, where MAJOR.MINRO.PATCH correspond to the latest model description format version implemented and LIB may be bumpbed for library changes that do not affect the format version.
+[This change was introduced with bioimageio.spec 0.5.3.1](#bioimageiospec-0531).
+
 ## Δ Changelog
 
 ### bioimageio.spec Python package
 
+#### bioimageio.spec 0.5.3.1
+
+note: the versioning scheme was changed as our previous `post` releases include changes beyond what a post release should entail (only changing docstrings, etc).
+This was motivated by the desire to keep the library version in sync with the (model) format version to avoid confusion.
+To keep this relation, but avoid overbearing post releases a library version number is now added as the 4th part MAJOR.MINOR.PATCH.LIB_VERSION.
+
+* add `load_model_description` and `load_dataset_description`
+* add `ensure_description_is_model` and `ensure_description_is_dataset`
+* expose `perform_io_checks` and `known_files` from `ValidationContext` to `load_description` and `load_description_and_validate_format_only`
+
 #### bioimageio.spec 0.5.3post4
 
 * fix pinning of pydantic

bioimageio/spec/VERSION

@@ -1,3 +1,3 @@
 {
-    "version": "0.5.3post4"
+    "version": "0.5.3.1"
 }

bioimageio/spec/__init__.py

@@ -16,10 +16,12 @@
 from ._internal.common_nodes import InvalidDescr as InvalidDescr
 from ._internal.constants import VERSION
 from ._internal.validation_context import ValidationContext as ValidationContext
+from ._io import load_dataset_description as load_dataset_description
 from ._io import load_description as load_description
 from ._io import (
     load_description_and_validate_format_only as load_description_and_validate_format_only,
 )
+from ._io import load_model_description as load_model_description
 from ._io import save_bioimageio_yaml_only as save_bioimageio_yaml_only
 from ._package import save_bioimageio_package as save_bioimageio_package
 from ._package import (

bioimageio/spec/_description.py

@@ -5,7 +5,7 @@
 from pydantic import Discriminator
 from typing_extensions import Annotated
 
-from ._build_description import DISCOVER, build_description_impl, get_rd_class_impl
+from ._description_impl import DISCOVER, build_description_impl, get_rd_class_impl
 from ._internal.common_nodes import InvalidDescr
 from ._internal.io import BioimageioYamlContent, BioimageioYamlSource
 from ._internal.types import FormatVersionPlaceholder
@@ -131,6 +131,22 @@ def build_description(
     context: Optional[ValidationContext] = None,
     format_version: Union[FormatVersionPlaceholder, str] = DISCOVER,
 ) -> Union[ResourceDescr, InvalidDescr]:
+    """build a bioimage.io resource description from an RDF's content.
+
+    Use `load_description` if you want to build a resource description from an rdf.yaml
+    or bioimage.io zip-package.
+
+    Args:
+        content: loaded rdf.yaml file (loaded with YAML, not bioimageio.spec)
+        context: validation context to use during validation
+        format_version: (optional) use this argument to load the resource and
+                        convert its metadata to a higher format_version
+
+    Returns:
+        An object holding all metadata of the bioimage.io resource
+
+    """
+
     return build_description_impl(
         content,
         context=context,
@@ -162,3 +178,51 @@ def update_format(
 ) -> BioimageioYamlContent:
     """update a bioimageio.yaml file without validating it"""
     raise NotImplementedError("Oh no! This feature is not yet implemented")
+
+
+def ensure_description_is_model(
+    rd: Union[InvalidDescr, ResourceDescr],
+) -> AnyModelDescr:
+    if isinstance(rd, InvalidDescr):
+        rd.validation_summary.display()
+        raise ValueError("resource description is invalid")
+
+    if rd.type != "model":
+        rd.validation_summary.display()
+        raise ValueError(
+            f"expected a model resource, but got resource type '{rd.type}'"
+        )
+
+    assert not isinstance(
+        rd,
+        (
+            GenericDescr02,
+            GenericDescr03,
+        ),
+    )
+
+    return rd
+
+
+def ensure_description_is_dataset(
+    rd: Union[InvalidDescr, ResourceDescr],
+) -> AnyDatasetDescr:
+    if isinstance(rd, InvalidDescr):
+        rd.validation_summary.display()
+        raise ValueError("resource description is invalid")
+
+    if rd.type != "dataset":
+        rd.validation_summary.display()
+        raise ValueError(
+            f"expected a dataset resource, but got resource type '{rd.type}'"
+        )
+
+    assert not isinstance(
+        rd,
+        (
+            GenericDescr02,
+            GenericDescr03,
+        ),
+    )
+
+    return rd

bioimageio/spec/_build_description.py → bioimageio/spec/_description_impl.py

@@ -1,3 +1,5 @@
+"""implementation details for building a bioimage.io resource description"""
+
 from typing import Any, Callable, List, Mapping, Optional, Type, TypeVar, Union
 
 from ._internal.common_nodes import InvalidDescr, ResourceDescrBase

bioimageio/spec/_internal/common_nodes.py

@@ -127,9 +127,9 @@ def _get_data(cls, valid_string_data: str) -> Dict[str, Any]:
 
     @classmethod
     def _validate(cls, value: str) -> Self:
-        contrained_str_type = Annotated[str, StringConstraints(pattern=cls._pattern)]
-        contrained_str_adapter = TypeAdapter(cast(str, contrained_str_type))
-        valid_string_data = contrained_str_adapter.validate_python(value)
+        constrained_str_type = Annotated[str, StringConstraints(pattern=cls._pattern)]
+        constrained_str_adapter: TypeAdapter[str] = TypeAdapter(constrained_str_type)
+        valid_string_data = constrained_str_adapter.validate_python(value)
         data = cls._get_data(valid_string_data)
         self = cls(valid_string_data)
         object.__setattr__(self, "_node", self._node_class.model_validate(data))
@@ -309,7 +309,7 @@ def _ignore_future_patch(cls, data: Union[Dict[Any, Any], Any], /) -> Any:
         return data
 
     @model_validator(mode="after")
-    def _set_init_validation_summary(self):
+    def _set_init_validation_summary(self) -> Self:
         context = validation_context_var.get()
         self._validation_summary = ValidationSummary(
             name="bioimageio validation",

bioimageio/spec/_internal/field_warning.py

@@ -41,7 +41,7 @@ def warn(
     if isinstance(typ, get_args(AnnotationMetaData)):
         typ = Annotated[Any, typ]
 
-    validator = TypeAdapter(typ)
+    validator: TypeAdapter[Any] = TypeAdapter(typ)
 
     return AfterWarner(
         validator.validate_python, severity=severity, msg=msg, context={"typ": typ}

bioimageio/spec/_internal/io.py

@@ -171,25 +171,9 @@ def _validate(cls, value: Union[PurePath, str]):
         return cls(PurePath(value))
 
 
-class RelativePath(
-    RelativePathBase[Union[AbsoluteFilePath, AbsoluteDirectory, HttpUrl]], frozen=True
-):
-    def get_absolute(
-        self, root: "RootHttpUrl | Path | AnyUrl"
-    ) -> "AbsoluteFilePath | AbsoluteDirectory | HttpUrl":
-        absolute = self._get_absolute_impl(root)
-        if (
-            isinstance(absolute, Path)
-            and (context := validation_context_var.get()).perform_io_checks
-            and str(self.root) not in context.known_files
-            and not absolute.exists()
-        ):
-            raise ValueError(f"{absolute} does not exist")
-
-        return absolute
-
-
 class RelativeFilePath(RelativePathBase[Union[AbsoluteFilePath, HttpUrl]], frozen=True):
+    """A path relative to the `rdf.yaml` file (also if the RDF source is a URL)."""
+
     def model_post_init(self, __context: Any) -> None:
         if not self.root.parts:  # an empty path can only be a directory
             raise ValueError(f"{self.root} is not a valid file path.")
@@ -508,7 +492,9 @@ class HashKwargs(TypedDict):
     sha256: NotRequired[Optional[Sha256]]
 
 
-_file_source_adapter = TypeAdapter(FileSource)
+_file_source_adapter: TypeAdapter[Union[HttpUrl, RelativeFilePath, FilePath]] = (
+    TypeAdapter(FileSource)
+)
 
 
 def interprete_file_source(file_source: PermissiveFileSource) -> FileSource:

bioimageio/spec/_io.py

@@ -1,20 +1,27 @@
-from typing import Literal, TextIO, Union, cast
+from typing import Dict, Literal, Optional, TextIO, Union, cast
 
 from loguru import logger
 from pydantic import FilePath, NewPath
 
+from bioimageio.spec._internal.io_basics import Sha256
+
 from ._description import (
     DISCOVER,
     InvalidDescr,
     ResourceDescr,
     build_description,
     dump_description,
+    ensure_description_is_dataset,
+    ensure_description_is_model,
 )
+from ._internal._settings import settings
 from ._internal.common_nodes import ResourceDescrBase
 from ._internal.io import BioimageioYamlContent, YamlValue
 from ._internal.io_utils import open_bioimageio_yaml, write_yaml
-from ._internal.validation_context import ValidationContext
+from ._internal.validation_context import validation_context_var
 from .common import PermissiveFileSource
+from .dataset import AnyDatasetDescr
+from .model import AnyModelDescr
 from .summary import ValidationSummary
 
 
@@ -23,28 +30,98 @@ def load_description(
     /,
     *,
     format_version: Union[Literal["discover"], Literal["latest"], str] = DISCOVER,
+    perform_io_checks: bool = settings.perform_io_checks,
+    known_files: Optional[Dict[str, Sha256]] = None,
 ) -> Union[ResourceDescr, InvalidDescr]:
+    """load a bioimage.io resource description
+
+    Args:
+        source: Path or URL to an rdf.yaml or a bioimage.io package
+                (zip-file with rdf.yaml in it).
+        format_version: (optional) Use this argument to load the resource and
+                        convert its metadata to a higher format_version.
+        perform_io_checks: Wether or not to perform validation that requires file io,
+                           e.g. downloading a remote files. The existence of local
+                           absolute file paths is still being checked.
+        known_files: Allows to bypass download and hashing of referenced files
+                     (even if perform_io_checks is True).
+
+    Returns:
+        An object holding all metadata of the bioimage.io resource
+
+    """
     if isinstance(source, ResourceDescrBase):
         name = getattr(source, "name", f"{str(source)[:10]}...")
         logger.warning("returning already loaded description '{}' as is", name)
         return source  # pyright: ignore[reportReturnType]
 
     opened = open_bioimageio_yaml(source)
 
+    context = validation_context_var.get().replace(
+        root=opened.original_root,
+        file_name=opened.original_file_name,
+        perform_io_checks=perform_io_checks,
+        known_files=known_files,
+    )
+
     return build_description(
         opened.content,
-        context=ValidationContext(
-            root=opened.original_root, file_name=opened.original_file_name
-        ),
+        context=context,
+        format_version=format_version,
+    )
+
+
+def load_model_description(
+    source: PermissiveFileSource,
+    /,
+    *,
+    format_version: Union[Literal["discover"], Literal["latest"], str] = DISCOVER,
+    perform_io_checks: bool = settings.perform_io_checks,
+    known_files: Optional[Dict[str, Sha256]] = None,
+) -> AnyModelDescr:
+    """same as `load_description`, but addtionally ensures that the loaded
+    description is valid and of type 'model'.
+    """
+    rd = load_description(
+        source,
         format_version=format_version,
+        perform_io_checks=perform_io_checks,
+        known_files=known_files,
     )
+    return ensure_description_is_model(rd)
+
+
+def load_dataset_description(
+    source: PermissiveFileSource,
+    /,
+    *,
+    format_version: Union[Literal["discover"], Literal["latest"], str] = DISCOVER,
+    perform_io_checks: bool = settings.perform_io_checks,
+    known_files: Optional[Dict[str, Sha256]] = None,
+) -> AnyDatasetDescr:
+    """same as `load_description`, but addtionally ensures that the loaded
+    description is valid and of type 'dataset'.
+    """
+    rd = load_description(
+        source,
+        format_version=format_version,
+        perform_io_checks=perform_io_checks,
+        known_files=known_files,
+    )
+    return ensure_description_is_dataset(rd)
 
 
 def save_bioimageio_yaml_only(
     rd: Union[ResourceDescr, BioimageioYamlContent, InvalidDescr],
     /,
     file: Union[NewPath, FilePath, TextIO],
 ):
+    """write the metadata of a resource description (`rd`) to `file`
+    without writing any of the referenced files in it.
+
+    Note: To save a resource description with its associated files as a package,
+    use `save_bioimageio_package` or `save_bioimageio_package_as_folder`.
+    """
     if isinstance(rd, ResourceDescrBase):
         content = dump_description(rd)
     else:
@@ -58,7 +135,31 @@ def load_description_and_validate_format_only(
     /,
     *,
     format_version: Union[Literal["discover"], Literal["latest"], str] = DISCOVER,
+    perform_io_checks: bool = settings.perform_io_checks,
+    known_files: Optional[Dict[str, Sha256]] = None,
 ) -> ValidationSummary:
-    rd = load_description(source, format_version=format_version)
+    """load a bioimage.io resource description
+
+    Args:
+        source: Path or URL to an rdf.yaml or a bioimage.io package
+                (zip-file with rdf.yaml in it).
+        format_version: (optional) Use this argument to load the resource and
+                        convert its metadata to a higher format_version.
+        perform_io_checks: Wether or not to perform validation that requires file io,
+                           e.g. downloading a remote files. The existence of local
+                           absolute file paths is still being checked.
+        known_files: Allows to bypass download and hashing of referenced files
+                     (even if perform_io_checks is True).
+
+    Returns:
+        Validation summary of the bioimage.io resource found at `source`.
+
+    """
+    rd = load_description(
+        source,
+        format_version=format_version,
+        perform_io_checks=perform_io_checks,
+        known_files=known_files,
+    )
     assert rd.validation_summary is not None
     return rd.validation_summary

bioimageio/spec/model/v0_4.py

@@ -49,6 +49,7 @@
 from .._internal.io import FileDescr as FileDescr
 from .._internal.io_basics import AbsoluteFilePath as AbsoluteFilePath
 from .._internal.io_basics import Sha256 as Sha256
+from .._internal.io_utils import load_array
 from .._internal.packaging_context import packaging_context_var
 from .._internal.types import Datetime as Datetime
 from .._internal.types import Identifier as Identifier
@@ -74,7 +75,6 @@
 from ..generic.v0_2 import RelativeFilePath as RelativeFilePath
 from ..generic.v0_2 import ResourceId as ResourceId
 from ..generic.v0_2 import Uploader as Uploader
-from ..utils import load_array
 from ._v0_4_converter import convert_from_older_format