Rest commit

src/omnipy/__init__.py

@@ -137,6 +137,7 @@
 from omnipy.modules.tables.tasks import (remove_columns,
                                          rename_col_names,
                                          transpose_columns_with_data_files)
+from omnipy.util.contexts import print_exception
 
 # if typing.TYPE_CHECKING:
 
@@ -288,5 +289,6 @@
     'union_all',
     'remove_columns',
     'rename_col_names',
-    'transpose_columns_with_data_files'
+    'transpose_columns_with_data_files',
+    'print_exception',
 ]

src/omnipy/compute/mixins/serialize.py

@@ -199,7 +199,7 @@ def _generate_datetime_str(self):
     def _all_job_output_file_paths_in_reverse_order_for_last_run(
             persist_data_dir_path: Path, job_name: str) -> Generator[Path, None, None]:
 
-        sorted_date_dirs = iter(sorted(os.listdir(persist_data_dir_path)))
+        sorted_date_dirs = iter(reversed(sorted(os.listdir(persist_data_dir_path))))
 
         try:
             last_dir = next(sorted_date_dirs)

src/omnipy/data/dataset.py

@@ -34,8 +34,11 @@
 from omnipy.util.tabulate import tabulate
 from omnipy.util.web import download_file_to_memory
 
+# ModelT = TypeVar('ModelT', bound=Model, default=Model[object])
 ModelT = TypeVar('ModelT', bound=Model)
+# GeneralModelT = TypeVar('GeneralModelT', bound=Model, default=Model[object])
 GeneralModelT = TypeVar('GeneralModelT', bound=Model)
+
 _DatasetT = TypeVar('_DatasetT')
 
 DATA_KEY = 'data'
@@ -176,7 +179,7 @@ def __init__(  # noqa: C901
         if value != Undefined:
             assert data == Undefined, \
                 'Not allowed to combine positional and "data" keyword argument'
-            assert len(kwargs) == 0 or self.get_model_class().is_param_model(), \
+            assert len(kwargs) == 0, \
                 'Not allowed to combine positional and keyword arguments'
             super_kwargs[DATA_KEY] = value
 
@@ -185,18 +188,18 @@ def __init__(  # noqa: C901
                 "Not allowed to combine 'data' with other keyword arguments"
             super_kwargs[DATA_KEY] = data
 
-        model_cls = self.get_model_class()
+        # model_cls = self.get_model_class()
         if kwargs:
             if DATA_KEY not in super_kwargs:
-                assert isinstance(model_cls, TypeVar) or not model_cls.is_param_model(), \
-                    ('If any keyword arguments are defined, parametrized datasets require at least '
-                     'one positional argument in the __init__ method (typically providing the data '
-                     'in the form of a dict from name to content for each data file).')
-
+                #         assert isinstance(model_cls, TypeVar) or not model_cls.is_param_model(), \
+                #             ('If any keyword arguments are defined, parametrized datasets require at least '
+                #              'one positional argument in the __init__ method (typically providing the data '
+                #              'in the form of a dict from name to content for each data file).')
+                #
                 super_kwargs[DATA_KEY] = kwargs
                 kwargs = {}
 
-        if model_cls == ModelT:
+        if self.get_model_class() == ModelT:
             self._raise_no_model_exception()
 
         dataset_as_input = DATA_KEY in super_kwargs \
@@ -251,24 +254,18 @@ def get_model_class(cls) -> Type[Model]:
         model_type = cls._get_data_field().type_
         return model_type
 
-    # TODO: Update _raise_no_model_exception() text. Model is now a requirement
     @staticmethod
     def _raise_no_model_exception() -> None:
         raise TypeError(
-            'Note: The Dataset class requires a concrete model to be specified as '
+            'Note: The Dataset class requires a Model class (or a subclass) to be specified as '
             'a type hierarchy within brackets either directly, e.g.:\n\n'
-            '\tmodel = Dataset[list[int]]()\n\n'
+            '\tmodel = Dataset[Model[list[int]]]()\n\n'
             'or indirectly in a subclass definition, e.g.:\n\n'
-            '\tclass MyNumberListDataset(Dataset[list[int]]): ...\n\n'
-            'In both cases, the use of the Model class or a subclass is encouraged if anything '
-            'other than the simplest cases, e.g.:\n\n'
+            '\tclass MyNumberListDataset(Dataset[Model[list[int]]]): ...\n\n'
+            'For anything other than the simplest cases, the definition of Model and Dataset '
+            'subclasses is encouraged , e.g.:\n\n'
             '\tclass MyNumberListModel(Model[list[int]]): ...\n'
-            '\tclass MyDataset(Dataset[MyNumberListModel]): ...\n\n'
-            'Usage of Dataset without a type specification results in this exception. '
-            'Similar use of the Model class do not currently result in an exception, only '
-            'a warning message the first time this is done. However, this is just a '
-            '"poor man\'s exception" due to complex technicalities in that class. Please '
-            'explicitly specify types in both cases. ')
+            '\tclass MyDataset(Dataset[MyNumberListModel]): ...\n\n')
 
     def _set_standard_field_description(self) -> None:
         self.__fields__[DATA_KEY].field_info.description = self._get_standard_field_description()
@@ -570,7 +567,7 @@ def _table_repr(self) -> str:
             ((i,
               k,
               type(v).__name__,
-              v.__len__() if hasattr(v, '__len__') else 'N/A',
+              len(v) if hasattr(v, '__len__') else 'N/A',
               humanize.naturalsize(objsize.get_deep_size(v)))
              for i, (k, v) in enumerate(self.items())),
             ('#', 'Data file name', 'Type', 'Length', 'Size (in memory)'),

src/omnipy/data/helpers.py

@@ -42,7 +42,7 @@ class MethodInfo(NamedTuple):
 # (https://docs.python.org/3.10/reference/datamodel.html)
 _SPECIAL_METHODS_INFO_DICT: dict[str, MethodInfo] = {
     # 3.3.1. Basic customization ############################################
-    '__bool__': MethodInfo(state_changing=False, returns_same_type=YesNoMaybe.NO),
+    # '__bool__': MethodInfo(state_changing=False, returns_same_type=YesNoMaybe.NO),
     # 3.3.7. Emulating container types ######################################
     '__len__': MethodInfo(state_changing=False, returns_same_type=YesNoMaybe.NO),
     '__length_hint__': MethodInfo(state_changing=False, returns_same_type=YesNoMaybe.NO),
@@ -114,6 +114,8 @@ class MethodInfo(NamedTuple):
     '__trunc__': MethodInfo(state_changing=False, returns_same_type=YesNoMaybe.MAYBE),
     '__floor__': MethodInfo(state_changing=False, returns_same_type=YesNoMaybe.MAYBE),
     '__ceil__': MethodInfo(state_changing=False, returns_same_type=YesNoMaybe.MAYBE),
+    # - Hash and other standard methods ----------------------------------
+    '__hash__': MethodInfo(state_changing=False, returns_same_type=YesNoMaybe.NO),
 }
 
 

src/omnipy/data/model.py

@@ -1,5 +1,5 @@
 from collections import defaultdict
-from collections.abc import Iterable, Mapping, Sequence
+from collections.abc import Callable, Iterable, Mapping, Sequence
 from contextlib import contextmanager, suppress
 import functools
 import inspect
@@ -10,7 +10,6 @@
 from types import GenericAlias, ModuleType, NoneType, UnionType
 from typing import (Annotated,
                     Any,
-                    Callable,
                     cast,
                     ContextManager,
                     ForwardRef,
@@ -74,6 +73,9 @@
 _RootT = TypeVar('_RootT', bound=object | None, default=object)
 _ModelT = TypeVar('_ModelT')
 
+_ParamRootT = TypeVar('_ParamRootT', default=object | None)
+_KwargValT = TypeVar('_KwargValT', default=object)
+
 ROOT_KEY = '__root__'
 
 # TODO: Refactor Dataset and Model using mixins (including below functions)
@@ -246,9 +248,9 @@ def _get_default_value_from_model(cls, model: type[_RootT] | TypeForm | TypeVar)
         origin_type = get_origin(model)
         args = get_args(model)
 
-        if origin_type is Annotated:
-            model = remove_annotated_plus_optional_if_present(model)
-            return cls._get_default_value_from_model(model)
+        # if origin_type is Annotated:
+        #     model = remove_annotated_plus_optional_if_present(model)
+        #     return cls._get_default_value_from_model(model)
 
         if origin_type in (None, ()):
             origin_type = model
@@ -273,6 +275,9 @@ def _get_default_value_from_model(cls, model: type[_RootT] | TypeForm | TypeVar)
         if origin_type is Literal:
             return args[0]
 
+        if origin_type is Callable:
+            return cast(_RootT, lambda: None)
+
         if origin_type is ForwardRef or type(origin_type) is ForwardRef:
             raise TypeError(f'Cannot instantiate model "{model}". ')
 
@@ -538,9 +543,9 @@ def __del__(self):
         self.snapshot_holder.schedule_deepcopy_content_ids_for_deletion(contents_id)
 
     @classmethod
-    def clone_model_cls(cls: type[_ModelT], model_name: str) -> type[_ModelT]:
-        new_model: type[_ModelT] = type(model_name, (cls,), {})
-        return new_model
+    def clone_model_cls(cls: type[_ModelT], new_model_cls_name: str) -> type[_ModelT]:
+        new_model_cls: type[_ModelT] = type(new_model_cls_name, (cls,), {})
+        return new_model_cls
 
     @staticmethod
     def _raise_no_model_exception() -> None:
@@ -839,8 +844,8 @@ def _parse_with_root_type_if_model(cls,
                                        value: _RootT | None,
                                        root_field: ModelField,
                                        root_type: TypeForm) -> _RootT:
-        if get_origin(root_type) is Annotated:
-            root_type = remove_annotated_plus_optional_if_present(root_type)
+        # if get_origin(root_type) is Annotated:
+        #     root_type = remove_annotated_plus_optional_if_present(root_type)
 
         if get_origin(root_type) is Union:
             last_error_holder = LastErrorHolder()
@@ -955,6 +960,7 @@ def is_nested_type(cls) -> bool:
         return not cls.inner_type(with_args=True) == cls.outer_type(with_args=True)
 
     @classmethod
+    # Refactor: Remove is_param_model
     def is_param_model(cls) -> bool:
         if cls.outer_type() is list:
             type_to_check = cls.inner_type(with_args=True)
@@ -973,7 +979,7 @@ def _get_root_field(cls) -> ModelField:
     def _get_root_type(cls, outer: bool, with_args: bool) -> TypeForm | None:
         root_field = cls._get_root_field()
         root_type = root_field.outer_type_ if outer else root_field.type_
-        root_type = remove_annotated_plus_optional_if_present(root_type)
+        # root_type = remove_annotated_plus_optional_if_present(root_type)
         return root_type if with_args else ensure_plain_type(root_type)
 
     # @classmethod
@@ -1167,7 +1173,7 @@ def _iadd(other):
             try:
                 method = cast(Callable, self._getattr_from_contents_obj(name))
             except AttributeError as e:
-                if name in ('__int__', '__bool__', '__float__', '__complex__'):
+                if name in ('__int__', '__float__', '__complex__'):
                     raise ValueError from e
                 if name == '__len__':
                     raise TypeError(f"object of type '{self.__class__.__name__}' has no len()")
@@ -1291,49 +1297,49 @@ def _convert_to_model_if_reasonable(  # noqa: C901
             for type_to_check in all_type_variants(outer_type):
                 # TODO: Remove inner_type_to_check loop when Annotated hack is removed with
                 #       pydantic v2
-                type_to_check = cast(type | GenericAlias,
-                                     remove_annotated_plus_optional_if_present(type_to_check))
-                for inner_type_to_check in all_type_variants(type_to_check):
-                    plain_inner_type_to_check = ensure_plain_type(inner_type_to_check)
-                    # if plain_inner_type_to_check in (ForwardRef, TypeVar, Literal, None):
-                    if plain_inner_type_to_check in (ForwardRef, TypeVar, None):
-                        continue
-
-                    if level_up:
-                        inner_type_args = get_args(inner_type_to_check)
-                        if len(inner_type_args) == 0:
-                            inner_type_args = (inner_type_to_check,)
-                        if inner_type_args:
-                            for level_up_type_to_check in all_type_variants(
-                                    inner_type_args[level_up_arg_idx]):
-                                level_up_type_to_check = self._fix_tuple_type_from_args(
-                                    level_up_type_to_check)
-                                if self._is_instance_or_literal(
-                                        ret,
-                                        ensure_plain_type(level_up_type_to_check),
-                                        level_up_type_to_check,
-                                ):
-                                    try:
-                                        return Model[level_up_type_to_check](ret)  # type: ignore
-                                    except ValidationError:
-                                        if raise_validation_errors:
-                                            raise
-                                    except TypeError:
-                                        pass
+                # type_to_check = cast(type | GenericAlias,
+                #                      remove_annotated_plus_optional_if_present(type_to_check))
+                # for inner_type_to_check in all_type_variants(type_to_check):
+                plain_type_to_check = ensure_plain_type(type_to_check)
+                # if plain_type_to_check in (ForwardRef, TypeVar, Literal, None):
+                if plain_type_to_check in (ForwardRef, TypeVar, None):
+                    continue
+
+                if level_up:
+                    type_args = get_args(type_to_check)
+                    if len(type_args) == 0:
+                        type_args = (type_to_check,)
+                    if type_args:
+                        for level_up_type_to_check in all_type_variants(
+                                type_args[level_up_arg_idx]):
+                            level_up_type_to_check = self._fix_tuple_type_from_args(
+                                level_up_type_to_check)
+                            if self._is_instance_or_literal(
+                                    ret,
+                                    ensure_plain_type(level_up_type_to_check),
+                                    level_up_type_to_check,
+                            ):
+                                try:
+                                    return Model[level_up_type_to_check](ret)  # type: ignore
+                                except ValidationError:
+                                    if raise_validation_errors:
+                                        raise
+                                except TypeError:
+                                    pass
 
-                    else:
-                        if self._is_instance_or_literal(
-                                ret,
-                                plain_inner_type_to_check,
-                                inner_type_to_check,
-                        ):
-                            try:
-                                return self.__class__(ret)
-                            except ValidationError:
-                                if raise_validation_errors:
-                                    raise
-                            except TypeError:
-                                pass
+                else:
+                    if self._is_instance_or_literal(
+                            ret,
+                            plain_type_to_check,
+                            type_to_check,
+                    ):
+                        try:
+                            return self.__class__(ret)
+                        except ValidationError:
+                            if raise_validation_errors:
+                                raise
+                        except TypeError:
+                            pass
 
         return cast(_ReturnT, ret)
 
@@ -1427,9 +1433,20 @@ def __repr__(self) -> str:
                 return self._table_repr()
         return self._trad_repr()
 
-    def __hash__(self) -> int:
+    def __bool__(self):
+        if self._get_real_contents():
+            return True
+        else:
+            return False
+
+    def __call__(self, *args: object, **kwargs: object) -> object:
+        if not hasattr(self._get_real_contents(), '__call__'):
+            raise TypeError(f"'{self.__class__.__name__}' object is not callable")
         return self._special_method(
-            '__hash__', MethodInfo(state_changing=False, returns_same_type=YesNoMaybe.NO))
+            '__call__',
+            MethodInfo(state_changing=True, returns_same_type=YesNoMaybe.NO),
+            *args,
+            **kwargs)
 
     def view(self):
         from omnipy.modules.pandas.models import PandasModel
@@ -1518,10 +1535,6 @@ def _is_table():
         return out
 
 
-_ParamRootT = TypeVar('_ParamRootT', default=object | None)
-_KwargValT = TypeVar('_KwargValT', default=object)
-
-
 class DataWithParams(GenericModel, Generic[_ParamRootT, _KwargValT]):
     data: _ParamRootT
     params: dict[str, _KwargValT]
@@ -1579,7 +1592,7 @@ def _validate_and_set_contents_with_params(self, contents: _ParamRootT, **kwargs
         self._validate_and_set_value(DataWithParams(data=contents, params=kwargs))
 
 
-_ParamModelT = TypeVar('_ParamModelT', bound='ParamModel')
+_ParamModelT = TypeVar('_ParamModelT', bound='ParamModel', default='ParamModel')
 
 
 class ListOfParamModel(ParamModel[list[_ParamModelT

src/omnipy/modules/json/datasets.py

@@ -1,4 +1,6 @@
-from typing import Generic, TypeVar
+from typing import Generic
+
+from typing_extensions import TypeVar
 
 from omnipy.data.dataset import Dataset
 from omnipy.data.model import Model
@@ -29,7 +31,7 @@
 # TODO: call omnipy modules something else than modules, to distinguish from Python modules.
 #       Perhaps plugins?
 #
-_JsonModelT = TypeVar('_JsonModelT', bound=Model)
+_JsonModelT = TypeVar('_JsonModelT', bound=Model, default=JsonModel)
 
 
 class _JsonBaseDataset(Dataset[_JsonModelT], Generic[_JsonModelT]):

src/omnipy/modules/prefect/engine/prefect.py

@@ -84,6 +84,7 @@ def task_flow(*inner_args, **inner_kwargs):
     # LinearFlowRunnerEngine
     def _init_linear_flow(self, linear_flow: IsLinearFlow) -> Any:
         assert isinstance(self._config, PrefectEngineConfig)
+        # flow_kwargs = dict(name=linear_flow.name, persist_result=True, result_storage='S3/minio-s3')
         flow_kwargs = dict(name=linear_flow.name,)
         call_func = self.default_linear_flow_run_decorator(linear_flow)