From 44bdf2577f393c85f0db0a7aaad7041a66def753 Mon Sep 17 00:00:00 2001 From: Sveinung Gundersen Date: Mon, 2 Sep 2024 11:43:04 +0200 Subject: [PATCH] Rest commit --- src/omnipy/__init__.py | 4 +- src/omnipy/compute/mixins/serialize.py | 2 +- src/omnipy/data/dataset.py | 39 +++-- src/omnipy/data/helpers.py | 4 +- src/omnipy/data/model.py | 135 ++++++++++-------- src/omnipy/modules/json/datasets.py | 6 +- src/omnipy/modules/prefect/engine/prefect.py | 1 + .../modules/prefect/settings/logging.yml | 83 ++++++----- src/omnipy/modules/raw/tasks.py | 14 +- src/omnipy/util/helpers.py | 1 + tests/data/test_model.py | 58 +++++--- .../integration/novel/full/helpers/models.py | 2 + .../novel/full/test_multi_model_dataset.py | 1 + tests/modules/frozen/test_models.py | 4 + 14 files changed, 207 insertions(+), 147 deletions(-) diff --git a/src/omnipy/__init__.py b/src/omnipy/__init__.py index 5013e53e..36431041 100644 --- a/src/omnipy/__init__.py +++ b/src/omnipy/__init__.py @@ -137,6 +137,7 @@ from omnipy.modules.tables.tasks import (remove_columns, rename_col_names, transpose_columns_with_data_files) +from omnipy.util.contexts import print_exception # if typing.TYPE_CHECKING: @@ -288,5 +289,6 @@ 'union_all', 'remove_columns', 'rename_col_names', - 'transpose_columns_with_data_files' + 'transpose_columns_with_data_files', + 'print_exception', ] diff --git a/src/omnipy/compute/mixins/serialize.py b/src/omnipy/compute/mixins/serialize.py index f9ec77e0..43158bdf 100644 --- a/src/omnipy/compute/mixins/serialize.py +++ b/src/omnipy/compute/mixins/serialize.py @@ -199,7 +199,7 @@ def _generate_datetime_str(self): def _all_job_output_file_paths_in_reverse_order_for_last_run( persist_data_dir_path: Path, job_name: str) -> Generator[Path, None, None]: - sorted_date_dirs = iter(sorted(os.listdir(persist_data_dir_path))) + sorted_date_dirs = iter(reversed(sorted(os.listdir(persist_data_dir_path)))) try: last_dir = next(sorted_date_dirs) diff --git a/src/omnipy/data/dataset.py b/src/omnipy/data/dataset.py index e37d8fda..8cb43ba5 100644 --- a/src/omnipy/data/dataset.py +++ b/src/omnipy/data/dataset.py @@ -34,8 +34,11 @@ from omnipy.util.tabulate import tabulate from omnipy.util.web import download_file_to_memory +# ModelT = TypeVar('ModelT', bound=Model, default=Model[object]) ModelT = TypeVar('ModelT', bound=Model) +# GeneralModelT = TypeVar('GeneralModelT', bound=Model, default=Model[object]) GeneralModelT = TypeVar('GeneralModelT', bound=Model) + _DatasetT = TypeVar('_DatasetT') DATA_KEY = 'data' @@ -176,7 +179,7 @@ def __init__( # noqa: C901 if value != Undefined: assert data == Undefined, \ 'Not allowed to combine positional and "data" keyword argument' - assert len(kwargs) == 0 or self.get_model_class().is_param_model(), \ + assert len(kwargs) == 0, \ 'Not allowed to combine positional and keyword arguments' super_kwargs[DATA_KEY] = value @@ -185,18 +188,18 @@ def __init__( # noqa: C901 "Not allowed to combine 'data' with other keyword arguments" super_kwargs[DATA_KEY] = data - model_cls = self.get_model_class() + # model_cls = self.get_model_class() if kwargs: if DATA_KEY not in super_kwargs: - assert isinstance(model_cls, TypeVar) or not model_cls.is_param_model(), \ - ('If any keyword arguments are defined, parametrized datasets require at least ' - 'one positional argument in the __init__ method (typically providing the data ' - 'in the form of a dict from name to content for each data file).') - + # assert isinstance(model_cls, TypeVar) or not model_cls.is_param_model(), \ + # ('If any keyword arguments are defined, parametrized datasets require at least ' + # 'one positional argument in the __init__ method (typically providing the data ' + # 'in the form of a dict from name to content for each data file).') + # super_kwargs[DATA_KEY] = kwargs kwargs = {} - if model_cls == ModelT: + if self.get_model_class() == ModelT: self._raise_no_model_exception() dataset_as_input = DATA_KEY in super_kwargs \ @@ -251,24 +254,18 @@ def get_model_class(cls) -> Type[Model]: model_type = cls._get_data_field().type_ return model_type - # TODO: Update _raise_no_model_exception() text. Model is now a requirement @staticmethod def _raise_no_model_exception() -> None: raise TypeError( - 'Note: The Dataset class requires a concrete model to be specified as ' + 'Note: The Dataset class requires a Model class (or a subclass) to be specified as ' 'a type hierarchy within brackets either directly, e.g.:\n\n' - '\tmodel = Dataset[list[int]]()\n\n' + '\tmodel = Dataset[Model[list[int]]]()\n\n' 'or indirectly in a subclass definition, e.g.:\n\n' - '\tclass MyNumberListDataset(Dataset[list[int]]): ...\n\n' - 'In both cases, the use of the Model class or a subclass is encouraged if anything ' - 'other than the simplest cases, e.g.:\n\n' + '\tclass MyNumberListDataset(Dataset[Model[list[int]]]): ...\n\n' + 'For anything other than the simplest cases, the definition of Model and Dataset ' + 'subclasses is encouraged , e.g.:\n\n' '\tclass MyNumberListModel(Model[list[int]]): ...\n' - '\tclass MyDataset(Dataset[MyNumberListModel]): ...\n\n' - 'Usage of Dataset without a type specification results in this exception. ' - 'Similar use of the Model class do not currently result in an exception, only ' - 'a warning message the first time this is done. However, this is just a ' - '"poor man\'s exception" due to complex technicalities in that class. Please ' - 'explicitly specify types in both cases. ') + '\tclass MyDataset(Dataset[MyNumberListModel]): ...\n\n') def _set_standard_field_description(self) -> None: self.__fields__[DATA_KEY].field_info.description = self._get_standard_field_description() @@ -570,7 +567,7 @@ def _table_repr(self) -> str: ((i, k, type(v).__name__, - v.__len__() if hasattr(v, '__len__') else 'N/A', + len(v) if hasattr(v, '__len__') else 'N/A', humanize.naturalsize(objsize.get_deep_size(v))) for i, (k, v) in enumerate(self.items())), ('#', 'Data file name', 'Type', 'Length', 'Size (in memory)'), diff --git a/src/omnipy/data/helpers.py b/src/omnipy/data/helpers.py index d5dcf9bc..84aa4ce8 100644 --- a/src/omnipy/data/helpers.py +++ b/src/omnipy/data/helpers.py @@ -42,7 +42,7 @@ class MethodInfo(NamedTuple): # (https://docs.python.org/3.10/reference/datamodel.html) _SPECIAL_METHODS_INFO_DICT: dict[str, MethodInfo] = { # 3.3.1. Basic customization ############################################ - '__bool__': MethodInfo(state_changing=False, returns_same_type=YesNoMaybe.NO), + # '__bool__': MethodInfo(state_changing=False, returns_same_type=YesNoMaybe.NO), # 3.3.7. Emulating container types ###################################### '__len__': MethodInfo(state_changing=False, returns_same_type=YesNoMaybe.NO), '__length_hint__': MethodInfo(state_changing=False, returns_same_type=YesNoMaybe.NO), @@ -114,6 +114,8 @@ class MethodInfo(NamedTuple): '__trunc__': MethodInfo(state_changing=False, returns_same_type=YesNoMaybe.MAYBE), '__floor__': MethodInfo(state_changing=False, returns_same_type=YesNoMaybe.MAYBE), '__ceil__': MethodInfo(state_changing=False, returns_same_type=YesNoMaybe.MAYBE), + # - Hash and other standard methods ---------------------------------- + '__hash__': MethodInfo(state_changing=False, returns_same_type=YesNoMaybe.NO), } diff --git a/src/omnipy/data/model.py b/src/omnipy/data/model.py index a0b8627a..44059eb7 100644 --- a/src/omnipy/data/model.py +++ b/src/omnipy/data/model.py @@ -1,5 +1,5 @@ from collections import defaultdict -from collections.abc import Iterable, Mapping, Sequence +from collections.abc import Callable, Iterable, Mapping, Sequence from contextlib import contextmanager, suppress import functools import inspect @@ -10,7 +10,6 @@ from types import GenericAlias, ModuleType, NoneType, UnionType from typing import (Annotated, Any, - Callable, cast, ContextManager, ForwardRef, @@ -74,6 +73,9 @@ _RootT = TypeVar('_RootT', bound=object | None, default=object) _ModelT = TypeVar('_ModelT') +_ParamRootT = TypeVar('_ParamRootT', default=object | None) +_KwargValT = TypeVar('_KwargValT', default=object) + ROOT_KEY = '__root__' # TODO: Refactor Dataset and Model using mixins (including below functions) @@ -246,9 +248,9 @@ def _get_default_value_from_model(cls, model: type[_RootT] | TypeForm | TypeVar) origin_type = get_origin(model) args = get_args(model) - if origin_type is Annotated: - model = remove_annotated_plus_optional_if_present(model) - return cls._get_default_value_from_model(model) + # if origin_type is Annotated: + # model = remove_annotated_plus_optional_if_present(model) + # return cls._get_default_value_from_model(model) if origin_type in (None, ()): origin_type = model @@ -273,6 +275,9 @@ def _get_default_value_from_model(cls, model: type[_RootT] | TypeForm | TypeVar) if origin_type is Literal: return args[0] + if origin_type is Callable: + return cast(_RootT, lambda: None) + if origin_type is ForwardRef or type(origin_type) is ForwardRef: raise TypeError(f'Cannot instantiate model "{model}". ') @@ -538,9 +543,9 @@ def __del__(self): self.snapshot_holder.schedule_deepcopy_content_ids_for_deletion(contents_id) @classmethod - def clone_model_cls(cls: type[_ModelT], model_name: str) -> type[_ModelT]: - new_model: type[_ModelT] = type(model_name, (cls,), {}) - return new_model + def clone_model_cls(cls: type[_ModelT], new_model_cls_name: str) -> type[_ModelT]: + new_model_cls: type[_ModelT] = type(new_model_cls_name, (cls,), {}) + return new_model_cls @staticmethod def _raise_no_model_exception() -> None: @@ -839,8 +844,8 @@ def _parse_with_root_type_if_model(cls, value: _RootT | None, root_field: ModelField, root_type: TypeForm) -> _RootT: - if get_origin(root_type) is Annotated: - root_type = remove_annotated_plus_optional_if_present(root_type) + # if get_origin(root_type) is Annotated: + # root_type = remove_annotated_plus_optional_if_present(root_type) if get_origin(root_type) is Union: last_error_holder = LastErrorHolder() @@ -955,6 +960,7 @@ def is_nested_type(cls) -> bool: return not cls.inner_type(with_args=True) == cls.outer_type(with_args=True) @classmethod + # Refactor: Remove is_param_model def is_param_model(cls) -> bool: if cls.outer_type() is list: type_to_check = cls.inner_type(with_args=True) @@ -973,7 +979,7 @@ def _get_root_field(cls) -> ModelField: def _get_root_type(cls, outer: bool, with_args: bool) -> TypeForm | None: root_field = cls._get_root_field() root_type = root_field.outer_type_ if outer else root_field.type_ - root_type = remove_annotated_plus_optional_if_present(root_type) + # root_type = remove_annotated_plus_optional_if_present(root_type) return root_type if with_args else ensure_plain_type(root_type) # @classmethod @@ -1167,7 +1173,7 @@ def _iadd(other): try: method = cast(Callable, self._getattr_from_contents_obj(name)) except AttributeError as e: - if name in ('__int__', '__bool__', '__float__', '__complex__'): + if name in ('__int__', '__float__', '__complex__'): raise ValueError from e if name == '__len__': raise TypeError(f"object of type '{self.__class__.__name__}' has no len()") @@ -1291,49 +1297,49 @@ def _convert_to_model_if_reasonable( # noqa: C901 for type_to_check in all_type_variants(outer_type): # TODO: Remove inner_type_to_check loop when Annotated hack is removed with # pydantic v2 - type_to_check = cast(type | GenericAlias, - remove_annotated_plus_optional_if_present(type_to_check)) - for inner_type_to_check in all_type_variants(type_to_check): - plain_inner_type_to_check = ensure_plain_type(inner_type_to_check) - # if plain_inner_type_to_check in (ForwardRef, TypeVar, Literal, None): - if plain_inner_type_to_check in (ForwardRef, TypeVar, None): - continue - - if level_up: - inner_type_args = get_args(inner_type_to_check) - if len(inner_type_args) == 0: - inner_type_args = (inner_type_to_check,) - if inner_type_args: - for level_up_type_to_check in all_type_variants( - inner_type_args[level_up_arg_idx]): - level_up_type_to_check = self._fix_tuple_type_from_args( - level_up_type_to_check) - if self._is_instance_or_literal( - ret, - ensure_plain_type(level_up_type_to_check), - level_up_type_to_check, - ): - try: - return Model[level_up_type_to_check](ret) # type: ignore - except ValidationError: - if raise_validation_errors: - raise - except TypeError: - pass + # type_to_check = cast(type | GenericAlias, + # remove_annotated_plus_optional_if_present(type_to_check)) + # for inner_type_to_check in all_type_variants(type_to_check): + plain_type_to_check = ensure_plain_type(type_to_check) + # if plain_type_to_check in (ForwardRef, TypeVar, Literal, None): + if plain_type_to_check in (ForwardRef, TypeVar, None): + continue + + if level_up: + type_args = get_args(type_to_check) + if len(type_args) == 0: + type_args = (type_to_check,) + if type_args: + for level_up_type_to_check in all_type_variants( + type_args[level_up_arg_idx]): + level_up_type_to_check = self._fix_tuple_type_from_args( + level_up_type_to_check) + if self._is_instance_or_literal( + ret, + ensure_plain_type(level_up_type_to_check), + level_up_type_to_check, + ): + try: + return Model[level_up_type_to_check](ret) # type: ignore + except ValidationError: + if raise_validation_errors: + raise + except TypeError: + pass - else: - if self._is_instance_or_literal( - ret, - plain_inner_type_to_check, - inner_type_to_check, - ): - try: - return self.__class__(ret) - except ValidationError: - if raise_validation_errors: - raise - except TypeError: - pass + else: + if self._is_instance_or_literal( + ret, + plain_type_to_check, + type_to_check, + ): + try: + return self.__class__(ret) + except ValidationError: + if raise_validation_errors: + raise + except TypeError: + pass return cast(_ReturnT, ret) @@ -1427,9 +1433,20 @@ def __repr__(self) -> str: return self._table_repr() return self._trad_repr() - def __hash__(self) -> int: + def __bool__(self): + if self._get_real_contents(): + return True + else: + return False + + def __call__(self, *args: object, **kwargs: object) -> object: + if not hasattr(self._get_real_contents(), '__call__'): + raise TypeError(f"'{self.__class__.__name__}' object is not callable") return self._special_method( - '__hash__', MethodInfo(state_changing=False, returns_same_type=YesNoMaybe.NO)) + '__call__', + MethodInfo(state_changing=True, returns_same_type=YesNoMaybe.NO), + *args, + **kwargs) def view(self): from omnipy.modules.pandas.models import PandasModel @@ -1518,10 +1535,6 @@ def _is_table(): return out -_ParamRootT = TypeVar('_ParamRootT', default=object | None) -_KwargValT = TypeVar('_KwargValT', default=object) - - class DataWithParams(GenericModel, Generic[_ParamRootT, _KwargValT]): data: _ParamRootT params: dict[str, _KwargValT] @@ -1579,7 +1592,7 @@ def _validate_and_set_contents_with_params(self, contents: _ParamRootT, **kwargs self._validate_and_set_value(DataWithParams(data=contents, params=kwargs)) -_ParamModelT = TypeVar('_ParamModelT', bound='ParamModel') +_ParamModelT = TypeVar('_ParamModelT', bound='ParamModel', default='ParamModel') class ListOfParamModel(ParamModel[list[_ParamModelT diff --git a/src/omnipy/modules/json/datasets.py b/src/omnipy/modules/json/datasets.py index d4f9114c..7822d828 100644 --- a/src/omnipy/modules/json/datasets.py +++ b/src/omnipy/modules/json/datasets.py @@ -1,4 +1,6 @@ -from typing import Generic, TypeVar +from typing import Generic + +from typing_extensions import TypeVar from omnipy.data.dataset import Dataset from omnipy.data.model import Model @@ -29,7 +31,7 @@ # TODO: call omnipy modules something else than modules, to distinguish from Python modules. # Perhaps plugins? # -_JsonModelT = TypeVar('_JsonModelT', bound=Model) +_JsonModelT = TypeVar('_JsonModelT', bound=Model, default=JsonModel) class _JsonBaseDataset(Dataset[_JsonModelT], Generic[_JsonModelT]): diff --git a/src/omnipy/modules/prefect/engine/prefect.py b/src/omnipy/modules/prefect/engine/prefect.py index cf113dc3..803612d0 100644 --- a/src/omnipy/modules/prefect/engine/prefect.py +++ b/src/omnipy/modules/prefect/engine/prefect.py @@ -84,6 +84,7 @@ def task_flow(*inner_args, **inner_kwargs): # LinearFlowRunnerEngine def _init_linear_flow(self, linear_flow: IsLinearFlow) -> Any: assert isinstance(self._config, PrefectEngineConfig) + # flow_kwargs = dict(name=linear_flow.name, persist_result=True, result_storage='S3/minio-s3') flow_kwargs = dict(name=linear_flow.name,) call_func = self.default_linear_flow_run_decorator(linear_flow) diff --git a/src/omnipy/modules/prefect/settings/logging.yml b/src/omnipy/modules/prefect/settings/logging.yml index fd95625a..abd789b2 100644 --- a/src/omnipy/modules/prefect/settings/logging.yml +++ b/src/omnipy/modules/prefect/settings/logging.yml @@ -14,15 +14,14 @@ formatters: datefmt: "%H:%M:%S" standard: + (): prefect.logging.formatters.PrefectFormatter format: "%(asctime)s.%(msecs)03d | %(levelname)-7s | %(name)s - %(message)s" + flow_run_fmt: "%(asctime)s.%(msecs)03d | %(levelname)-7s | Flow run %(flow_run_name)r - %(message)s" + task_run_fmt: "%(asctime)s.%(msecs)03d | %(levelname)-7s | Task run %(task_run_name)r - %(message)s" datefmt: "%H:%M:%S" - flow_runs: - format: "%(asctime)s.%(msecs)03d | %(levelname)-7s | Flow run %(flow_run_name)r - %(message)s" - datefmt: "%H:%M:%S" - - task_runs: - format: "%(asctime)s.%(msecs)03d | %(levelname)-7s | Task run %(task_run_name)r - %(message)s" + debug: + format: "%(asctime)s.%(msecs)03d | %(levelname)-7s | %(threadName)-12s | %(name)s - %(message)s" datefmt: "%H:%M:%S" json: @@ -42,68 +41,74 @@ handlers: console: level: 0 - class: logging.StreamHandler + class: prefect.logging.handlers.PrefectConsoleHandler formatter: standard + styles: + log.web_url: bright_blue + log.local_url: bright_blue - console_flow_runs: - level: 0 - class: logging.StreamHandler - formatter: flow_runs + log.info_level: cyan + log.warning_level: yellow3 + log.error_level: red3 + log.critical_level: bright_red - console_task_runs: - level: 0 - class: logging.StreamHandler - formatter: task_runs + log.completed_state: green + log.cancelled_state: yellow3 + log.failed_state: red3 + log.crashed_state: bright_red + + log.flow_run_name: magenta + log.flow_name: bold magenta - orion: +# api: +# level: 0 +# class: prefect.logging.handlers.APILogHandler + + + debug: level: 0 - class: prefect.logging.handlers.OrionHandler + class: logging.StreamHandler + formatter: debug loggers: prefect: level: "${PREFECT_LOGGING_LEVEL}" -# handlers: [console] - handlers: [] -# propagate: no - propagate: yes prefect.extra: level: "${PREFECT_LOGGING_LEVEL}" -# handlers: [orion, console] - handlers: [orion] -# propagate: no - propagate: yes + handlers: [api] prefect.flow_runs: level: NOTSET -# handlers: [orion, console_flow_runs] - handlers: [orion] -# propagate: no - propagate: yes + handlers: [api] prefect.task_runs: level: NOTSET -# handlers: [orion, console_task_runs] - handlers: [orion] -# propagate: no - propagate: yes + handlers: [api] - prefect.orion: + prefect.server: level: "${PREFECT_LOGGING_SERVER_LEVEL}" + prefect.client: + level: "${PREFECT_LOGGING_LEVEL}" + + prefect.infrastructure: + level: "${PREFECT_LOGGING_LEVEL}" + + prefect._internal: + level: "${PREFECT_LOGGING_INTERNAL_LEVEL}" + propagate: false + handlers: [debug] + uvicorn: level: "${PREFECT_LOGGING_SERVER_LEVEL}" - handlers: [console] - propagate: no fastapi: level: "${PREFECT_LOGGING_SERVER_LEVEL}" - handlers: [console] - propagate: no ## The root logger: any logger without propagation disabled sends to here as well #root: # # By default, we display warning level logs from any library in the console # # to match Python's default behavior while formatting logs nicely # level: WARNING -# handlers: [console] +# handlers: [console] \ No newline at end of file diff --git a/src/omnipy/modules/raw/tasks.py b/src/omnipy/modules/raw/tasks.py index a31bb744..b0ff99db 100644 --- a/src/omnipy/modules/raw/tasks.py +++ b/src/omnipy/modules/raw/tasks.py @@ -1,17 +1,19 @@ +from collections import deque from copy import deepcopy from functools import reduce from io import StringIO from itertools import chain from operator import add, ior import os -from typing import TypeVar from chardet import UniversalDetector +from typing_extensions import TypeVar from omnipy.compute.task import TaskTemplate from omnipy.compute.typing import mypy_fix_task_template from omnipy.data.dataset import Dataset, Model +from ...util.setdeque import SetDeque from .datasets import StrDataset from .protocols import IsModifyAllLinesCallable, IsModifyContentsCallable, IsModifyEachLineCallable @@ -82,18 +84,22 @@ def modify_all_lines( return os.linesep.join(modified_lines) -_ModelT = TypeVar('_ModelT', bound=Model) +_SequenceModelT = TypeVar( + '_SequenceModelT', bound=Model, default=Model[str | bytes | list | tuple | deque]) @mypy_fix_task_template @TaskTemplate() -def concat_all(dataset: Dataset[_ModelT]) -> _ModelT: +def concat_all(dataset: Dataset[_SequenceModelT]) -> _SequenceModelT: return reduce(add, (val for val in dataset.values())) +_UniqueModelT = TypeVar('_UniqueModelT', bound=Model, default=Model[dict | set | SetDeque]) + + @mypy_fix_task_template @TaskTemplate() -def union_all(dataset: Dataset[_ModelT]) -> _ModelT: +def union_all(dataset: Dataset[_UniqueModelT]) -> _UniqueModelT: all_vals = tuple(val for val in dataset.values()) assert len(all_vals) > 0 first_val = deepcopy(all_vals[0]) diff --git a/src/omnipy/util/helpers.py b/src/omnipy/util/helpers.py index 6d88ad5b..68059913 100644 --- a/src/omnipy/util/helpers.py +++ b/src/omnipy/util/helpers.py @@ -377,6 +377,7 @@ def get_deepcopy_object_ids(self) -> SetDeque[int]: return SetDeque(self._sub_obj_ids.keys()) def setup_deepcopy(self, obj): + print(f'setup_deepcopy({obj})') assert self._cur_deepcopy_obj_id is None, \ f'self._cur_deepcopy_obj_id is not None, but {self._cur_deepcopy_obj_id}' assert len(self._cur_keep_alive_list) == 0, \ diff --git a/tests/data/test_model.py b/tests/data/test_model.py index e6e11482..b03cdc61 100644 --- a/tests/data/test_model.py +++ b/tests/data/test_model.py @@ -1,4 +1,4 @@ -from collections.abc import Sequence +from collections.abc import Callable, Sequence import gc from math import floor import os @@ -6,7 +6,6 @@ from types import MappingProxyType, MethodType, NotImplementedType from typing import (Annotated, Any, - Callable, cast, ForwardRef, Generic, @@ -235,13 +234,13 @@ def test_get_inner_outer_type() -> None: assert dict_of_strings_to_list_of_ints_model.inner_type() == list assert dict_of_strings_to_list_of_ints_model.inner_type(with_args=True) == list[int] assert dict_of_strings_to_list_of_ints_model.is_nested_type() is True - - fake_optional_model = Model[Annotated[Optional[dict[str, list[int]]], 'someone else']]() - assert fake_optional_model.outer_type() == dict - assert fake_optional_model.outer_type(with_args=True) == dict[str, list[int]] - assert fake_optional_model.inner_type() == list - assert fake_optional_model.inner_type(with_args=True) == list[int] - assert fake_optional_model.is_nested_type() is True + # + # fake_optional_model = Model[Annotated[Optional[dict[str, list[int]]], 'someone else']]() + # assert fake_optional_model.outer_type() == dict + # assert fake_optional_model.outer_type(with_args=True) == dict[str, list[int]] + # assert fake_optional_model.inner_type() == list + # assert fake_optional_model.inner_type(with_args=True) == list[int] + # assert fake_optional_model.is_nested_type() is True def test_equality_other_models() -> None: @@ -747,10 +746,11 @@ class FirstTypeNotInstantiatableUnionModel(Model[Any | str]): assert FirstTypeNotInstantiatableUnionModel().to_data() == '' - with pytest.raises(TypeError): + class NoTypeInstantiatableUnionModel(Model[Any | Type]): + ... - class NoTypeInstantiatableUnionModel(Model[Any | Type]): - ... + with pytest.raises(TypeError): + NoTypeInstantiatableUnionModel() def test_union_default_value_if_any_none() -> None: @@ -1129,7 +1129,7 @@ class GenericListModel(Model[list[BaseT]], Generic[BaseT]): class ListModel(GenericListModel['FullModel']): ... - FullModel: TypeAlias = Union[ListModel, MaybeNumberModel] + FullModel: TypeAlias = Union[MaybeNumberModel, ListModel] ListModel.update_forward_refs(FullModel=FullModel) @@ -1752,6 +1752,7 @@ class SimplePydanticModel(BaseModel): model = Model[SimplePydanticModel](SimplePydanticModel(value=[123])) # type: ignore[arg-type] _assert_no_snapshot(model) + _assert_no_snapshot(model.contents.value) # Just accessing a field of a pydantic model through __getattr__ is enough to trigger a snapshot # of the parent @@ -2206,6 +2207,32 @@ def test_mimic_simple_list_operator_with_auto_convert( 'abc' + model # type: ignore[operator] +def test_mimic_hash_method(): + hashable_model = Model[str]('Hello World!') + assert hash(hashable_model) != 0 + + unhashable_model = Model[list[int]]() + with pytest.raises(TypeError): + hash(unhashable_model) + + +def test_mimic_call_method(): + callable_model = Model[Callable](lambda x: x + 1) + + assert callable_model(1) == 2 + + class MyClass: + ... + + not_callable_class_model = Model[MyClass]() + with pytest.raises(TypeError): + not_callable_class_model(1) + + not_callable_builtin_model = Model[int]() + with pytest.raises(TypeError): + not_callable_builtin_model(1) + + def test_mimic_sequence_convert_for_concat( runtime: Annotated[IsRuntime, pytest.fixture], skip_test_if_dynamically_convert_elements_to_models: Annotated[None, pytest.fixture], @@ -2390,8 +2417,6 @@ def test_mimic_concatenation_for_converted_models( def test_mimic_concatenation_for_converted_models_with_incompatible_contents_except_to_data( - runtime: Annotated[IsRuntime, pytest.fixture], - assert_model_if_dyn_conv_else_val: Annotated[AssertModelOrValFunc, pytest.fixture], ) -> None: class MyList(Generic[T]): def __init__(self, *args: T): @@ -2716,6 +2741,7 @@ def test_mimic_concat_less_than_five_model_add_variants_with_other_type_in_and_i def test_mimic_concat_all_less_than_five_model_add_variants_with_unsupported_input( all_add_variants: Annotated[tuple[bool, bool, bool, bool, bool], pytest.fixture], all_less_than_five_model_add_variants: Annotated[Model[MyNumberBase], pytest.fixture], + skip_test_if_dynamically_convert_elements_to_models: Annotated[None, pytest.fixture], ): has_add, has_radd, has_iadd, other_type_in, other_type_out = all_add_variants less_than_five_model = all_less_than_five_model_add_variants @@ -3789,8 +3815,6 @@ def test_parametrized_model_new() -> None: # assert ParamUpperStrModel().is_param_model() assert ParamUpperStrModel('foo').contents == 'foo' - asd = ParamUpperStrModel.adjust - # reveal_type(asd) MyUpperStrModel = ParamUpperStrModel.adjust('MyUpperStrModel', upper=True) assert MyUpperStrModel('bar').contents == 'BAR' diff --git a/tests/integration/novel/full/helpers/models.py b/tests/integration/novel/full/helpers/models.py index fafde829..2a48460b 100644 --- a/tests/integration/novel/full/helpers/models.py +++ b/tests/integration/novel/full/helpers/models.py @@ -69,6 +69,8 @@ def record_schema_factory(data_file: str, class Config(BaseConfig): extra = Extra.forbid + # Force config.dynamically_convert... is False + return create_model( data_file, __base__=RecordSchemaBase, diff --git a/tests/integration/novel/full/test_multi_model_dataset.py b/tests/integration/novel/full/test_multi_model_dataset.py index d0481b15..6080a5ff 100644 --- a/tests/integration/novel/full/test_multi_model_dataset.py +++ b/tests/integration/novel/full/test_multi_model_dataset.py @@ -110,6 +110,7 @@ def test_specialize_record_models_signature_and_return_type_func( @pc.parametrize_with_cases('case', cases='.cases.flows', has_tag='specialize_record_models') def test_run_specialize_record_models_consistent_types( runtime_all_engines: Annotated[None, pytest.fixture], # noqa + skip_test_if_dynamically_convert_elements_to_models, case: FlowCase): specialize_record_models = case.flow_template.apply() diff --git a/tests/modules/frozen/test_models.py b/tests/modules/frozen/test_models.py index 1d4b08e8..27538e88 100644 --- a/tests/modules/frozen/test_models.py +++ b/tests/modules/frozen/test_models.py @@ -5,6 +5,7 @@ import pytest_cases as pc from omnipy.data.model import Model +from omnipy.modules.frozen.models import NestedFrozenDictsOrTuplesModel from omnipy.modules.frozen.typedefs import FrozenDict from ..helpers.classes import CaseInfo @@ -36,6 +37,8 @@ class FrozenDictOfInt2NoneModel(Model[FrozenDict[int, NoneModel]]): @pc.parametrize_with_cases('case', cases='.cases.frozen_data') def test_nested_frozen_models(case: CaseInfo) -> None: + # NestedFrozenDictsOrTuplesModel[str, None | int](None) + for field in fields(case.data_points): name = field.name for model_cls in case.model_classes_for_data_point(name): @@ -51,6 +54,7 @@ def test_nested_frozen_models(case: CaseInfo) -> None: model_cls(data) # print(f'Error: {e}') else: + print(data) model_obj = model_cls(data) # print(f'repr(model_obj): {repr(model_obj)}')