Skip to content

Commit

Permalink
Rest of changes to extract commits from
Browse files Browse the repository at this point in the history
  • Loading branch information
sveinugu committed Aug 16, 2024
1 parent 5423d4b commit 2837605
Show file tree
Hide file tree
Showing 59 changed files with 1,784 additions and 1,337 deletions.
8 changes: 4 additions & 4 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.4.0
rev: v4.6.0
hooks:
- id: check-yaml
types: [ yaml ]
Expand All @@ -10,7 +10,7 @@ repos:
- id: check-ast
types: [ python ]
- repo: https://github.com/python-poetry/poetry
rev: 1.6.0
rev: 1.8.0
hooks:
- id: poetry-check
- repo: https://github.com/google/yapf
Expand All @@ -29,7 +29,7 @@ repos:
stages: [ manual ]
types: [ python ]
- repo: https://github.com/pycqa/isort
rev: 5.12.0
rev: 5.13.2
hooks:
- id: isort
name: isort
Expand All @@ -41,7 +41,7 @@ repos:
stages: [ manual ]
types: [ python ]
- repo: https://github.com/pycqa/flake8
rev: 6.1.0
rev: 7.0.0
hooks:
- id: flake8
additional_dependencies: [flake8-quotes==3.3.2]
794 changes: 50 additions & 744 deletions docs/notebooks/isa_json_data_wrangling_example.ipynb

Large diffs are not rendered by default.

5 changes: 5 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,11 @@ objsize = "^0.7.0"
humanize = "^4.9.0"
httpx = "^0.26.0"
pydantic = {version = "<2", extras = ["email"]}
bidict = "^0.23.1"
line-profiler-pycharm = "^1.1.0"
boltons = "^24.0.0"
jupyter = "^1.0.0"
collections-extended = "^2.0.2"

[tool.poetry.group.dev.dependencies]
deepdiff = "^6.2.1"
Expand Down
62 changes: 62 additions & 0 deletions scripts/deepcopy_memo_frozen.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
from typing import TypeAlias

from omnipy.modules.frozen.models import NestedFrozenDictsOrTuplesModel
from omnipy.modules.general.models import Model
from tests.modules.general.cases.raw.examples import (e_complex_key_dict,
e_int_key_dict,
e_none_key_dict,
ej_frozendict_iterable_scalar,
ej_frozendict_iterable_scalar_empty,
ej_frozendict_wrong_scalar,
ej_tuple_iterable_scalar,
ej_tuple_wrong_scalar,
ej_type,
f_complex,
f_dict,
f_frozendict,
f_int,
f_list,
f_none,
f_set,
f_str,
f_tuple)

FSK: TypeAlias = int | str | complex # for keys
FSV: TypeAlias = None | int | str | complex # for values


def run_nested_frozen_dicts_or_tuples_model() -> None:
_two_level_list: list[FSV | list[FSV] | dict[str, FSV]] = \
f_list + [list(f_list)] + [dict(f_dict)]
_two_level_dict: dict[str, str | list[FSV] | dict[str, FSV]] = \
{'a': f_str, 'b': list(f_list), 'c': dict(f_dict)}

number_model = Model[int](123)

print(number_model.snapshot_holder._deepcopy_memo)

number_model.snapshot_holder._deepcopy_memo.clear()

print(number_model.snapshot_holder._deepcopy_memo)

nested_frozen_dicts_or_tuples = (
list(f_list + [list(f_list), dict(f_dict), list(_two_level_list), dict(_two_level_dict)]))
nested_frozen_dicts_or_tuples_model = NestedFrozenDictsOrTuplesModel(
nested_frozen_dicts_or_tuples)

print(len(number_model.snapshot_holder._deepcopy_memo))
k = 0
v = 0
for k, v in number_model.snapshot_holder._deepcopy_memo.items():
print(f'{k}: {v}')
del k
del v

del nested_frozen_dicts_or_tuples_model
number_model.snapshot_holder.delete_scheduled_deepcopy_content_ids()
print(number_model.snapshot_holder._deepcopy_memo)

assert len(number_model.snapshot_holder._deepcopy_memo) == 0


run_nested_frozen_dicts_or_tuples_model()
11 changes: 11 additions & 0 deletions scripts/omnipy_import.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
def omnipy_import() -> None:
import omnipy
import omnipy._dynamic_all
omnipy.data
from omnipy import Model
import omnipy.data

# omnipy._dynamic_all.data


omnipy_import()
60 changes: 60 additions & 0 deletions scripts/param.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
from typing import Generic, get_args, get_origin, Literal

from pydantic.fields import UndefinedType
from typing_extensions import TypeVar

from omnipy import Model

D = TypeVar('D', default=object)
S = TypeVar('S', default=object)
T = TypeVar('T', default=Literal[','])


class Param(Model[UndefinedType | S | D], Generic[S, D]):
@classmethod
def _parse_data(cls, data: UndefinedType | S | D):
override, default = get_args(cls.outer_type(with_args=True))[1:3]
print(f'default: {repr(default)}, override: {repr(override)}, data: {repr(data)}')
assert get_origin(default) == Literal
target = override if get_origin(override) == Literal else default
print(
f'data: {repr(data)}, target: {repr(target)}, check: {type(data) is not UndefinedType and data != target}'
)
if type(data) is not UndefinedType and data != target:
print('I amm asserting!')
raise AssertionError(
f'Data other than the default or override literals are not allowed: {data}')
return target


DefaultDelim = Literal[',']


class DelimitParam(Param[S, DefaultDelim], Generic[S]):
...


class Y(Model[list[str] | tuple[str, DelimitParam]]):
...


class ValueWithParam(Model[tuple[DelimitParam[S], T]], Generic[T, S]):
@classmethod
def _parse_data(cls, data: tuple[T, DelimitParam[S]]):
print(f'data: {repr(data)}')
return data


class D(Model[T], Generic[T]):
...


D[Literal['asd']]()


class Splitter(Model[list[str] | str | Model[T]], Generic[T]):
...


class A(Model[tuple[T, S]], Generic[T, S]):
...
32 changes: 32 additions & 0 deletions scripts/profile_module_as_script.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
import cProfile
from importlib import import_module
import os
import pstats
import sys

if __name__ == '__main__':
script_path = os.path.abspath(sys.argv[1])
project_path = os.path.abspath(__file__ + '/../..')

print(f'script_path: {script_path}')
print(f'project_path: {project_path}')

assert script_path.startswith(project_path)
module = '.'.join(script_path.split('.')[0][len(project_path) + 1:].split('/'))

# print(f'package: {package}')
print(f'module: {module}')

sys.path.append(project_path)

mod = import_module(module)

profiler = cProfile.Profile()
profiler.enable()

mod.run()

profiler.disable()
stats = pstats.Stats(profiler).sort_stats('cumtime')
stats.dump_stats('latest4.pstats')
print('dumped')
83 changes: 71 additions & 12 deletions src/omnipy/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,16 +9,44 @@
LinearFlow,
LinearFlowTemplate)
from omnipy.compute.task import Task, TaskTemplate
from omnipy.data.dataset import Dataset, ListOfParamModelDataset, ParamDataset
from omnipy.data.dataset import Dataset, ListOfParamModelDataset, MultiModelDataset, ParamDataset
from omnipy.data.model import ListOfParamModel, Model, ParamModel
from omnipy.hub.runtime import runtime
# from omnipy.util.helpers import recursive_module_import
from omnipy.modules.general.models import (Chain2,
Chain3,
Chain4,
Chain5,
Chain6,
NotIterableExceptStrOrBytesModel)
from omnipy.modules.general.tasks import convert_dataset, import_directory, split_dataset
from omnipy.modules.isa import (flatten_isa_json,
FlattenedIsaJsonDataset,
FlattenedIsaJsonModel,
IsaJsonDataset,
IsaJsonModel)
from omnipy.modules.isa.models import IsaInvestigationModel, IsaTopLevelModel
from omnipy.modules.isa.models.assay_schema import IsaAssayJsonModel
from omnipy.modules.isa.models.comment_schema import IsaCommentModel
from omnipy.modules.isa.models.data_schema import IsaDataModel
from omnipy.modules.isa.models.factor_schema import IsaFactorModel
from omnipy.modules.isa.models.factor_value_schema import IsaFactorValueModel
from omnipy.modules.isa.models.material_attribute_schema import IsaMaterialAttributeModel
from omnipy.modules.isa.models.material_attribute_value_schema import IsaMaterialAttributeValueModel
from omnipy.modules.isa.models.material_schema import IsaMaterialModel
from omnipy.modules.isa.models.ontology_annotation_schema import IsaOntologyReferenceModel
from omnipy.modules.isa.models.ontology_source_reference_schema import \
IsaOntologySourceReferenceModel
from omnipy.modules.isa.models.organization_schema import IsaOrganizationModel
from omnipy.modules.isa.models.person_schema import IsaPersonModel
from omnipy.modules.isa.models.process_parameter_value_schema import IsaProcessParameterValueModel
from omnipy.modules.isa.models.process_schema import IsaProcessOrProtocolApplicationModel
from omnipy.modules.isa.models.protocol_parameter_schema import IsaProtocolParameterModel
from omnipy.modules.isa.models.protocol_schema import IsaProtocolModel
from omnipy.modules.isa.models.publication_schema import IsaPublicationModel
from omnipy.modules.isa.models.sample_schema import IsaSampleModel
from omnipy.modules.isa.models.source_schema import IsaSourceModel
from omnipy.modules.isa.models.study_group import IsaStudyGroupModel
from omnipy.modules.isa.models.study_schema import IsaStudyModel
from omnipy.modules.json.datasets import (JsonDataset,
JsonDictDataset,
JsonDictOfDictsDataset,
Expand Down Expand Up @@ -66,7 +94,7 @@
JsonOnlyDictsModel,
JsonOnlyListsModel,
JsonScalarModel)
from omnipy.modules.json.tasks import transpose_dicts_2_lists
from omnipy.modules.json.tasks import convert_dataset_string_to_json, transpose_dicts_2_lists
from omnipy.modules.pandas.models import (ListOfPandasDatasetsWithSameNumberOfFiles,
PandasDataset,
PandasModel)
Expand All @@ -89,8 +117,10 @@
JoinColumnsToLinesModel,
JoinItemsModel,
JoinLinesModel,
Params,
SplitLinesToColumnsModel,
SplitToItemsModel,
SplitToItemsModelNew,
SplitToLinesModel,
StrModel)
from omnipy.modules.raw.tasks import (concat_all,
Expand All @@ -100,12 +130,14 @@
modify_each_line,
union_all)
from omnipy.modules.tables.datasets import TableOfPydanticRecordsDataset, TableWithColNamesDataset
from omnipy.modules.tables.models import TableOfPydanticRecordsModel, TableWithColNamesModel
from omnipy.modules.tables.models import (PydanticRecordModel,
TableOfPydanticRecordsModel,
TableWithColNamesModel)
from omnipy.modules.tables.tasks import (remove_columns,
rename_col_names,
transpose_columns_with_data_files)

# from omnipy.util.helpers import recursive_module_import
# if typing.TYPE_CHECKING:

ROOT_DIR = os.path.dirname(os.path.abspath(__file__))

Expand All @@ -122,13 +154,37 @@
'Dataset',
'ParamDataset',
'ListOfParamModelDataset',
'MultiModelDataset',
'Model',
'ParamModel',
'ListOfParamModel',
'FlattenedIsaJsonDataset',
'FlattenedIsaJsonModel',
'IsaJsonModel',
'IsaJsonDataset',
'IsaInvestigationModel',
'IsaTopLevelModel',
'IsaAssayJsonModel',
'IsaCommentModel',
'IsaDataModel',
'IsaFactorModel',
'IsaFactorValueModel',
'IsaMaterialAttributeModel',
'IsaMaterialAttributeValueModel',
'IsaMaterialModel',
'IsaOntologyReferenceModel',
'IsaOntologySourceReferenceModel',
'IsaOrganizationModel',
'IsaPersonModel',
'IsaProcessParameterValueModel',
'IsaProcessOrProtocolApplicationModel',
'IsaProtocolParameterModel',
'IsaProtocolModel',
'IsaPublicationModel',
'IsaSampleModel',
'IsaSourceModel',
'IsaStudyGroupModel',
'IsaStudyModel',
'JsonCustomDictModel',
'JsonCustomListModel',
'JsonDataset',
Expand Down Expand Up @@ -188,19 +244,29 @@
'SplitToLinesModel',
'JoinLinesModel',
'SplitToItemsModel',
'SplitToItemsModelNew',
'JoinItemsModel',
'SplitLinesToColumnsModel',
'JoinColumnsToLinesModel',
'StrModel',
'TableOfPydanticRecordsDataset',
'TableWithColNamesDataset',
'PydanticRecordModel',
'TableOfPydanticRecordsModel',
'TableWithColNamesModel',
'Params',
'NotIterableExceptStrOrBytesModel',
'Chain2',
'Chain3',
'Chain4',
'Chain5',
'Chain6',
'import_directory',
'split_dataset',
'convert_dataset',
'flatten_isa_json',
'flatten_nested_json',
'convert_dataset_string_to_json',
'transpose_dicts_2_lists',
'transpose_dict_of_dicts_2_list_of_dicts',
'transpose_dicts_of_lists_of_dicts_2_lists_of_dicts',
Expand All @@ -221,10 +287,3 @@
'rename_col_names',
'transpose_columns_with_data_files'
]

#
# def __getattr__(attr_name: str) -> object:
# omnipy = importlib.import_module(__name__)
# all_modules = []
# recursive_module_import(omnipy, all_modules)
# print(all_modules)
Loading

0 comments on commit 2837605

Please sign in to comment.