From 8c66d89ca05bffcddca406861173b7641e5292da Mon Sep 17 00:00:00 2001 From: Sveinung Gundersen Date: Thu, 16 May 2024 07:51:46 +0200 Subject: [PATCH] Tmp commit --- .../isa_json_data_wrangling_example.ipynb | 794 ++---------------- pyproject.toml | 1 + src/omnipy/api/protocols/private/data.py | 24 + src/omnipy/api/protocols/private/util.py | 10 +- src/omnipy/api/protocols/public/config.py | 1 + src/omnipy/api/protocols/public/hub.py | 8 +- src/omnipy/config/data.py | 1 + src/omnipy/config/root_log.py | 2 +- src/omnipy/data/data_class_creator.py | 36 + src/omnipy/data/dataset.py | 11 +- src/omnipy/data/model.py | 137 +-- src/omnipy/engine/local.py | 9 +- src/omnipy/hub/runtime.py | 51 +- src/omnipy/modules/pandas/models.py | 4 +- src/omnipy/modules/prefect/engine/prefect.py | 7 + src/omnipy/util/helpers.py | 336 +++++++- tests/compute/conftest.py | 6 - tests/conftest.py | 30 +- tests/data/helpers/mocks.py | 9 + tests/data/helpers/models.py | 2 +- tests/data/test_data_class_creator.py | 99 +++ tests/data/test_model.py | 134 ++- tests/hub/helpers/mocks.py | 27 +- tests/hub/test_root_log.py | 12 +- tests/hub/test_runtime.py | 84 +- tests/util/test_helpers.py | 359 +++++++- 26 files changed, 1274 insertions(+), 920 deletions(-) create mode 100644 src/omnipy/api/protocols/private/data.py create mode 100644 src/omnipy/data/data_class_creator.py create mode 100644 tests/data/test_data_class_creator.py diff --git a/docs/notebooks/isa_json_data_wrangling_example.ipynb b/docs/notebooks/isa_json_data_wrangling_example.ipynb index a47fc43c..7d6ba05c 100644 --- a/docs/notebooks/isa_json_data_wrangling_example.ipynb +++ b/docs/notebooks/isa_json_data_wrangling_example.ipynb @@ -10,127 +10,10 @@ "start_time": "2024-02-07T20:16:33.675335Z" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requirement already satisfied: omnipy in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (0.14.12)\n", - "Requirement already satisfied: aiostream<0.5.0,>=0.4.5 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from omnipy) (0.4.5)\n", - "Requirement already satisfied: chardet<6.0.0,>=5.2.0 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from omnipy) (5.2.0)\n", - "Requirement already satisfied: devtools<0.13.0,>=0.12.2 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from omnipy) (0.12.2)\n", - "Requirement already satisfied: httpx<0.27.0,>=0.26.0 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from omnipy) (0.26.0)\n", - "Requirement already satisfied: humanize<5.0.0,>=4.9.0 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from omnipy) (4.9.0)\n", - "Requirement already satisfied: inflection<0.6.0,>=0.5.1 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from omnipy) (0.5.1)\n", - "Requirement already satisfied: isort<6.0.0,>=5.12.0 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from omnipy) (5.13.2)\n", - "Requirement already satisfied: objsize<0.8.0,>=0.7.0 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from omnipy) (0.7.0)\n", - "Requirement already satisfied: pandas<2.2 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from omnipy) (2.1.4)\n", - "Requirement already satisfied: pathspec==0.12.1 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from omnipy) (0.12.1)\n", - "Requirement already satisfied: prefect<3.0.0,>=2.7.3 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from omnipy) (2.14.20)\n", - "Requirement already satisfied: pydantic<2.0.0,>=1.10.2 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from omnipy) (1.10.14)\n", - "Requirement already satisfied: pytest-asyncio<0.21.0,>=0.20.2 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from omnipy) (0.20.3)\n", - "Requirement already satisfied: pytest-cases<4.0.0,>=3.6.13 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from omnipy) (3.8.2)\n", - "Requirement already satisfied: requests<3.0.0,>=2.25.1 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from omnipy) (2.31.0)\n", - "Requirement already satisfied: typing-inspect<0.9.0,>=0.8.0 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from omnipy) (0.8.0)\n", - "Requirement already satisfied: asttokens<3.0.0,>=2.0.0 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from devtools<0.13.0,>=0.12.2->omnipy) (2.4.1)\n", - "Requirement already satisfied: executing>=1.1.1 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from devtools<0.13.0,>=0.12.2->omnipy) (2.0.1)\n", - "Requirement already satisfied: pygments>=2.15.0 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from devtools<0.13.0,>=0.12.2->omnipy) (2.17.2)\n", - "Requirement already satisfied: anyio in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from httpx<0.27.0,>=0.26.0->omnipy) (3.7.1)\n", - "Requirement already satisfied: certifi in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from httpx<0.27.0,>=0.26.0->omnipy) (2024.2.2)\n", - "Requirement already satisfied: httpcore==1.* in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from httpx<0.27.0,>=0.26.0->omnipy) (1.0.2)\n", - "Requirement already satisfied: idna in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from httpx<0.27.0,>=0.26.0->omnipy) (3.6)\n", - "Requirement already satisfied: sniffio in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from httpx<0.27.0,>=0.26.0->omnipy) (1.3.0)\n", - "Requirement already satisfied: h11<0.15,>=0.13 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from httpcore==1.*->httpx<0.27.0,>=0.26.0->omnipy) (0.14.0)\n", - "Requirement already satisfied: numpy<2,>=1.22.4 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from pandas<2.2->omnipy) (1.26.3)\n", - "Requirement already satisfied: python-dateutil>=2.8.2 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from pandas<2.2->omnipy) (2.8.2)\n", - "Requirement already satisfied: pytz>=2020.1 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from pandas<2.2->omnipy) (2023.4)\n", - "Requirement already satisfied: tzdata>=2022.1 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from pandas<2.2->omnipy) (2023.4)\n", - "Requirement already satisfied: aiosqlite>=0.17.0 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from prefect<3.0.0,>=2.7.3->omnipy) (0.19.0)\n", - "Requirement already satisfied: alembic<2.0.0,>=1.7.5 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from prefect<3.0.0,>=2.7.3->omnipy) (1.13.1)\n", - "Requirement already satisfied: apprise<2.0.0,>=1.1.0 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from prefect<3.0.0,>=2.7.3->omnipy) (1.7.2)\n", - "Requirement already satisfied: asyncpg>=0.23 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from prefect<3.0.0,>=2.7.3->omnipy) (0.29.0)\n", - "Requirement already satisfied: click<8.2,>=8.0 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from prefect<3.0.0,>=2.7.3->omnipy) (8.1.7)\n", - "Requirement already satisfied: cryptography>=36.0.1 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from prefect<3.0.0,>=2.7.3->omnipy) (42.0.2)\n", - "Requirement already satisfied: dateparser<2.0.0,>=1.1.1 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from prefect<3.0.0,>=2.7.3->omnipy) (1.2.0)\n", - "Requirement already satisfied: docker<7.0,>=4.0 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from prefect<3.0.0,>=2.7.3->omnipy) (6.1.3)\n", - "Requirement already satisfied: graphviz>=0.20.1 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from prefect<3.0.0,>=2.7.3->omnipy) (0.20.1)\n", - "Requirement already satisfied: griffe>=0.20.0 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from prefect<3.0.0,>=2.7.3->omnipy) (0.40.0)\n", - "Requirement already satisfied: jinja2<4.0.0,>=3.0.0 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from prefect<3.0.0,>=2.7.3->omnipy) (3.1.3)\n", - "Requirement already satisfied: kubernetes<30.0.0,>=24.2.0 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from prefect<3.0.0,>=2.7.3->omnipy) (29.0.0)\n", - "Requirement already satisfied: readchar<5.0.0,>=4.0.0 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from prefect<3.0.0,>=2.7.3->omnipy) (4.0.5)\n", - "Requirement already satisfied: sqlalchemy!=1.4.33,<3.0.0,>=1.4.22 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from sqlalchemy[asyncio]!=1.4.33,<3.0.0,>=1.4.22->prefect<3.0.0,>=2.7.3->omnipy) (2.0.25)\n", - "Requirement already satisfied: typer>=0.4.2 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from prefect<3.0.0,>=2.7.3->omnipy) (0.9.0)\n", - "Requirement already satisfied: asgi-lifespan<3.0,>=1.0 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from prefect<3.0.0,>=2.7.3->omnipy) (2.1.0)\n", - "Requirement already satisfied: cachetools<6.0,>=5.3 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from prefect<3.0.0,>=2.7.3->omnipy) (5.3.2)\n", - "Requirement already satisfied: cloudpickle<4.0,>=2.0 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from prefect<3.0.0,>=2.7.3->omnipy) (3.0.0)\n", - "Requirement already satisfied: coolname<3.0.0,>=1.0.4 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from prefect<3.0.0,>=2.7.3->omnipy) (2.2.0)\n", - "Requirement already satisfied: croniter<3.0.0,>=1.0.12 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from prefect<3.0.0,>=2.7.3->omnipy) (2.0.1)\n", - "Requirement already satisfied: fsspec>=2022.5.0 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from prefect<3.0.0,>=2.7.3->omnipy) (2024.2.0)\n", - "Requirement already satisfied: jsonpatch<2.0,>=1.32 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from prefect<3.0.0,>=2.7.3->omnipy) (1.33)\n", - "Requirement already satisfied: jsonschema<5.0.0,>=3.2.0 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from prefect<3.0.0,>=2.7.3->omnipy) (4.21.1)\n", - "Requirement already satisfied: orjson<4.0,>=3.7 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from prefect<3.0.0,>=2.7.3->omnipy) (3.9.13)\n", - "Requirement already satisfied: packaging<24.3,>=21.3 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from prefect<3.0.0,>=2.7.3->omnipy) (23.2)\n", - "Requirement already satisfied: python-slugify<9.0,>=5.0 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from prefect<3.0.0,>=2.7.3->omnipy) (8.0.3)\n", - "Requirement already satisfied: pyyaml<7.0.0,>=5.4.1 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from prefect<3.0.0,>=2.7.3->omnipy) (6.0.1)\n", - "Requirement already satisfied: rich<14.0,>=11.0 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from prefect<3.0.0,>=2.7.3->omnipy) (13.7.0)\n", - "Requirement already satisfied: ruamel.yaml>=0.17.0 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from prefect<3.0.0,>=2.7.3->omnipy) (0.18.5)\n", - "Requirement already satisfied: starlette<0.33.0,>=0.27.0 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from prefect<3.0.0,>=2.7.3->omnipy) (0.32.0.post1)\n", - "Requirement already satisfied: toml>=0.10.0 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from prefect<3.0.0,>=2.7.3->omnipy) (0.10.2)\n", - "Requirement already satisfied: typing-extensions<5.0.0,>=4.5.0 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from prefect<3.0.0,>=2.7.3->omnipy) (4.9.0)\n", - "Requirement already satisfied: ujson<6.0.0,>=5.8.0 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from prefect<3.0.0,>=2.7.3->omnipy) (5.9.0)\n", - "Requirement already satisfied: uvicorn>=0.14.0 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from prefect<3.0.0,>=2.7.3->omnipy) (0.27.0.post1)\n", - "Requirement already satisfied: websockets<13.0,>=10.4 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from prefect<3.0.0,>=2.7.3->omnipy) (12.0)\n", - "Requirement already satisfied: pendulum<3.0 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from prefect<3.0.0,>=2.7.3->omnipy) (2.1.2)\n", - "Requirement already satisfied: pytest>=6.1.0 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from pytest-asyncio<0.21.0,>=0.20.2->omnipy) (7.4.4)\n", - "Requirement already satisfied: decopatch in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from pytest-cases<4.0.0,>=3.6.13->omnipy) (1.4.10)\n", - "Requirement already satisfied: makefun>=1.15.1 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from pytest-cases<4.0.0,>=3.6.13->omnipy) (1.15.2)\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from requests<3.0.0,>=2.25.1->omnipy) (3.3.2)\n", - "Requirement already satisfied: urllib3<3,>=1.21.1 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from requests<3.0.0,>=2.25.1->omnipy) (2.2.0)\n", - "Requirement already satisfied: mypy-extensions>=0.3.0 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from typing-inspect<0.9.0,>=0.8.0->omnipy) (1.0.0)\n", - "Requirement already satisfied: Mako in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from alembic<2.0.0,>=1.7.5->prefect<3.0.0,>=2.7.3->omnipy) (1.3.2)\n", - "Requirement already satisfied: exceptiongroup in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from anyio->httpx<0.27.0,>=0.26.0->omnipy) (1.2.0)\n", - "Requirement already satisfied: requests-oauthlib in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from apprise<2.0.0,>=1.1.0->prefect<3.0.0,>=2.7.3->omnipy) (1.3.1)\n", - "Requirement already satisfied: markdown in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from apprise<2.0.0,>=1.1.0->prefect<3.0.0,>=2.7.3->omnipy) (3.5.2)\n", - "Requirement already satisfied: six>=1.12.0 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from asttokens<3.0.0,>=2.0.0->devtools<0.13.0,>=0.12.2->omnipy) (1.16.0)\n", - "Requirement already satisfied: async-timeout>=4.0.3 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from asyncpg>=0.23->prefect<3.0.0,>=2.7.3->omnipy) (4.0.3)\n", - "Requirement already satisfied: cffi>=1.12 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from cryptography>=36.0.1->prefect<3.0.0,>=2.7.3->omnipy) (1.16.0)\n", - "Requirement already satisfied: regex!=2019.02.19,!=2021.8.27 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from dateparser<2.0.0,>=1.1.1->prefect<3.0.0,>=2.7.3->omnipy) (2023.12.25)\n", - "Requirement already satisfied: tzlocal in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from dateparser<2.0.0,>=1.1.1->prefect<3.0.0,>=2.7.3->omnipy) (5.2)\n", - "Requirement already satisfied: websocket-client>=0.32.0 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from docker<7.0,>=4.0->prefect<3.0.0,>=2.7.3->omnipy) (1.7.0)\n", - "Requirement already satisfied: colorama>=0.4 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from griffe>=0.20.0->prefect<3.0.0,>=2.7.3->omnipy) (0.4.6)\n", - "Requirement already satisfied: h2<5,>=3 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from httpx[http2]!=0.23.2,>=0.23->prefect<3.0.0,>=2.7.3->omnipy) (4.1.0)\n", - "Requirement already satisfied: MarkupSafe>=2.0 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from jinja2<4.0.0,>=3.0.0->prefect<3.0.0,>=2.7.3->omnipy) (2.1.5)\n", - "Requirement already satisfied: jsonpointer>=1.9 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from jsonpatch<2.0,>=1.32->prefect<3.0.0,>=2.7.3->omnipy) (2.4)\n", - "Requirement already satisfied: attrs>=22.2.0 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from jsonschema<5.0.0,>=3.2.0->prefect<3.0.0,>=2.7.3->omnipy) (23.2.0)\n", - "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from jsonschema<5.0.0,>=3.2.0->prefect<3.0.0,>=2.7.3->omnipy) (2023.12.1)\n", - "Requirement already satisfied: referencing>=0.28.4 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from jsonschema<5.0.0,>=3.2.0->prefect<3.0.0,>=2.7.3->omnipy) (0.33.0)\n", - "Requirement already satisfied: rpds-py>=0.7.1 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from jsonschema<5.0.0,>=3.2.0->prefect<3.0.0,>=2.7.3->omnipy) (0.17.1)\n", - "Requirement already satisfied: google-auth>=1.0.1 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from kubernetes<30.0.0,>=24.2.0->prefect<3.0.0,>=2.7.3->omnipy) (2.27.0)\n", - "Requirement already satisfied: oauthlib>=3.2.2 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from kubernetes<30.0.0,>=24.2.0->prefect<3.0.0,>=2.7.3->omnipy) (3.2.2)\n", - "Requirement already satisfied: pytzdata>=2020.1 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from pendulum<3.0->prefect<3.0.0,>=2.7.3->omnipy) (2020.1)\n", - "Requirement already satisfied: email-validator>=1.0.3 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from pydantic[email]!=2.0.0,!=2.0.1,!=2.1.0,<3.0.0,>=1.10.0->prefect<3.0.0,>=2.7.3->omnipy) (2.1.0.post1)\n", - "Requirement already satisfied: iniconfig in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from pytest>=6.1.0->pytest-asyncio<0.21.0,>=0.20.2->omnipy) (2.0.0)\n", - "Requirement already satisfied: pluggy<2.0,>=0.12 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from pytest>=6.1.0->pytest-asyncio<0.21.0,>=0.20.2->omnipy) (1.4.0)\n", - "Requirement already satisfied: tomli>=1.0.0 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from pytest>=6.1.0->pytest-asyncio<0.21.0,>=0.20.2->omnipy) (2.0.1)\n", - "Requirement already satisfied: text-unidecode>=1.3 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from python-slugify<9.0,>=5.0->prefect<3.0.0,>=2.7.3->omnipy) (1.3)\n", - "Requirement already satisfied: setuptools>=41.0 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from readchar<5.0.0,>=4.0.0->prefect<3.0.0,>=2.7.3->omnipy) (69.0.3)\n", - "Requirement already satisfied: markdown-it-py>=2.2.0 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from rich<14.0,>=11.0->prefect<3.0.0,>=2.7.3->omnipy) (3.0.0)\n", - "Requirement already satisfied: ruamel.yaml.clib>=0.2.7 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from ruamel.yaml>=0.17.0->prefect<3.0.0,>=2.7.3->omnipy) (0.2.8)\n", - "Requirement already satisfied: greenlet!=0.4.17 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from sqlalchemy[asyncio]!=1.4.33,<3.0.0,>=1.4.22->prefect<3.0.0,>=2.7.3->omnipy) (3.0.3)\n", - "Requirement already satisfied: pycparser in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from cffi>=1.12->cryptography>=36.0.1->prefect<3.0.0,>=2.7.3->omnipy) (2.21)\n", - "Requirement already satisfied: dnspython>=2.0.0 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from email-validator>=1.0.3->pydantic[email]!=2.0.0,!=2.0.1,!=2.1.0,<3.0.0,>=1.10.0->prefect<3.0.0,>=2.7.3->omnipy) (2.5.0)\n", - "Requirement already satisfied: pyasn1-modules>=0.2.1 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from google-auth>=1.0.1->kubernetes<30.0.0,>=24.2.0->prefect<3.0.0,>=2.7.3->omnipy) (0.3.0)\n", - "Requirement already satisfied: rsa<5,>=3.1.4 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from google-auth>=1.0.1->kubernetes<30.0.0,>=24.2.0->prefect<3.0.0,>=2.7.3->omnipy) (4.9)\n", - "Requirement already satisfied: hyperframe<7,>=6.0 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from h2<5,>=3->httpx[http2]!=0.23.2,>=0.23->prefect<3.0.0,>=2.7.3->omnipy) (6.0.1)\n", - "Requirement already satisfied: hpack<5,>=4.0 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from h2<5,>=3->httpx[http2]!=0.23.2,>=0.23->prefect<3.0.0,>=2.7.3->omnipy) (4.0.0)\n", - "Requirement already satisfied: mdurl~=0.1 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from markdown-it-py>=2.2.0->rich<14.0,>=11.0->prefect<3.0.0,>=2.7.3->omnipy) (0.1.2)\n", - "Requirement already satisfied: pyasn1<0.6.0,>=0.4.6 in /Users/sveinugu/PycharmProjects/omnipy/.venv/lib/python3.10/site-packages (from pyasn1-modules>=0.2.1->google-auth>=1.0.1->kubernetes<30.0.0,>=24.2.0->prefect<3.0.0,>=2.7.3->omnipy) (0.5.1)\n" - ] - } - ], "source": [ "!pip install omnipy" - ] + ], + "outputs": [] }, { "cell_type": "code", @@ -146,10 +29,10 @@ "outputs_hidden": false } }, - "outputs": [], "source": [ "from omnipy import JsonDataset, IsaJsonDataset" - ] + ], + "outputs": [] }, { "cell_type": "code", @@ -165,10 +48,10 @@ "outputs_hidden": false } }, - "outputs": [], "source": [ "isa_ds = IsaJsonDataset()" - ] + ], + "outputs": [] }, { "cell_type": "code", @@ -184,39 +67,10 @@ "outputs_hidden": false } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Downloading https://raw.githubusercontent.com/elixir-europe/biohackathon-projects-2023/256ec768e646f7767d414d22ba1e3deac9436daa/27/isa-test-data/ISA-BH2023-ALL/isa-bh2023-all.json...\n", - "Creating compressed file /var/folders/11/0t40j64s3xz2n9_620p1qdw80000gp/T/tmp87jr8aj5/isa-bh2023-all.json.tar.gz from the contents of \"/var/folders/11/0t40j64s3xz2n9_620p1qdw80000gp/T/tmp87jr8aj5/isa-bh2023-all.json\"\n", - "Reading dataset from a gzipped tarpack at \"/var/folders/11/0t40j64s3xz2n9_620p1qdw80000gp/T/tmp87jr8aj5/isa-bh2023-all.json.tar.gz\" with serializer type: \"JsonDatasetToTarFileSerializer\"\n" - ] - }, - { - "ename": "ValidationError", - "evalue": "106 validation errors for IsaJsonModel\n__root__ -> studies -> 0 -> __root__ -> submissionDate\n invalid datetime format (type=value_error.datetime)\n__root__ -> studies -> 0 -> __root__ -> submissionDate\n invalid date format (type=value_error.date)\n__root__ -> studies -> 0 -> __root__ -> submissionDate\n ensure this value has at most 0 characters (type=value_error.any_str.max_length; limit_value=0)\n__root__ -> studies -> 0 -> __root__ -> publicReleaseDate\n invalid datetime format (type=value_error.datetime)\n__root__ -> studies -> 0 -> __root__ -> publicReleaseDate\n invalid date format (type=value_error.date)\n__root__ -> studies -> 0 -> __root__ -> publicReleaseDate\n ensure this value has at most 0 characters (type=value_error.any_str.max_length; limit_value=0)\n__root__ -> studies -> 0 -> __root__ -> people -> 1 -> __root__ -> email\n value is not a valid email address (type=value_error.email)\n__root__ -> studies -> 0 -> __root__ -> people -> 1 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 1 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 1 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 2 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 2 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 3 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 3 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 4 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 4 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 5 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 5 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 6 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 6 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 7 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 7 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 8 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 8 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 1 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 1 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 2 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 2 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 3 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 3 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 4 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 4 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 5 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 5 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 6 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 6 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 7 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 7 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 8 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 8 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 9 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 9 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 10 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 10 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 11 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 11 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 12 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 12 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 13 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 13 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 14 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 14 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 15 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 15 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 16 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 16 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 17 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 17 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 18 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 18 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 19 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 19 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 20 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 20 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 21 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 21 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 22 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 22 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 23 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 23 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 24 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 24 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 25 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 25 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 26 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 26 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 27 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 27 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 28 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 28 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 29 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 29 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 30 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 30 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 31 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 31 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 32 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 32 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 2 -> __root__ -> dataFiles -> 1 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 2 -> __root__ -> dataFiles -> 1 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 2 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__\n (type=assertion_error)\n__root__ -> __root__ -> comments\n extra fields not permitted (type=value_error.extra)\n__root__ -> __root__ -> description\n extra fields not permitted (type=value_error.extra)\n__root__ -> __root__ -> identifier\n extra fields not permitted (type=value_error.extra)\n__root__ -> __root__ -> ontologySourceReferences\n extra fields not permitted (type=value_error.extra)\n__root__ -> __root__ -> people\n extra fields not permitted (type=value_error.extra)\n__root__ -> __root__ -> publicReleaseDate\n extra fields not permitted (type=value_error.extra)\n__root__ -> __root__ -> publications\n extra fields not permitted (type=value_error.extra)\n__root__ -> __root__ -> studies\n extra fields not permitted (type=value_error.extra)\n__root__ -> __root__ -> submissionDate\n extra fields not permitted (type=value_error.extra)\n__root__ -> __root__ -> title\n extra fields not permitted (type=value_error.extra)\n__root__ -> __root__\n (type=assertion_error)\n__root__\n (type=assertion_error)", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mValidationError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[28], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43misa_ds\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mload\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mhttps://raw.githubusercontent.com/elixir-europe/biohackathon-projects-2023/256ec768e646f7767d414d22ba1e3deac9436daa/27/isa-test-data/ISA-BH2023-ALL/isa-bh2023-all.json\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/PycharmProjects/omnipy/src/omnipy/data/dataset.py:470\u001b[0m, in \u001b[0;36mDataset.load\u001b[0;34m(self, path_or_url, by_file_suffix)\u001b[0m\n\u001b[1;32m 467\u001b[0m loaded_dataset \u001b[38;5;241m=\u001b[39m serializer_registry\u001b[38;5;241m.\u001b[39mload_from_tar_file_path_based_on_dataset_cls(\n\u001b[1;32m 468\u001b[0m \u001b[38;5;28mself\u001b[39m, tar_gz_file_path, \u001b[38;5;28mself\u001b[39m)\n\u001b[1;32m 469\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m loaded_dataset \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 470\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mabsorb\u001b[49m\u001b[43m(\u001b[49m\u001b[43mloaded_dataset\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 471\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m\n\u001b[1;32m 472\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n", - "File \u001b[0;32m~/PycharmProjects/omnipy/src/omnipy/data/dataset.py:365\u001b[0m, in \u001b[0;36mDataset.absorb\u001b[0;34m(self, other)\u001b[0m\n\u001b[1;32m 364\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mabsorb\u001b[39m(\u001b[38;5;28mself\u001b[39m, other: \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mDataset\u001b[39m\u001b[38;5;124m'\u001b[39m):\n\u001b[0;32m--> 365\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfrom_data\u001b[49m\u001b[43m(\u001b[49m\u001b[43mother\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mto_data\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mupdate\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/PycharmProjects/omnipy/src/omnipy/data/dataset.py:346\u001b[0m, in \u001b[0;36mDataset.from_data\u001b[0;34m(self, data, update)\u001b[0m\n\u001b[1;32m 343\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mcallback_func\u001b[39m(model: ModelT, contents: Any):\n\u001b[1;32m 344\u001b[0m model\u001b[38;5;241m.\u001b[39mfrom_data(contents)\n\u001b[0;32m--> 346\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_from_dict_with_callback\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mupdate\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcallback_func\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/PycharmProjects/omnipy/src/omnipy/data/dataset.py:361\u001b[0m, in \u001b[0;36mDataset._from_dict_with_callback\u001b[0;34m(self, data, update, callback_func)\u001b[0m\n\u001b[1;32m 359\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m data_file, contents \u001b[38;5;129;01min\u001b[39;00m data\u001b[38;5;241m.\u001b[39mitems():\n\u001b[1;32m 360\u001b[0m new_model \u001b[38;5;241m=\u001b[39m model_cls()\n\u001b[0;32m--> 361\u001b[0m \u001b[43mcallback_func\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnew_model\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcontents\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 362\u001b[0m \u001b[38;5;28mself\u001b[39m[data_file] \u001b[38;5;241m=\u001b[39m new_model\n", - "File \u001b[0;32m~/PycharmProjects/omnipy/src/omnipy/data/dataset.py:344\u001b[0m, in \u001b[0;36mDataset.from_data..callback_func\u001b[0;34m(model, contents)\u001b[0m\n\u001b[1;32m 343\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mcallback_func\u001b[39m(model: ModelT, contents: Any):\n\u001b[0;32m--> 344\u001b[0m \u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfrom_data\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcontents\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/PycharmProjects/omnipy/src/omnipy/data/model.py:501\u001b[0m, in \u001b[0;36mModel.from_data\u001b[0;34m(self, value)\u001b[0m\n\u001b[1;32m 500\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mfrom_data\u001b[39m(\u001b[38;5;28mself\u001b[39m, value: \u001b[38;5;28mobject\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 501\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_validate_and_set_contents\u001b[49m\u001b[43m(\u001b[49m\u001b[43mvalue\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/PycharmProjects/omnipy/src/omnipy/data/model.py:403\u001b[0m, in \u001b[0;36mModel._validate_and_set_contents\u001b[0;34m(self, new_contents)\u001b[0m\n\u001b[1;32m 402\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_validate_and_set_contents\u001b[39m(\u001b[38;5;28mself\u001b[39m, new_contents: RootT) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 403\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcontents \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_validate_contents_from_value\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnew_contents\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 404\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_take_snapshot_of_validated_contents()\n", - "File \u001b[0;32m~/PycharmProjects/omnipy/src/omnipy/data/model.py:413\u001b[0m, in \u001b[0;36mModel._validate_contents_from_value\u001b[0;34m(self, value)\u001b[0m\n\u001b[1;32m 411\u001b[0m values, fields_set, validation_error \u001b[38;5;241m=\u001b[39m validate_model(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__class__\u001b[39m, {ROOT_KEY: value})\n\u001b[1;32m 412\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m validation_error:\n\u001b[0;32m--> 413\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m validation_error\n\u001b[1;32m 414\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m values[ROOT_KEY]\n", - "\u001b[0;31mValidationError\u001b[0m: 106 validation errors for IsaJsonModel\n__root__ -> studies -> 0 -> __root__ -> submissionDate\n invalid datetime format (type=value_error.datetime)\n__root__ -> studies -> 0 -> __root__ -> submissionDate\n invalid date format (type=value_error.date)\n__root__ -> studies -> 0 -> __root__ -> submissionDate\n ensure this value has at most 0 characters (type=value_error.any_str.max_length; limit_value=0)\n__root__ -> studies -> 0 -> __root__ -> publicReleaseDate\n invalid datetime format (type=value_error.datetime)\n__root__ -> studies -> 0 -> __root__ -> publicReleaseDate\n invalid date format (type=value_error.date)\n__root__ -> studies -> 0 -> __root__ -> publicReleaseDate\n ensure this value has at most 0 characters (type=value_error.any_str.max_length; limit_value=0)\n__root__ -> studies -> 0 -> __root__ -> people -> 1 -> __root__ -> email\n value is not a valid email address (type=value_error.email)\n__root__ -> studies -> 0 -> __root__ -> people -> 1 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 1 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 1 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 2 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 2 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 3 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 3 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 4 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 4 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 5 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 5 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 6 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 6 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 7 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 7 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 8 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 8 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 1 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 1 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 2 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 2 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 3 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 3 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 4 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 4 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 5 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 5 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 6 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 6 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 7 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 7 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 8 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 8 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 9 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 9 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 10 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 10 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 11 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 11 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 12 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 12 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 13 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 13 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 14 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 14 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 15 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 15 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 16 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 16 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 17 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 17 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 18 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 18 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 19 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 19 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 20 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 20 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 21 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 21 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 22 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 22 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 23 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 23 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 24 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 24 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 25 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 25 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 26 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 26 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 27 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 27 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 28 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 28 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 29 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 29 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 30 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 30 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 31 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 31 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 32 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 32 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 2 -> __root__ -> dataFiles -> 1 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 2 -> __root__ -> dataFiles -> 1 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 2 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__\n (type=assertion_error)\n__root__ -> __root__ -> comments\n extra fields not permitted (type=value_error.extra)\n__root__ -> __root__ -> description\n extra fields not permitted (type=value_error.extra)\n__root__ -> __root__ -> identifier\n extra fields not permitted (type=value_error.extra)\n__root__ -> __root__ -> ontologySourceReferences\n extra fields not permitted (type=value_error.extra)\n__root__ -> __root__ -> people\n extra fields not permitted (type=value_error.extra)\n__root__ -> __root__ -> publicReleaseDate\n extra fields not permitted (type=value_error.extra)\n__root__ -> __root__ -> publications\n extra fields not permitted (type=value_error.extra)\n__root__ -> __root__ -> studies\n extra fields not permitted (type=value_error.extra)\n__root__ -> __root__ -> submissionDate\n extra fields not permitted (type=value_error.extra)\n__root__ -> __root__ -> title\n extra fields not permitted (type=value_error.extra)\n__root__ -> __root__\n (type=assertion_error)\n__root__\n (type=assertion_error)" - ] - } - ], "source": [ "#isa_ds.load('https://raw.githubusercontent.com/elixir-europe/biohackathon-projects-2023/256ec768e646f7767d414d22ba1e3deac9436daa/27/isa-test-data/ISA-BH2023-ALL/isa-bh2023-all.json')" - ] + ], + "outputs": [] }, { "cell_type": "code", @@ -232,10 +86,10 @@ "outputs_hidden": false } }, - "outputs": [], "source": [ "json_ds = JsonDataset()" - ] + ], + "outputs": [] }, { "cell_type": "code", @@ -251,20 +105,10 @@ "outputs_hidden": false } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Downloading https://raw.githubusercontent.com/elixir-europe/biohackathon-projects-2023/256ec768e646f7767d414d22ba1e3deac9436daa/27/isa-test-data/ISA-BH2023-ALL/isa-bh2023-all.json...\n", - "Creating compressed file /var/folders/11/0t40j64s3xz2n9_620p1qdw80000gp/T/tmpas_oyzw_/isa-bh2023-all.json.tar.gz from the contents of \"/var/folders/11/0t40j64s3xz2n9_620p1qdw80000gp/T/tmpas_oyzw_/isa-bh2023-all.json\"\n", - "Reading dataset from a gzipped tarpack at \"/var/folders/11/0t40j64s3xz2n9_620p1qdw80000gp/T/tmpas_oyzw_/isa-bh2023-all.json.tar.gz\" with serializer type: \"JsonDatasetToTarFileSerializer\"\n" - ] - } - ], "source": [ "json_ds.load('https://raw.githubusercontent.com/elixir-europe/biohackathon-projects-2023/256ec768e646f7767d414d22ba1e3deac9436daa/27/isa-test-data/ISA-BH2023-ALL/isa-bh2023-all.json')" - ] + ], + "outputs": [] }, { "cell_type": "code", @@ -280,25 +124,10 @@ "outputs_hidden": false } }, - "outputs": [ - { - "data": { - "text/plain": [ - "╭─────┬──────────────────┬───────────┬──────────┬────────────────────╮\n", - "│ # │ Data file name │ Type │ Length │ Size (in memory) │\n", - "├─────┼──────────────────┼───────────┼──────────┼────────────────────┤\n", - "│ 0 │ isa-bh2023-all │ JsonModel │ 10 │ 3.2 MB │\n", - "╰─────┴──────────────────┴───────────┴──────────┴────────────────────╯" - ] - }, - "execution_count": 27, - "metadata": {}, - "output_type": "execute_result" - } - ], "source": [ "json_ds" - ] + ], + "outputs": [] }, { "cell_type": "code", @@ -314,41 +143,10 @@ "outputs_hidden": false } }, - "outputs": [ - { - "data": { - "text/plain": [ - "╭───────┬───────────────────────────────────╮\n", - "│ # │ JsonModel │\n", - "├───────┼───────────────────────────────────┤\n", - "│ 0 │ JsonModel( │\n", - "│ 1 │ JsonAnyDictM( │\n", - "│ 2 │ { │\n", - "│ 3 │ 'comments': JsonAnyListM( │\n", - "│ 4 │ [], │\n", - "│ 5 │ ), │\n", - "│ 6 │ 'description': JsonScalarM( │\n", - "│ 7 │ '', │\n", - "├───────┼───────────────────────────────────┤\n", - "│ 28523 │ '', │\n", - "│ 28524 │ ), │\n", - "│ 28525 │ 'title': JsonScalarM( │\n", - "│ 28526 │ '', │\n", - "│ 28527 │ ), │\n", - "│ 28528 │ }, │\n", - "│ 28529 │ ), │\n", - "│ 28530 │ ) (JsonModel) len=10 │\n", - "╰───────┴───────────────────────────────────╯" - ] - }, - "execution_count": 29, - "metadata": {}, - "output_type": "execute_result" - } - ], "source": [ "json_ds[0]" - ] + ], + "outputs": [] }, { "cell_type": "code", @@ -360,100 +158,69 @@ "outputs_hidden": false } }, - "outputs": [ - { - "ename": "ValidationError", - "evalue": "106 validation errors for IsaJsonModel\n__root__ -> studies -> 0 -> __root__ -> submissionDate\n invalid datetime format (type=value_error.datetime)\n__root__ -> studies -> 0 -> __root__ -> submissionDate\n invalid date format (type=value_error.date)\n__root__ -> studies -> 0 -> __root__ -> submissionDate\n ensure this value has at most 0 characters (type=value_error.any_str.max_length; limit_value=0)\n__root__ -> studies -> 0 -> __root__ -> publicReleaseDate\n invalid datetime format (type=value_error.datetime)\n__root__ -> studies -> 0 -> __root__ -> publicReleaseDate\n invalid date format (type=value_error.date)\n__root__ -> studies -> 0 -> __root__ -> publicReleaseDate\n ensure this value has at most 0 characters (type=value_error.any_str.max_length; limit_value=0)\n__root__ -> studies -> 0 -> __root__ -> people -> 1 -> __root__ -> email\n value is not a valid email address (type=value_error.email)\n__root__ -> studies -> 0 -> __root__ -> people -> 1 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 1 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 1 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 2 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 2 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 3 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 3 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 4 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 4 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 5 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 5 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 6 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 6 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 7 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 7 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 8 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 8 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 1 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 1 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 2 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 2 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 3 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 3 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 4 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 4 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 5 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 5 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 6 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 6 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 7 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 7 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 8 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 8 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 9 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 9 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 10 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 10 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 11 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 11 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 12 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 12 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 13 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 13 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 14 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 14 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 15 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 15 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 16 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 16 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 17 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 17 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 18 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 18 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 19 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 19 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 20 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 20 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 21 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 21 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 22 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 22 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 23 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 23 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 24 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 24 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 25 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 25 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 26 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 26 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 27 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 27 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 28 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 28 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 29 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 29 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 30 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 30 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 31 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 31 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 32 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 32 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 2 -> __root__ -> dataFiles -> 1 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 2 -> __root__ -> dataFiles -> 1 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 2 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__\n (type=assertion_error)\n__root__ -> __root__ -> comments\n extra fields not permitted (type=value_error.extra)\n__root__ -> __root__ -> description\n extra fields not permitted (type=value_error.extra)\n__root__ -> __root__ -> identifier\n extra fields not permitted (type=value_error.extra)\n__root__ -> __root__ -> ontologySourceReferences\n extra fields not permitted (type=value_error.extra)\n__root__ -> __root__ -> people\n extra fields not permitted (type=value_error.extra)\n__root__ -> __root__ -> publicReleaseDate\n extra fields not permitted (type=value_error.extra)\n__root__ -> __root__ -> publications\n extra fields not permitted (type=value_error.extra)\n__root__ -> __root__ -> studies\n extra fields not permitted (type=value_error.extra)\n__root__ -> __root__ -> submissionDate\n extra fields not permitted (type=value_error.extra)\n__root__ -> __root__ -> title\n extra fields not permitted (type=value_error.extra)\n__root__ -> __root__\n (type=assertion_error)\n__root__\n (type=assertion_error)", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mValidationError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[30], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43misa_ds\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mabsorb\u001b[49m\u001b[43m(\u001b[49m\u001b[43mjson_ds\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/PycharmProjects/omnipy/src/omnipy/data/dataset.py:365\u001b[0m, in \u001b[0;36mDataset.absorb\u001b[0;34m(self, other)\u001b[0m\n\u001b[1;32m 364\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mabsorb\u001b[39m(\u001b[38;5;28mself\u001b[39m, other: \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mDataset\u001b[39m\u001b[38;5;124m'\u001b[39m):\n\u001b[0;32m--> 365\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfrom_data\u001b[49m\u001b[43m(\u001b[49m\u001b[43mother\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mto_data\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mupdate\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/PycharmProjects/omnipy/src/omnipy/data/dataset.py:346\u001b[0m, in \u001b[0;36mDataset.from_data\u001b[0;34m(self, data, update)\u001b[0m\n\u001b[1;32m 343\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mcallback_func\u001b[39m(model: ModelT, contents: Any):\n\u001b[1;32m 344\u001b[0m model\u001b[38;5;241m.\u001b[39mfrom_data(contents)\n\u001b[0;32m--> 346\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_from_dict_with_callback\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mupdate\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcallback_func\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/PycharmProjects/omnipy/src/omnipy/data/dataset.py:361\u001b[0m, in \u001b[0;36mDataset._from_dict_with_callback\u001b[0;34m(self, data, update, callback_func)\u001b[0m\n\u001b[1;32m 359\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m data_file, contents \u001b[38;5;129;01min\u001b[39;00m data\u001b[38;5;241m.\u001b[39mitems():\n\u001b[1;32m 360\u001b[0m new_model \u001b[38;5;241m=\u001b[39m model_cls()\n\u001b[0;32m--> 361\u001b[0m \u001b[43mcallback_func\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnew_model\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcontents\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 362\u001b[0m \u001b[38;5;28mself\u001b[39m[data_file] \u001b[38;5;241m=\u001b[39m new_model\n", - "File \u001b[0;32m~/PycharmProjects/omnipy/src/omnipy/data/dataset.py:344\u001b[0m, in \u001b[0;36mDataset.from_data..callback_func\u001b[0;34m(model, contents)\u001b[0m\n\u001b[1;32m 343\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mcallback_func\u001b[39m(model: ModelT, contents: Any):\n\u001b[0;32m--> 344\u001b[0m \u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfrom_data\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcontents\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/PycharmProjects/omnipy/src/omnipy/data/model.py:501\u001b[0m, in \u001b[0;36mModel.from_data\u001b[0;34m(self, value)\u001b[0m\n\u001b[1;32m 500\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mfrom_data\u001b[39m(\u001b[38;5;28mself\u001b[39m, value: \u001b[38;5;28mobject\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 501\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_validate_and_set_contents\u001b[49m\u001b[43m(\u001b[49m\u001b[43mvalue\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/PycharmProjects/omnipy/src/omnipy/data/model.py:403\u001b[0m, in \u001b[0;36mModel._validate_and_set_contents\u001b[0;34m(self, new_contents)\u001b[0m\n\u001b[1;32m 402\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_validate_and_set_contents\u001b[39m(\u001b[38;5;28mself\u001b[39m, new_contents: RootT) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 403\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcontents \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_validate_contents_from_value\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnew_contents\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 404\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_take_snapshot_of_validated_contents()\n", - "File \u001b[0;32m~/PycharmProjects/omnipy/src/omnipy/data/model.py:413\u001b[0m, in \u001b[0;36mModel._validate_contents_from_value\u001b[0;34m(self, value)\u001b[0m\n\u001b[1;32m 411\u001b[0m values, fields_set, validation_error \u001b[38;5;241m=\u001b[39m validate_model(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__class__\u001b[39m, {ROOT_KEY: value})\n\u001b[1;32m 412\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m validation_error:\n\u001b[0;32m--> 413\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m validation_error\n\u001b[1;32m 414\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m values[ROOT_KEY]\n", - "\u001b[0;31mValidationError\u001b[0m: 106 validation errors for IsaJsonModel\n__root__ -> studies -> 0 -> __root__ -> submissionDate\n invalid datetime format (type=value_error.datetime)\n__root__ -> studies -> 0 -> __root__ -> submissionDate\n invalid date format (type=value_error.date)\n__root__ -> studies -> 0 -> __root__ -> submissionDate\n ensure this value has at most 0 characters (type=value_error.any_str.max_length; limit_value=0)\n__root__ -> studies -> 0 -> __root__ -> publicReleaseDate\n invalid datetime format (type=value_error.datetime)\n__root__ -> studies -> 0 -> __root__ -> publicReleaseDate\n invalid date format (type=value_error.date)\n__root__ -> studies -> 0 -> __root__ -> publicReleaseDate\n ensure this value has at most 0 characters (type=value_error.any_str.max_length; limit_value=0)\n__root__ -> studies -> 0 -> __root__ -> people -> 1 -> __root__ -> email\n value is not a valid email address (type=value_error.email)\n__root__ -> studies -> 0 -> __root__ -> people -> 1 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 1 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 1 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 2 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 2 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 3 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 3 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 4 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 4 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 5 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 5 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 6 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 6 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 7 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 7 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 8 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__ -> dataFiles -> 8 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 0 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 1 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 1 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 2 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 2 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 3 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 3 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 4 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 4 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 5 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 5 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 6 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 6 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 7 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 7 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 8 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 8 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 9 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 9 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 10 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 10 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 11 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 11 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 12 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 12 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 13 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 13 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 14 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 14 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 15 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 15 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 16 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 16 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 17 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 17 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 18 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 18 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 19 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 19 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 20 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 20 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 21 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 21 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 22 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 22 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 23 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 23 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 24 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 24 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 25 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 25 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 26 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 26 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 27 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 27 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 28 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 28 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 29 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 29 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 30 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 30 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 31 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 31 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 32 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__ -> dataFiles -> 32 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 1 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 2 -> __root__ -> dataFiles -> 1 -> __root__ -> type\n value is not a valid enumeration member; permitted: 'Raw Data File', 'Derived Data File', 'Image File', 'Acquisition Parameter Data File', 'Derived Spectral Data File', 'Protein Assignment File', 'Raw Spectral Data File', 'Peptide Assignment File', 'Array Data File', 'Derived Array Data File', 'Post Translational Modification Assignment File', 'Derived Array Data Matrix File', 'Free Induction Decay Data File', 'Metabolite Assignment File', 'Array Data Matrix File' (type=type_error.enum; enum_values=[, , , , , , , , , , , , , , ])\n__root__ -> studies -> 0 -> __root__ -> assays -> 2 -> __root__ -> dataFiles -> 1 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__ -> assays -> 2 -> __root__\n (type=assertion_error)\n__root__ -> studies -> 0 -> __root__\n (type=assertion_error)\n__root__ -> __root__ -> comments\n extra fields not permitted (type=value_error.extra)\n__root__ -> __root__ -> description\n extra fields not permitted (type=value_error.extra)\n__root__ -> __root__ -> identifier\n extra fields not permitted (type=value_error.extra)\n__root__ -> __root__ -> ontologySourceReferences\n extra fields not permitted (type=value_error.extra)\n__root__ -> __root__ -> people\n extra fields not permitted (type=value_error.extra)\n__root__ -> __root__ -> publicReleaseDate\n extra fields not permitted (type=value_error.extra)\n__root__ -> __root__ -> publications\n extra fields not permitted (type=value_error.extra)\n__root__ -> __root__ -> studies\n extra fields not permitted (type=value_error.extra)\n__root__ -> __root__ -> submissionDate\n extra fields not permitted (type=value_error.extra)\n__root__ -> __root__ -> title\n extra fields not permitted (type=value_error.extra)\n__root__ -> __root__\n (type=assertion_error)\n__root__\n (type=assertion_error)" - ] - } - ], "source": [ "#isa_ds.absorb(json_ds)" - ] + ], + "outputs": [] }, { "cell_type": "code", "execution_count": 10, "id": "fdc69c1d-8a9d-4246-8321-9d72af9cbe93", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'15/08/2021'" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], "source": [ "json_ds[0]['studies'][0]['submissionDate'].contents" - ] + ], + "outputs": [] }, { "cell_type": "code", "execution_count": 34, "id": "e86a1594-431e-458f-ac2b-63dc0636ff52", "metadata": {}, - "outputs": [], "source": [ "from datetime import datetime\n", "\n", "def to_iso_format(date_str: str) -> str:\n", " return datetime.strptime(date_str, '%d/%m/%Y').date().isoformat()" - ] + ], + "outputs": [] }, { "cell_type": "code", "execution_count": 35, "id": "ca11cd2e-198b-47db-9f1a-5b071eb3f409", "metadata": {}, - "outputs": [], "source": [ "json_ds[0]['studies'][0]['submissionDate'] = to_iso_format(json_ds[0]['studies'][0]['submissionDate'].contents)" - ] + ], + "outputs": [] }, { "cell_type": "code", "execution_count": 36, "id": "939166d5-b9f0-40e8-a68d-a965072c4eb4", "metadata": {}, - "outputs": [], "source": [ "json_ds[0]['studies'][0]['publicReleaseDate'] = to_iso_format(json_ds[0]['studies'][0]['publicReleaseDate'].contents)" - ] + ], + "outputs": [] }, { "cell_type": "code", "execution_count": 37, "id": "73482924-df6c-46e7-a59a-4de280b5243c", "metadata": {}, - "outputs": [], "source": [ "json_ds[0]['studies'][0]['people'][1]['email'] = 'somebody@somewhere.com'" - ] + ], + "outputs": [] }, { "cell_type": "code", "execution_count": 42, "id": "2dbfb495-4808-4db7-87ae-2c177c1f2c92", "metadata": {}, - "outputs": [], "source": [ "file_type_map = {\n", " 'Spectral Raw Data File': 'Raw Spectral Data File',\n", @@ -473,42 +240,28 @@ "# Possible new syntax:\n", "#\n", "# json_ds[0]['studies'][:]['assays'][:]['dataFiles'][:].for_each(fix_file_type)" - ] + ], + "outputs": [] }, { "cell_type": "code", "execution_count": 39, "id": "2ced82c6-e776-4761-b258-b1a2c5b1a770", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "╭─────┬──────────────────┬───────────┬──────────┬────────────────────╮\n", - "│ # │ Data file name │ Type │ Length │ Size (in memory) │\n", - "├─────┼──────────────────┼───────────┼──────────┼────────────────────┤\n", - "│ 0 │ isa-bh2023-all │ JsonModel │ 10 │ 3.2 MB │\n", - "╰─────┴──────────────────┴───────────┴──────────┴────────────────────╯" - ] - }, - "execution_count": 39, - "metadata": {}, - "output_type": "execute_result" - } - ], "source": [ "json_ds" - ] + ], + "outputs": [] }, { "cell_type": "code", "execution_count": 40, "id": "43627300-7267-4745-b1c7-e7d07e5c928b", "metadata": {}, - "outputs": [], "source": [ "isa_ds.absorb(json_ds)" - ] + ], + "outputs": [] }, { "cell_type": "code", @@ -520,527 +273,80 @@ "start_time": "2024-02-08T10:13:39.829474Z" } }, - "outputs": [ - { - "data": { - "text/plain": [ - "╭─────┬──────────────────┬──────────────┬──────────┬────────────────────╮\n", - "│ # │ Data file name │ Type │ Length │ Size (in memory) │\n", - "├─────┼──────────────────┼──────────────┼──────────┼────────────────────┤\n", - "│ 0 │ isa-bh2023-all │ IsaJsonModel │ N/A │ 2.3 MB │\n", - "╰─────┴──────────────────┴──────────────┴──────────┴────────────────────╯" - ] - }, - "execution_count": 41, - "metadata": {}, - "output_type": "execute_result" - } - ], "source": [ "isa_ds" - ] + ], + "outputs": [] }, { "cell_type": "code", "execution_count": 18, "id": "5509b72b-2f31-4e18-8603-ee8e0909b633", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "╭───┬────────────────────────╮\n", - "│ # │ JsonScalarM │\n", - "├───┼────────────────────────┤\n", - "│ 0 │ JsonScalarM( │\n", - "│ 1 │ '2021-08-15', │\n", - "│ 2 │ ) (JsonScalarM) len=10 │\n", - "╰───┴────────────────────────╯" - ] - }, - "execution_count": 18, - "metadata": {}, - "output_type": "execute_result" - } - ], "source": [ "json_ds[0]['studies'][0]['submissionDate']" - ] + ], + "outputs": [] }, { "cell_type": "code", "execution_count": 19, "id": "3f5275cb-0715-4e5d-83e8-42c619abbc98", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "╭───────┬───────────────────────────╮\n", - "│ # │ IsaInvestigationModel │\n", - "├───────┼───────────────────────────┤\n", - "│ 0 │ IsaInvestigationModel( │\n", - "│ 1 │ IsaInvestigationSchema( │\n", - "│ 2 │ field_id=None, │\n", - "│ 3 │ field_context=None, │\n", - "│ 4 │ field_type=None, │\n", - "│ 5 │ filename=None, │\n", - "│ 6 │ identifier='', │\n", - "│ 7 │ title='', │\n", - "├───────┼───────────────────────────┤\n", - "│ 25886 │ ), │\n", - "│ 25887 │ ], │\n", - "│ 25888 │ ), │\n", - "│ 25889 │ ), │\n", - "│ 25890 │ ], │\n", - "│ 25891 │ comments=[], │\n", - "│ 25892 │ ), │\n", - "│ 25893 │ ) (IsaInvestigationModel) │\n", - "╰───────┴───────────────────────────╯" - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - } - ], "source": [ "isa_ds[0].investigation" - ] + ], + "outputs": [] }, { "cell_type": "code", "execution_count": 20, "id": "daa5fed2-b90a-4bc0-942c-c086ee539a73", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[╭───────┬──────────────────────────────────────╮\n", - " │ # │ IsaStudyModel │\n", - " ├───────┼──────────────────────────────────────┤\n", - " │ 0 │ IsaStudyModel( │\n", - " │ 1 │ IsaStudySchema( │\n", - " │ 2 │ field_id=None, │\n", - " │ 3 │ field_context=None, │\n", - " │ 4 │ field_type=None, │\n", - " │ 5 │ filename='s_BH2023-study.txt', │\n", - " │ 6 │ identifier='BH2023', │\n", - " │ 7 │ title=( │\n", - " ├───────┼──────────────────────────────────────┤\n", - " │ 24031 │ field_type=None, │\n", - " │ 24032 │ name='Study Grant Number', │\n", - " │ 24033 │ value='', │\n", - " │ 24034 │ ), │\n", - " │ 24035 │ ), │\n", - " │ 24036 │ ], │\n", - " │ 24037 │ ), │\n", - " │ 24038 │ ) (IsaStudyModel) │\n", - " ╰───────┴──────────────────────────────────────╯]" - ] - }, - "execution_count": 20, - "metadata": {}, - "output_type": "execute_result" - } - ], "source": [ "isa_ds[0].investigation.studies" - ] + ], + "outputs": [] }, { "cell_type": "code", "execution_count": 21, "id": "7455712e-f991-4c57-b9d3-73657dc61879", "metadata": {}, - "outputs": [], "source": [ "from omnipy.modules.isa.flows import flatten_isa_json" - ] + ], + "outputs": [] }, { "cell_type": "code", "execution_count": 22, "id": "4c998644-e409-48aa-9f43-7e7af6c24e60", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[OMNIPY] Thu Feb 8 12:29:14 2024 - INFO: Initialized \"linear-flow-flatten-isa-json-thoughtful-pheasant\" [omnipy.log.registry.RunStateRegistry]\n", - "[OMNIPY] Thu Feb 8 12:29:14 2024 - INFO: Started running \"linear-flow-flatten-isa-json-thoughtful-pheasant\"... [omnipy.log.registry.RunStateRegistry]\n", - "[OMNIPY] Thu Feb 8 12:29:14 2024 - INFO: Initialized \"task-convert-dataset-satisfied-falcon\" [omnipy.log.registry.RunStateRegistry]\n", - "[OMNIPY] Thu Feb 8 12:29:14 2024 - INFO: Started running \"task-convert-dataset-satisfied-falcon\"... [omnipy.log.registry.RunStateRegistry]\n", - "[OMNIPY] Thu Feb 8 12:29:14 2024 - INFO: Finished running \"task-convert-dataset-satisfied-falcon\"! [omnipy.log.registry.RunStateRegistry]\n", - "[OMNIPY] Thu Feb 8 12:29:14 2024 - INFO: Writing dataset as a gzipped tarpack to \"/Users/sveinugu/PycharmProjects/omnipy/docs/ notebooks/outputs/2024_02_08-12_29_14/00_task_convert_dataset.tar.gz\" [omnipy.compute.task.TaskWithMixins]\n", - "[OMNIPY] Thu Feb 8 12:29:14 2024 - INFO: Initialized \"func-flow-transpose-dict-of-dicts-2-list-of-dicts-prehistoric-panther\" [omnipy.log.registry.RunStateRegistry]\n", - "[OMNIPY] Thu Feb 8 12:29:14 2024 - INFO: Started running \"func-flow-transpose-dict-of-dicts-2-list-of-dicts-prehistoric-panther\"... [omnipy.log.registry.RunStateRegistry]\n", - "[OMNIPY] Thu Feb 8 12:29:14 2024 - INFO: Initialized \"task-transpose-dicts-2-lists-overjoyed-boa\" [omnipy.log.registry.RunStateRegistry]\n", - "[OMNIPY] Thu Feb 8 12:29:14 2024 - INFO: Started running \"task-transpose-dicts-2-lists-overjoyed-boa\"... [omnipy.log.registry.RunStateRegistry]\n", - "[OMNIPY] Thu Feb 8 12:29:14 2024 - INFO: Finished running \"task-transpose-dicts-2-lists-overjoyed-boa\"! [omnipy.log.registry.RunStateRegistry]\n", - "[OMNIPY] Thu Feb 8 12:29:14 2024 - INFO: Writing dataset as a gzipped tarpack to \"/Users/sveinugu/PycharmProjects/omnipy/docs/ notebooks/outputs/2024_02_08-12_29_14/01_task_transpose_dicts_2_lists.tar.gz\" [omnipy.compute.task.TaskWithMixins]\n", - "[OMNIPY] Thu Feb 8 12:29:14 2024 - INFO: Finished running \"func-flow-transpose-dict-of-dicts-2-list-of-dicts-prehistoric-panther\"! [omnipy.log.registry.RunStateRegistry]\n", - "[OMNIPY] Thu Feb 8 12:29:14 2024 - INFO: Writing dataset as a gzipped tarpack to \"/Users/sveinugu/PycharmProjects/omnipy/docs/ notebooks/outputs/2024_02_08-12_29_14/02_func_flow_transpose_dict_of_dicts_2_list_of_dicts.tar.gz\" [omnipy.compute.flow.FuncFlowWithMixins]\n", - "[OMNIPY] Thu Feb 8 12:29:14 2024 - INFO: Initialized \"func-flow-flatten-nested-json-fresh-dodo\" [omnipy.log.registry.RunStateRegistry]\n", - "[OMNIPY] Thu Feb 8 12:29:14 2024 - INFO: Started running \"func-flow-flatten-nested-json-fresh-dodo\"... [omnipy.log.registry.RunStateRegistry]\n", - "[OMNIPY] Thu Feb 8 12:29:14 2024 - INFO: Initialized \"task-flatten-outer-level-of-all-data-files-carmine-lynx\" [omnipy.log.registry.RunStateRegistry]\n", - "[OMNIPY] Thu Feb 8 12:29:14 2024 - INFO: Started running \"task-flatten-outer-level-of-all-data-files-carmine-lynx\"... [omnipy.log.registry.RunStateRegistry]\n", - "[OMNIPY] Thu Feb 8 12:29:14 2024 - INFO: Finished running \"task-flatten-outer-level-of-all-data-files-carmine-lynx\"! [omnipy.log.registry.RunStateRegistry]\n", - "[OMNIPY] Thu Feb 8 12:29:14 2024 - INFO: Results of task-flatten-outer-level-of-all-data-files-carmine-lynx is not a Dataset and cannot be automatically serialized and persisted! [omnipy.compute.task.TaskWithMixins]\n", - "[OMNIPY] Thu Feb 8 12:29:14 2024 - INFO: Initialized \"task-flatten-outer-level-of-all-data-files-gray-sponge\" [omnipy.log.registry.RunStateRegistry]\n", - "[OMNIPY] Thu Feb 8 12:29:14 2024 - INFO: Started running \"task-flatten-outer-level-of-all-data-files-gray-sponge\"... [omnipy.log.registry.RunStateRegistry]\n", - "[OMNIPY] Thu Feb 8 12:29:14 2024 - INFO: Finished running \"task-flatten-outer-level-of-all-data-files-gray-sponge\"! [omnipy.log.registry.RunStateRegistry]\n", - "[OMNIPY] Thu Feb 8 12:29:14 2024 - INFO: Results of task-flatten-outer-level-of-all-data-files-gray-sponge is not a Dataset and cannot be automatically serialized and persisted! [omnipy.compute.task.TaskWithMixins]\n", - "[OMNIPY] Thu Feb 8 12:29:14 2024 - INFO: Initialized \"task-flatten-outer-level-of-all-data-files-elated-echidna\" [omnipy.log.registry.RunStateRegistry]\n", - "[OMNIPY] Thu Feb 8 12:29:14 2024 - INFO: Started running \"task-flatten-outer-level-of-all-data-files-elated-echidna\"... [omnipy.log.registry.RunStateRegistry]\n", - "[OMNIPY] Thu Feb 8 12:29:14 2024 - INFO: Finished running \"task-flatten-outer-level-of-all-data-files-elated-echidna\"! [omnipy.log.registry.RunStateRegistry]\n", - "[OMNIPY] Thu Feb 8 12:29:14 2024 - INFO: Results of task-flatten-outer-level-of-all-data-files-elated-echidna is not a Dataset and cannot be automatically serialized and persisted! [omnipy.compute.task.TaskWithMixins]\n", - "[OMNIPY] Thu Feb 8 12:29:14 2024 - INFO: Initialized \"task-flatten-outer-level-of-all-data-files-russet-mink\" [omnipy.log.registry.RunStateRegistry]\n", - "[OMNIPY] Thu Feb 8 12:29:14 2024 - INFO: Started running \"task-flatten-outer-level-of-all-data-files-russet-mink\"... [omnipy.log.registry.RunStateRegistry]\n", - "[OMNIPY] Thu Feb 8 12:29:14 2024 - INFO: Finished running \"task-flatten-outer-level-of-all-data-files-russet-mink\"! [omnipy.log.registry.RunStateRegistry]\n", - "[OMNIPY] Thu Feb 8 12:29:14 2024 - INFO: Results of task-flatten-outer-level-of-all-data-files-russet-mink is not a Dataset and cannot be automatically serialized and persisted! [omnipy.compute.task.TaskWithMixins]\n", - "[OMNIPY] Thu Feb 8 12:29:14 2024 - INFO: Initialized \"task-flatten-outer-level-of-all-data-files-sticky-giraffe\" [omnipy.log.registry.RunStateRegistry]\n", - "[OMNIPY] Thu Feb 8 12:29:14 2024 - INFO: Started running \"task-flatten-outer-level-of-all-data-files-sticky-giraffe\"... [omnipy.log.registry.RunStateRegistry]\n", - "[OMNIPY] Thu Feb 8 12:29:14 2024 - INFO: Finished running \"task-flatten-outer-level-of-all-data-files-sticky-giraffe\"! [omnipy.log.registry.RunStateRegistry]\n", - "[OMNIPY] Thu Feb 8 12:29:14 2024 - INFO: Results of task-flatten-outer-level-of-all-data-files-sticky-giraffe is not a Dataset and cannot be automatically serialized and persisted! [omnipy.compute.task.TaskWithMixins]\n", - "[OMNIPY] Thu Feb 8 12:29:14 2024 - INFO: Initialized \"task-flatten-outer-level-of-all-data-files-beautiful-herring\" [omnipy.log.registry.RunStateRegistry]\n", - "[OMNIPY] Thu Feb 8 12:29:14 2024 - INFO: Started running \"task-flatten-outer-level-of-all-data-files-beautiful-herring\"... [omnipy.log.registry.RunStateRegistry]\n", - "[OMNIPY] Thu Feb 8 12:29:14 2024 - INFO: Finished running \"task-flatten-outer-level-of-all-data-files-beautiful-herring\"! [omnipy.log.registry.RunStateRegistry]\n", - "[OMNIPY] Thu Feb 8 12:29:14 2024 - INFO: Results of task-flatten-outer-level-of-all-data-files-beautiful-herring is not a Dataset and cannot be automatically serialized and persisted! [omnipy.compute.task.TaskWithMixins]\n", - "[OMNIPY] Thu Feb 8 12:29:14 2024 - INFO: Initialized \"task-flatten-outer-level-of-all-data-files-modest-oyster\" [omnipy.log.registry.RunStateRegistry]\n", - "[OMNIPY] Thu Feb 8 12:29:14 2024 - INFO: Started running \"task-flatten-outer-level-of-all-data-files-modest-oyster\"... [omnipy.log.registry.RunStateRegistry]\n", - "[OMNIPY] Thu Feb 8 12:29:14 2024 - INFO: Finished running \"task-flatten-outer-level-of-all-data-files-modest-oyster\"! [omnipy.log.registry.RunStateRegistry]\n", - "[OMNIPY] Thu Feb 8 12:29:14 2024 - INFO: Results of task-flatten-outer-level-of-all-data-files-modest-oyster is not a Dataset and cannot be automatically serialized and persisted! [omnipy.compute.task.TaskWithMixins]\n", - "[OMNIPY] Thu Feb 8 12:29:14 2024 - INFO: Finished running \"func-flow-flatten-nested-json-fresh-dodo\"! [omnipy.log.registry.RunStateRegistry]\n", - "[OMNIPY] Thu Feb 8 12:29:14 2024 - INFO: Writing dataset as a gzipped tarpack to \"/Users/sveinugu/PycharmProjects/omnipy/docs/ notebooks/outputs/2024_02_08-12_29_14/03_func_flow_flatten_nested_json.tar.gz\" [omnipy.compute.flow.FuncFlowWithMixins]\n", - "[OMNIPY] Thu Feb 8 12:29:14 2024 - INFO: Initialized \"task-convert-dataset-mysterious-rottweiler\" [omnipy.log.registry.RunStateRegistry]\n", - "[OMNIPY] Thu Feb 8 12:29:14 2024 - INFO: Started running \"task-convert-dataset-mysterious-rottweiler\"... [omnipy.log.registry.RunStateRegistry]\n", - "[OMNIPY] Thu Feb 8 12:29:14 2024 - INFO: Finished running \"task-convert-dataset-mysterious-rottweiler\"! [omnipy.log.registry.RunStateRegistry]\n", - "[OMNIPY] Thu Feb 8 12:29:14 2024 - INFO: Writing dataset as a gzipped tarpack to \"/Users/sveinugu/PycharmProjects/omnipy/docs/ notebooks/outputs/2024_02_08-12_29_14/04_task_convert_dataset.tar.gz\" [omnipy.compute.task.TaskWithMixins]\n", - "[OMNIPY] Thu Feb 8 12:29:14 2024 - INFO: Finished running \"linear-flow-flatten-isa-json-thoughtful-pheasant\"! [omnipy.log.registry.RunStateRegistry]\n", - "[OMNIPY] Thu Feb 8 12:29:14 2024 - INFO: Writing dataset as a gzipped tarpack to \"/Users/sveinugu/PycharmProjects/omnipy/docs/ notebooks/outputs/2024_02_08-12_29_14/05_linear_flow_flatten_isa_json.tar.gz\" [omnipy.compute.flow.LinearFlowWithMixins]\n" - ] - } - ], "source": [ "flattened_isa_ds = flatten_isa_json.run(isa_ds)" - ] + ], + "outputs": [] }, { "cell_type": "code", "execution_count": 23, "id": "623bd51a-0638-4920-b04a-7582489df9c2", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "╭─────┬───────────────────────────────────────────────────────────────────────────────────┬───────────────────────┬──────────┬────────────────────╮\n", - "│ # │ Data file name │ Type │ Length │ Size (in memory) │\n", - "├─────┼───────────────────────────────────────────────────────────────────────────────────┼───────────────────────┼──────────┼────────────────────┤\n", - "│ 0 │ investigation │ FlattenedIsaJsonModel │ 1 │ 7.6 kB │\n", - "│ 1 │ investigation.ontologySourceReferences │ FlattenedIsaJsonModel │ 5 │ 29.5 kB │\n", - "│ 2 │ investigation.publications │ FlattenedIsaJsonModel │ 0 │ 1.6 kB │\n", - "│ 3 │ investigation.people │ FlattenedIsaJsonModel │ 0 │ 1.6 kB │\n", - "│ 4 │ investigation.studies │ FlattenedIsaJsonModel │ 1 │ 9.0 kB │\n", - "│ 5 │ investigation.comments │ FlattenedIsaJsonModel │ 0 │ 1.6 kB │\n", - "│ 6 │ investigation.ontologySourceReferences.comments │ FlattenedIsaJsonModel │ 0 │ 1.6 kB │\n", - "│ 7 │ investigation.studies.publications │ FlattenedIsaJsonModel │ 1 │ 7.4 kB │\n", - "│ 8 │ investigation.studies.people │ FlattenedIsaJsonModel │ 2 │ 17.8 kB │\n", - "│ 9 │ investigation.studies.studyDesignDescriptors │ FlattenedIsaJsonModel │ 2 │ 12.2 kB │\n", - "│ 10 │ investigation.studies.protocols │ FlattenedIsaJsonModel │ 16 │ 92.9 kB │\n", - "│ 11 │ investigation.studies.materials │ FlattenedIsaJsonModel │ 1 │ 3.5 kB │\n", - "│ 12 │ investigation.studies.processSequence │ FlattenedIsaJsonModel │ 2 │ 13.8 kB │\n", - "│ 13 │ investigation.studies.assays │ FlattenedIsaJsonModel │ 5 │ 24.2 kB │\n", - "│ 14 │ investigation.studies.factors │ FlattenedIsaJsonModel │ 3 │ 13.9 kB │\n", - "│ 15 │ investigation.studies.characteristicCategories │ FlattenedIsaJsonModel │ 3 │ 12.0 kB │\n", - "│ 16 │ investigation.studies.unitCategories │ FlattenedIsaJsonModel │ 0 │ 1.6 kB │\n", - "│ 17 │ investigation.studies.comments │ FlattenedIsaJsonModel │ 7 │ 33.4 kB │\n", - "│ 18 │ investigation.studies.publications.status │ FlattenedIsaJsonModel │ 1 │ 6.9 kB │\n", - "│ 19 │ investigation.studies.publications.comments │ FlattenedIsaJsonModel │ 0 │ 1.6 kB │\n", - "│ 20 │ investigation.studies.people.roles │ FlattenedIsaJsonModel │ 4 │ 22.2 kB │\n", - "│ 21 │ investigation.studies.people.comments │ FlattenedIsaJsonModel │ 2 │ 10.8 kB │\n", - "│ 22 │ investigation.studies.studyDesignDescriptors.comments │ FlattenedIsaJsonModel │ 0 │ 1.6 kB │\n", - "│ 23 │ investigation.studies.protocols.protocolType │ FlattenedIsaJsonModel │ 16 │ 84.6 kB │\n", - "│ 24 │ investigation.studies.protocols.parameters │ FlattenedIsaJsonModel │ 28 │ 116.0 kB │\n", - "│ 25 │ investigation.studies.protocols.components │ FlattenedIsaJsonModel │ 0 │ 1.6 kB │\n", - "│ 26 │ investigation.studies.protocols.comments │ FlattenedIsaJsonModel │ 0 │ 1.6 kB │\n", - "│ 27 │ investigation.studies.materials.sources │ FlattenedIsaJsonModel │ 2 │ 9.9 kB │\n", - "│ 28 │ investigation.studies.materials.samples │ FlattenedIsaJsonModel │ 8 │ 34.4 kB │\n", - "│ 29 │ investigation.studies.materials.otherMaterials │ FlattenedIsaJsonModel │ 0 │ 1.6 kB │\n", - "│ 30 │ investigation.studies.processSequence.executesProtocol │ FlattenedIsaJsonModel │ 2 │ 17.4 kB │\n", - "│ 31 │ investigation.studies.processSequence.parameterValues │ FlattenedIsaJsonModel │ 2 │ 11.9 kB │\n", - "│ 32 │ investigation.studies.processSequence.inputs │ FlattenedIsaJsonModel │ 2 │ 11.9 kB │\n", - "│ 33 │ investigation.studies.processSequence.outputs │ FlattenedIsaJsonModel │ 8 │ 49.9 kB │\n", - "│ 34 │ investigation.studies.processSequence.comments │ FlattenedIsaJsonModel │ 0 │ 1.6 kB │\n", - "│ 35 │ investigation.studies.assays.measurementType │ FlattenedIsaJsonModel │ 5 │ 27.9 kB │\n", - "│ 36 │ investigation.studies.assays.technologyType │ FlattenedIsaJsonModel │ 5 │ 27.8 kB │\n", - "│ 37 │ investigation.studies.assays.dataFiles │ FlattenedIsaJsonModel │ 69 │ 319.3 kB │\n", - "│ 38 │ investigation.studies.assays.materials │ FlattenedIsaJsonModel │ 5 │ 11.1 kB │\n", - "│ 39 │ investigation.studies.assays.characteristicCategories │ FlattenedIsaJsonModel │ 3 │ 12.2 kB │\n", - "│ 40 │ investigation.studies.assays.unitCategories │ FlattenedIsaJsonModel │ 0 │ 1.6 kB │\n", - "│ 41 │ investigation.studies.assays.processSequence │ FlattenedIsaJsonModel │ 125 │ 657.2 kB │\n", - "│ 42 │ investigation.studies.assays.comments │ FlattenedIsaJsonModel │ 5 │ 24.8 kB │\n", - "│ 43 │ investigation.studies.factors.factorType │ FlattenedIsaJsonModel │ 3 │ 17.6 kB │\n", - "│ 44 │ investigation.studies.factors.comments │ FlattenedIsaJsonModel │ 0 │ 1.6 kB │\n", - "│ 45 │ investigation.studies.characteristicCategories.characteristicType │ FlattenedIsaJsonModel │ 3 │ 17.3 kB │\n", - "│ 46 │ investigation.studies.publications.status.comments │ FlattenedIsaJsonModel │ 0 │ 1.6 kB │\n", - "│ 47 │ investigation.studies.people.roles.comments │ FlattenedIsaJsonModel │ 0 │ 1.6 kB │\n", - "│ 48 │ investigation.studies.protocols.protocolType.comments │ FlattenedIsaJsonModel │ 0 │ 1.6 kB │\n", - "│ 49 │ investigation.studies.protocols.parameters.parameterName │ FlattenedIsaJsonModel │ 28 │ 147.2 kB │\n", - "│ 50 │ investigation.studies.materials.sources.characteristics │ FlattenedIsaJsonModel │ 6 │ 27.7 kB │\n", - "│ 51 │ investigation.studies.materials.sources.comments │ FlattenedIsaJsonModel │ 0 │ 1.6 kB │\n", - "│ 52 │ investigation.studies.materials.samples.characteristics │ FlattenedIsaJsonModel │ 8 │ 38.3 kB │\n", - "│ 53 │ investigation.studies.materials.samples.factorValues │ FlattenedIsaJsonModel │ 24 │ 122.1 kB │\n", - "│ 54 │ investigation.studies.materials.samples.derivesFrom │ FlattenedIsaJsonModel │ 8 │ 42.5 kB │\n", - "│ 55 │ investigation.studies.materials.samples.comments │ FlattenedIsaJsonModel │ 0 │ 1.6 kB │\n", - "│ 56 │ investigation.studies.processSequence.parameterValues.category │ FlattenedIsaJsonModel │ 2 │ 11.0 kB │\n", - "│ 57 │ investigation.studies.assays.measurementType.comments │ FlattenedIsaJsonModel │ 0 │ 1.6 kB │\n", - "│ 58 │ investigation.studies.assays.technologyType.comments │ FlattenedIsaJsonModel │ 0 │ 1.6 kB │\n", - "│ 59 │ investigation.studies.assays.dataFiles.comments │ FlattenedIsaJsonModel │ 75 │ 345.4 kB │\n", - "│ 60 │ investigation.studies.assays.materials.samples │ FlattenedIsaJsonModel │ 40 │ 242.6 kB │\n", - "│ 61 │ investigation.studies.assays.materials.otherMaterials │ FlattenedIsaJsonModel │ 40 │ 206.0 kB │\n", - "│ 62 │ investigation.studies.assays.characteristicCategories.characteristicType │ FlattenedIsaJsonModel │ 3 │ 17.4 kB │\n", - "│ 63 │ investigation.studies.assays.processSequence.executesProtocol │ FlattenedIsaJsonModel │ 125 │ 988.7 kB │\n", - "│ 64 │ investigation.studies.assays.processSequence.parameterValues │ FlattenedIsaJsonModel │ 229 │ 1.2 MB │\n", - "│ 65 │ investigation.studies.assays.processSequence.nextProcess │ FlattenedIsaJsonModel │ 120 │ 1.1 MB │\n", - "│ 66 │ investigation.studies.assays.processSequence.inputs │ FlattenedIsaJsonModel │ 120 │ 615.4 kB │\n", - "│ 67 │ investigation.studies.assays.processSequence.outputs │ FlattenedIsaJsonModel │ 68 │ 417.1 kB │\n", - "│ 68 │ investigation.studies.assays.processSequence.comments │ FlattenedIsaJsonModel │ 0 │ 1.6 kB │\n", - "│ 69 │ investigation.studies.assays.processSequence.previousProcess │ FlattenedIsaJsonModel │ 85 │ 757.0 kB │\n", - "│ 70 │ investigation.studies.factors.factorType.comments │ FlattenedIsaJsonModel │ 0 │ 1.6 kB │\n", - "│ 71 │ investigation.studies.characteristicCategories.characteristicType.comments │ FlattenedIsaJsonModel │ 0 │ 1.6 kB │\n", - "│ 72 │ investigation.studies.protocols.parameters.parameterName.comments │ FlattenedIsaJsonModel │ 0 │ 1.6 kB │\n", - "│ 73 │ investigation.studies.materials.sources.characteristics.category │ FlattenedIsaJsonModel │ 6 │ 26.6 kB │\n", - "│ 74 │ investigation.studies.materials.sources.characteristics.comments │ FlattenedIsaJsonModel │ 0 │ 1.6 kB │\n", - "│ 75 │ investigation.studies.materials.sources.characteristics.value │ FlattenedIsaJsonModel │ 2 │ 12.4 kB │\n", - "│ 76 │ investigation.studies.materials.samples.characteristics.category │ FlattenedIsaJsonModel │ 8 │ 35.0 kB │\n", - "│ 77 │ investigation.studies.materials.samples.characteristics.comments │ FlattenedIsaJsonModel │ 0 │ 1.6 kB │\n", - "│ 78 │ investigation.studies.materials.samples.factorValues.category │ FlattenedIsaJsonModel │ 24 │ 125.1 kB │\n", - "│ 79 │ investigation.studies.assays.materials.otherMaterials.characteristics │ FlattenedIsaJsonModel │ 8 │ 38.5 kB │\n", - "│ 80 │ investigation.studies.assays.materials.otherMaterials.comments │ FlattenedIsaJsonModel │ 0 │ 1.6 kB │\n", - "│ 81 │ investigation.studies.assays.characteristicCategories.characteristicType.comments │ FlattenedIsaJsonModel │ 0 │ 1.6 kB │\n", - "│ 82 │ investigation.studies.assays.processSequence.parameterValues.category │ FlattenedIsaJsonModel │ 229 │ 1.1 MB │\n", - "│ 83 │ investigation.studies.materials.sources.characteristics.value.comments │ FlattenedIsaJsonModel │ 0 │ 1.6 kB │\n", - "│ 84 │ investigation.studies.assays.materials.otherMaterials.characteristics.category │ FlattenedIsaJsonModel │ 8 │ 35.2 kB │\n", - "│ 85 │ investigation.studies.assays.materials.otherMaterials.characteristics.comments │ FlattenedIsaJsonModel │ 0 │ 1.6 kB │\n", - "╰─────┴───────────────────────────────────────────────────────────────────────────────────┴───────────────────────┴──────────┴────────────────────╯" - ] - }, - "execution_count": 23, - "metadata": {}, - "output_type": "execute_result" - } - ], "source": [ "flattened_isa_ds" - ] + ], + "outputs": [] }, { "cell_type": "code", "execution_count": 25, "id": "6db0a1e2-64dd-4d8b-a83e-551dae7fdf76", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
_omnipy_id_omnipy_ref@id@context@typenametype
0investigation.studies.assays.dataFiles.0investigation.studies.assays.0#data_file/b3530ec1-6bc5-4757-a526-81c1773530a6NoneNoneisotopologue-distribution-analysis.txtDerived Data File
1investigation.studies.assays.dataFiles.1investigation.studies.assays.0#data_file/2647db5f-0ed8-4536-aec7-8d5d4cfc65d3NoneNonems-data-0.mzmlRaw Spectral Data File
2investigation.studies.assays.dataFiles.2investigation.studies.assays.0#data_file/d3cf3b1f-f375-4763-8837-89d9f453116cNoneNonems-data-1.mzmlRaw Spectral Data File
3investigation.studies.assays.dataFiles.3investigation.studies.assays.0#data_file/ca61e6d1-0cab-45be-b333-e45583e06311NoneNonems-data-2.mzmlRaw Spectral Data File
4investigation.studies.assays.dataFiles.4investigation.studies.assays.0#data_file/0eb3dcc9-b79c-4b44-ac31-b977ea63ae2cNoneNonems-data-3.mzmlRaw Spectral Data File
........................
64investigation.studies.assays.dataFiles.64investigation.studies.assays.4#data_file/4e56fbc3-98a9-4a0a-80a4-8a2219674506NoneNonerna-seq-data-3.fastqRaw Data File
65investigation.studies.assays.dataFiles.65investigation.studies.assays.4#data_file/29b9c0d5-aea5-42cf-b859-77d5d99ad35dNoneNonerna-seq-data-4.fastqRaw Data File
66investigation.studies.assays.dataFiles.66investigation.studies.assays.4#data_file/a7f9abdd-364e-4c8b-b262-5278cae39387NoneNonerna-seq-data-5.fastqRaw Data File
67investigation.studies.assays.dataFiles.67investigation.studies.assays.4#data_file/bbe9f980-6627-4b76-a293-5b7fad0f645eNoneNonerna-seq-data-6.fastqRaw Data File
68investigation.studies.assays.dataFiles.68investigation.studies.assays.4#data_file/e0795889-209d-4f8a-8176-39d4ef8dd91eNoneNonerna-seq-data-7.fastqRaw Data File
\n", - "

69 rows × 7 columns

\n", - "
" - ], - "text/plain": [ - " _omnipy_id _omnipy_ref \\\n", - "0 investigation.studies.assays.dataFiles.0 investigation.studies.assays.0 \n", - "1 investigation.studies.assays.dataFiles.1 investigation.studies.assays.0 \n", - "2 investigation.studies.assays.dataFiles.2 investigation.studies.assays.0 \n", - "3 investigation.studies.assays.dataFiles.3 investigation.studies.assays.0 \n", - "4 investigation.studies.assays.dataFiles.4 investigation.studies.assays.0 \n", - ".. ... ... \n", - "64 investigation.studies.assays.dataFiles.64 investigation.studies.assays.4 \n", - "65 investigation.studies.assays.dataFiles.65 investigation.studies.assays.4 \n", - "66 investigation.studies.assays.dataFiles.66 investigation.studies.assays.4 \n", - "67 investigation.studies.assays.dataFiles.67 investigation.studies.assays.4 \n", - "68 investigation.studies.assays.dataFiles.68 investigation.studies.assays.4 \n", - "\n", - " @id @context @type \\\n", - "0 #data_file/b3530ec1-6bc5-4757-a526-81c1773530a6 None None \n", - "1 #data_file/2647db5f-0ed8-4536-aec7-8d5d4cfc65d3 None None \n", - "2 #data_file/d3cf3b1f-f375-4763-8837-89d9f453116c None None \n", - "3 #data_file/ca61e6d1-0cab-45be-b333-e45583e06311 None None \n", - "4 #data_file/0eb3dcc9-b79c-4b44-ac31-b977ea63ae2c None None \n", - ".. ... ... ... \n", - "64 #data_file/4e56fbc3-98a9-4a0a-80a4-8a2219674506 None None \n", - "65 #data_file/29b9c0d5-aea5-42cf-b859-77d5d99ad35d None None \n", - "66 #data_file/a7f9abdd-364e-4c8b-b262-5278cae39387 None None \n", - "67 #data_file/bbe9f980-6627-4b76-a293-5b7fad0f645e None None \n", - "68 #data_file/e0795889-209d-4f8a-8176-39d4ef8dd91e None None \n", - "\n", - " name type \n", - "0 isotopologue-distribution-analysis.txt Derived Data File \n", - "1 ms-data-0.mzml Raw Spectral Data File \n", - "2 ms-data-1.mzml Raw Spectral Data File \n", - "3 ms-data-2.mzml Raw Spectral Data File \n", - "4 ms-data-3.mzml Raw Spectral Data File \n", - ".. ... ... \n", - "64 rna-seq-data-3.fastq Raw Data File \n", - "65 rna-seq-data-4.fastq Raw Data File \n", - "66 rna-seq-data-5.fastq Raw Data File \n", - "67 rna-seq-data-6.fastq Raw Data File \n", - "68 rna-seq-data-7.fastq Raw Data File \n", - "\n", - "[69 rows x 7 columns]" - ] - }, - "execution_count": 25, - "metadata": {}, - "output_type": "execute_result" - } - ], "source": [ "flattened_isa_ds['investigation.studies.assays.dataFiles'].view()" - ] + ], + "outputs": [] } ], "metadata": { diff --git a/pyproject.toml b/pyproject.toml index edd8e9f9..838289f2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,6 +42,7 @@ objsize = "^0.7.0" humanize = "^4.9.0" httpx = "^0.26.0" pydantic = {version = "<2", extras = ["email"]} +bidict = "^0.23.1" [tool.poetry.group.dev.dependencies] deepdiff = "^6.2.1" diff --git a/src/omnipy/api/protocols/private/data.py b/src/omnipy/api/protocols/private/data.py new file mode 100644 index 00000000..83b802dd --- /dev/null +++ b/src/omnipy/api/protocols/private/data.py @@ -0,0 +1,24 @@ +from typing import Protocol + +from omnipy.api.protocols.public.config import IsDataConfig + + +class IsDataConfigHolder(Protocol): + """""" + @property + def config(self) -> IsDataConfig: + ... + + def set_config(self, config: IsDataConfig) -> None: + ... + + +class IsDataClassBase(Protocol): + """""" + @property + def _data_class_creator(self) -> IsDataConfigHolder: + ... + + @property + def config(self) -> IsDataConfig: + ... diff --git a/src/omnipy/api/protocols/private/util.py b/src/omnipy/api/protocols/private/util.py index 550edbb2..e0181b6b 100644 --- a/src/omnipy/api/protocols/private/util.py +++ b/src/omnipy/api/protocols/private/util.py @@ -1,7 +1,9 @@ -from typing import Callable, Protocol, runtime_checkable +from typing import Callable, Protocol, runtime_checkable, TypeVar from omnipy.api.typedefs import DecoratorClassT +_ContentT = TypeVar("_ContentT", covariant=True, bound=object) + @runtime_checkable class IsCallableParamAfterSelf(Protocol): @@ -15,3 +17,9 @@ class IsCallableClass(Protocol[DecoratorClassT]): """""" def __call__(self, *args: object, **kwargs: object) -> Callable[[Callable], DecoratorClassT]: ... + + +class HasContents(Protocol[_ContentT]): + @property + def contents(self) -> _ContentT: + ... diff --git a/src/omnipy/api/protocols/public/config.py b/src/omnipy/api/protocols/public/config.py index 01eaf494..4fb17c88 100644 --- a/src/omnipy/api/protocols/public/config.py +++ b/src/omnipy/api/protocols/public/config.py @@ -29,6 +29,7 @@ class IsJobConfig(Protocol): class IsDataConfig(Protocol): """""" interactive_mode: bool + dynamically_convert_elements_to_models: bool terminal_size_columns: int terminal_size_lines: int diff --git a/src/omnipy/api/protocols/public/hub.py b/src/omnipy/api/protocols/public/hub.py index 86f2b684..4a6c711f 100644 --- a/src/omnipy/api/protocols/public/hub.py +++ b/src/omnipy/api/protocols/public/hub.py @@ -4,6 +4,7 @@ from omnipy.api.enums import EngineChoice from omnipy.api.protocols.private.compute.job_creator import IsJobConfigHolder +from omnipy.api.protocols.private.data import IsDataConfigHolder from omnipy.api.protocols.private.engine import IsEngine from omnipy.api.protocols.private.log import IsRunStateRegistry from omnipy.api.protocols.public.config import (IsDataConfig, @@ -41,16 +42,20 @@ def __init__( engine: EngineChoice = EngineChoice.LOCAL, # noqa local: IsLocalRunnerConfig | None = None, # noqa prefect: IsPrefectEngineConfig | None = None, # noqa - root_log: 'IsRootLogConfigEntryPublisher | None' = None, # noqa + root_log: IsRootLogConfig | None = None, # noqa *args: object, **kwargs: object) -> None: ... + def reset_to_defaults(self): + ... + class IsRuntimeObjects(Protocol): """""" job_creator: IsJobConfigHolder + data_class_creator: IsDataConfigHolder local: IsEngine prefect: IsEngine registry: IsRunStateRegistry @@ -61,6 +66,7 @@ class IsRuntimeObjects(Protocol): def __init__( self, job_creator: IsJobConfigHolder | None = None, # noqa + data_class_creator: IsDataConfigHolder | None = None, # noqa local: IsEngine | None = None, # noqa prefect: IsEngine | None = None, # noqa registry: IsRunStateRegistry | None = None, # noqa diff --git a/src/omnipy/config/data.py b/src/omnipy/config/data.py index 8ba61bdf..8bcb922e 100644 --- a/src/omnipy/config/data.py +++ b/src/omnipy/config/data.py @@ -7,5 +7,6 @@ @dataclass class DataConfig: interactive_mode: bool = True + dynamically_convert_elements_to_models: bool = False terminal_size_columns: int = _terminal_size.columns terminal_size_lines: int = _terminal_size.lines diff --git a/src/omnipy/config/root_log.py b/src/omnipy/config/root_log.py index bd466856..89c6c68a 100644 --- a/src/omnipy/config/root_log.py +++ b/src/omnipy/config/root_log.py @@ -7,7 +7,7 @@ def _get_log_dir_path() -> str: - return str(Path.cwd().joinpath(Path('logs'))) + return str(Path.cwd() / 'logs') @dataclass diff --git a/src/omnipy/data/data_class_creator.py b/src/omnipy/data/data_class_creator.py new file mode 100644 index 00000000..9602ca4a --- /dev/null +++ b/src/omnipy/data/data_class_creator.py @@ -0,0 +1,36 @@ +from abc import ABCMeta + +from omnipy.api.protocols.private.data import IsDataConfigHolder +from omnipy.api.protocols.public.config import IsDataConfig +from omnipy.config.data import DataConfig + + +class DataClassCreator: + def __init__(self) -> None: + self._config: IsDataConfig = DataConfig() + + def set_config(self, config: IsDataConfig) -> None: + self._config = config + + @property + def config(self) -> IsDataConfig: + return self._config + + +class DataClassBaseMeta(ABCMeta): + """""" + _data_class_creator_obj = DataClassCreator() + + @property + def data_class_creator(self) -> IsDataConfigHolder: + return self._data_class_creator_obj + + +class DataClassBase(metaclass=DataClassBaseMeta): + @property + def _data_class_creator(self) -> IsDataConfigHolder: + return self.__class__.data_class_creator + + @property + def config(self) -> IsDataConfig: + return self.__class__.data_class_creator.config diff --git a/src/omnipy/data/dataset.py b/src/omnipy/data/dataset.py index 666a4e35..5cf8fcfa 100644 --- a/src/omnipy/data/dataset.py +++ b/src/omnipy/data/dataset.py @@ -24,10 +24,11 @@ from pydantic import Field, PrivateAttr, root_validator, ValidationError from pydantic.fields import ModelField, Undefined, UndefinedType from pydantic.generics import GenericModel +from pydantic.main import ModelMetaclass from pydantic.utils import lenient_isinstance, lenient_issubclass +from omnipy.data.data_class_creator import DataClassBase, DataClassBaseMeta from omnipy.data.model import (_cleanup_name_qualname_and_module, - _is_interactive_mode, _waiting_for_terminal_repr, DataWithParams, INTERACTIVE_MODULES, @@ -56,7 +57,11 @@ # BaseModel.copy() -class Dataset(GenericModel, Generic[ModelT], UserDict): +class _DatasetMetaclass(ModelMetaclass, DataClassBaseMeta): + ... + + +class Dataset(GenericModel, Generic[ModelT], UserDict, DataClassBase, metaclass=_DatasetMetaclass): """ Dict-based container of data files that follow a specific Model @@ -538,7 +543,7 @@ def __repr_args__(self): return [(k, v.contents) for k, v in self.data.items()] def __repr__(self): - if _is_interactive_mode() and not _waiting_for_terminal_repr(): + if self.config.interactive_mode and not _waiting_for_terminal_repr(): if get_calling_module_name() in INTERACTIVE_MODULES: _waiting_for_terminal_repr(True) return self._table_repr() diff --git a/src/omnipy/data/model.py b/src/omnipy/data/model.py index 053aaf70..48413440 100644 --- a/src/omnipy/data/model.py +++ b/src/omnipy/data/model.py @@ -36,6 +36,7 @@ from omnipy.api.exceptions import ParamException from omnipy.api.typedefs import TypeForm +from omnipy.data.data_class_creator import DataClassBase, DataClassBaseMeta from omnipy.data.methodinfo import MethodInfo, SPECIAL_METHODS_INFO from omnipy.util.contexts import AttribHolder, LastErrorHolder, nothing from omnipy.util.decorators import add_callback_after_call, no_context @@ -54,7 +55,8 @@ is_union, remove_annotated_plus_optional_if_present, remove_forward_ref_notation, - RestorableContents) + RestorableContents, + Snapshot) from omnipy.util.tabulate import tabulate _KeyT = TypeVar('_KeyT', bound=Hashable) @@ -101,11 +103,6 @@ def _cleanup_name_qualname_and_module(cls, created_model_or_dataset, model, orig created_model_or_dataset.__module__ = cls.__module__ -def _is_interactive_mode() -> bool: - from omnipy.hub.runtime import runtime - return runtime.config.data.interactive_mode if runtime else True - - def _get_terminal_size() -> os.terminal_size: from omnipy.hub.runtime import runtime @@ -130,7 +127,7 @@ def _waiting_for_terminal_repr(new_value: bool | None = None) -> bool: def is_model_instance(obj: object) -> bool: return lenient_isinstance(obj, Model) \ - and not is_none_type(obj) # Consequence of MyModelMetaclass hack + and not is_none_type(obj) # Consequence of _ModelMetaclass hack # def orjson_dumps(v, *, default): @@ -140,7 +137,7 @@ def is_model_instance(obj: object) -> bool: _restorable_content_cache: dict[int, RestorableContents] = {} -class MyModelMetaclass(ModelMetaclass): +class _ModelMetaclass(ModelMetaclass, DataClassBaseMeta): # Hack to overcome bug in pydantic/fields.py (v1.10.13), lines 636-641: # # if origin is None or origin is CollectionsHashable: @@ -154,14 +151,14 @@ class MyModelMetaclass(ModelMetaclass): # subfields, e.g. in `list[MyModel]` as `get_origin(MyModel) is None`. Here, we want allow_none # to be set to True so that Model is allowed to validate a None value. # - # TODO: Revisit the need for MyModelMetaclass hack in pydantic v2 + # TODO: Revisit the need for _ModelMetaclass hack in pydantic v2 def __instancecheck__(self, instance: Any) -> bool: if instance is None: return True return super().__instancecheck__(instance) -class Model(GenericModel, Generic[_RootT], metaclass=MyModelMetaclass): +class Model(GenericModel, Generic[_RootT], DataClassBase, metaclass=_ModelMetaclass): """ A data model containing a value parsed according to the model. @@ -395,7 +392,7 @@ def __init__( else: raise - self._take_snapshot_of_validated_contents() # initial snapshot + self._take_snapshot_of_validated_contents(initial=True) if not self.__class__.__doc__: self._set_standard_field_description() @@ -447,11 +444,20 @@ def update_forward_refs(cls, **localns: Any) -> None: cls.__name__ = remove_forward_ref_notation(cls.__name__) cls.__qualname__ = remove_forward_ref_notation(cls.__qualname__) - def validate_contents(self) -> None: - self._validate_and_set_contents(self.contents) + def validate_contents(self, restore_snapshot_if_interactive_and_invalid: bool = False) -> None: + self._validate_and_set_value( + self.contents, + restore_snapshot_if_interactive_and_invalid=restore_snapshot_if_interactive_and_invalid) - def _validate_and_set_contents(self, new_contents: object) -> None: - self.contents = self._validate_contents_from_value(new_contents) + def _validate_and_set_value(self, + new_contents: object, + restore_snapshot_if_interactive_and_invalid: bool = False) -> None: + if restore_snapshot_if_interactive_and_invalid and self.config.interactive_mode: + reset_solution = AttribHolder(self, 'contents', self.snapshot, reset_to_other=True) + else: + reset_solution = nothing() + with reset_solution: + self.contents = self._validate_contents_from_value(new_contents) self._take_snapshot_of_validated_contents() def _validate_contents_from_value(self, value: object) -> _RootT: @@ -469,10 +475,26 @@ def _get_restorable_contents(self): _restorable_content_cache[id(self)] = RestorableContents() return _restorable_content_cache.get(id(self)) - def _take_snapshot_of_validated_contents(self): - interactive_mode = _is_interactive_mode() - if interactive_mode: + @property + def snapshot(self) -> Snapshot: + return self._get_restorable_contents().get_last_snapshot() + + def snapshot_taken_of_same_obj(self, obj: object) -> bool: + return self._get_restorable_contents().last_snapshot_taken_of_same_obj(obj) + + def differs_from_snapshot(self, obj: object) -> bool: + return self._get_restorable_contents().differs_from_last_snapshot(obj) + + @property + def contents_validated(self) -> bool: + needs_validation = self.differs_from_snapshot(self.contents) \ + or not self.snapshot_taken_of_same_obj(self.contents) + return not needs_validation + + def _take_snapshot_of_validated_contents(self, initial: bool = False) -> None: + if initial or self.config.interactive_mode: self._get_restorable_contents().take_snapshot(self.contents) + print(f'{id(self.contents)} -> {id(self.snapshot)}: {self.contents}') @classmethod def _parse_data(cls, data: Any) -> _RootT: @@ -566,7 +588,7 @@ def to_data(self) -> object: return super().dict(by_alias=True)[ROOT_KEY] def from_data(self, value: object) -> None: - self._validate_and_set_contents(value) + self._validate_and_set_value(value) def absorb_and_replace(self, other: 'Model'): self.from_data(other.to_data()) @@ -670,56 +692,27 @@ def __setattr__(self, attr: str, value: Any) -> None: def _special_method( # noqa: C901 self, name: str, info: MethodInfo, *args: object, **kwargs: object) -> object: - try: - method = self._getattr_from_contents_obj(name) - except AttributeError: - if name == '__len__': - raise TypeError(f"object of type '{self.__class__.__name__}' has no len()") - else: - raise - - if info.state_changing: - restorable = self._get_restorable_contents() - reset_solution: ContextManager - - if _is_interactive_mode(): - if restorable.has_snapshot() \ - and restorable.last_snapshot_taken_of_same_obj(self.contents) \ - and restorable.differs_from_last_snapshot(self.contents): - # Current contents not validated - reset_contents_to_last_snapshot = AttribHolder( - self, 'contents', restorable.get_last_snapshot(), reset_to_other=True) - with reset_contents_to_last_snapshot: - validated_contents = self._validate_contents_from_value(self.contents) + if info.state_changing and self.config.interactive_mode: + if not self.contents_validated: + self.validate_contents(restore_snapshot_if_interactive_and_invalid=True) - reset_contents_to_validated_prev = AttribHolder( - self, 'contents', validated_contents, reset_to_other=True) - reset_solution = reset_contents_to_validated_prev - else: - reset_contents_to_prev = AttribHolder(self, 'contents', copy_attr=True) - reset_solution = reset_contents_to_prev - else: - reset_solution = nothing() - - with reset_solution: - ret = method(*args, **kwargs) - if _is_interactive_mode(): - needs_validation = restorable.differs_from_last_snapshot(self.contents) \ - if restorable.has_snapshot() else True - else: - needs_validation = True - if needs_validation: + reset_contents_to_prev = AttribHolder(self, 'contents', copy_attr=True) + with reset_contents_to_prev: + ret = self._call_special_method(name, *args, **kwargs) + if self.differs_from_snapshot(self.contents): self.validate_contents() + elif name == '__iter__' and isinstance(self, Iterable): _per_element_model_generator = self._get_per_element_model_generator( cast(Iterable, self.contents), level_up_arg_idx=0, ) - return _per_element_model_generator() else: - ret = method(*args, **kwargs) + ret = self._call_special_method(name, *args, **kwargs) + if info.state_changing: + self.validate_contents() if info.maybe_returns_same_type: level_up = False @@ -734,6 +727,18 @@ def _special_method( # noqa: C901 return ret + def _call_special_method(self, name: str, *args: object, **kwargs: object) -> object: + try: + method = self._getattr_from_contents_obj(name) + except AttributeError: + if name == '__len__': + raise TypeError(f"object of type '{self.__class__.__name__}' has no len()") + else: + raise + + ret = method(*args, **kwargs) + return ret + def _get_per_element_model_generator(self, elements: Iterable | None, level_up_arg_idx: int | slice) -> Callable[..., Generator]: @@ -754,7 +759,11 @@ def _convert_to_model_if_reasonable( # noqa: C901 level_up_arg_idx: int = 1) -> ('Model[_KeyT] | Model[_ValT] | Model[tuple[_KeyT, _ValT]] ' '| Model[_ReturnT] | Model[_RootT] | _ReturnT'): - if not is_model_instance(ret): + if level_up and not self.config.dynamically_convert_elements_to_models: + ... + elif is_model_instance(ret): + ... + else: outer_type = self.outer_type(with_args=True) # For double Models, e.g. Model[Model[int]], where _get_real_contents() have already # skipped the outer Model to get the `ret`, we need to do the same to compare the value @@ -810,7 +819,7 @@ def _fix_tuple_type_from_args( def __getattr__(self, attr: str) -> Any: contents_attr = self._getattr_from_contents_obj(attr) - if _is_interactive_mode() and not self._is_non_omnipy_pydantic_model(): + if self.config.interactive_mode and not self._is_non_omnipy_pydantic_model(): contents_holder_context = AttribHolder(self, 'contents', copy_attr=True) contents_cls_attr = self._getattr_from_contents_cls(attr) @@ -869,7 +878,7 @@ def __eq__(self, other: object) -> bool: and self.to_data() == cast(Model, other).to_data() # last line is just in case def __repr__(self) -> str: - if _is_interactive_mode() and not _waiting_for_terminal_repr(): + if self.config.interactive_mode and not _waiting_for_terminal_repr(): if get_calling_module_name() in INTERACTIVE_MODULES: _waiting_for_terminal_repr(True) return self._table_repr() @@ -1036,7 +1045,7 @@ def from_json(self, json_contents: str, **kwargs: _KwargValT) -> None: self._validate_and_set_contents_with_params(cast(_ParamRootT, self.contents), **kwargs) def _validate_and_set_contents_with_params(self, contents: _ParamRootT, **kwargs: _KwargValT): - self._validate_and_set_contents(DataWithParams(data=contents, params=kwargs)) + self._validate_and_set_value(DataWithParams(data=contents, params=kwargs)) _ParamModelT = TypeVar('_ParamModelT', bound='ParamModel') @@ -1071,7 +1080,7 @@ def _validate_and_set_contents_with_params( self, contents: list[_ParamModelT | DataWithParams[_ParamModelT, _KwargValT]], **kwargs: _KwargValT): - self._validate_and_set_contents([ + self._validate_and_set_value([ DataWithParams(data=cast(_ParamModelT, model).contents, params=kwargs) for model in contents ]) diff --git a/src/omnipy/engine/local.py b/src/omnipy/engine/local.py index 274e409a..b1fd3e0a 100644 --- a/src/omnipy/engine/local.py +++ b/src/omnipy/engine/local.py @@ -1,3 +1,4 @@ +from dataclasses import dataclass from typing import Any, Callable, Type from omnipy.api.protocols.public.compute import IsDagFlow, IsFuncFlow, IsLinearFlow, IsTask @@ -7,6 +8,12 @@ FuncFlowRunnerEngine, LinearFlowRunnerEngine, TaskRunnerEngine) +from omnipy.hub.entry import RuntimeEntryPublisher + + +@dataclass +class LocalRunnerConfigEntryPublisher(LocalRunnerConfig, RuntimeEntryPublisher): + ... class LocalRunner(TaskRunnerEngine, @@ -22,7 +29,7 @@ def _update_from_config(self) -> None: @classmethod def get_config_cls(cls) -> Type[IsLocalRunnerConfig]: - return LocalRunnerConfig + return LocalRunnerConfigEntryPublisher def _init_task(self, task: IsTask, call_func: Callable) -> Any: ... diff --git a/src/omnipy/hub/runtime.py b/src/omnipy/hub/runtime.py index 6631c848..19d0c4b0 100644 --- a/src/omnipy/hub/runtime.py +++ b/src/omnipy/hub/runtime.py @@ -3,6 +3,7 @@ from omnipy.api.enums import EngineChoice from omnipy.api.protocols.private.compute.job_creator import IsJobConfigHolder +from omnipy.api.protocols.private.data import IsDataConfigHolder from omnipy.api.protocols.private.engine import IsEngine from omnipy.api.protocols.private.log import IsRunStateRegistry from omnipy.api.protocols.public.config import (IsDataConfig, @@ -17,12 +18,13 @@ from omnipy.config.data import DataConfig from omnipy.config.engine import LocalRunnerConfig, PrefectEngineConfig from omnipy.config.job import JobConfig +from omnipy.data.data_class_creator import DataClassBase from omnipy.data.serializer import SerializerRegistry -from omnipy.engine.local import LocalRunner +from omnipy.engine.local import LocalRunner, LocalRunnerConfigEntryPublisher from omnipy.hub.entry import DataPublisher, RuntimeEntryPublisher from omnipy.hub.root_log import RootLogConfigEntryPublisher, RootLogObjects from omnipy.log.registry import RunStateRegistry -from omnipy.modules.prefect.engine.prefect import PrefectEngine +from omnipy.modules.prefect.engine.prefect import PrefectEngine, PrefectEngineConfigEntryPublisher from omnipy.util.helpers import called_from_omnipy_tests @@ -30,19 +32,43 @@ def _job_creator_factory(): return JobBase.job_creator +def _data_class_creator_factory(): + return DataClassBase.data_class_creator + + +def _data_config_factory(): + return _data_class_creator_factory().config + + @dataclass class RuntimeConfig(RuntimeEntryPublisher): job: IsJobConfig = field(default_factory=JobConfig) - data: IsDataConfig = field(default_factory=DataConfig) + data: IsDataConfig = field(default_factory=_data_config_factory) engine: EngineChoice = EngineChoice.LOCAL - local: IsLocalRunnerConfig = field(default_factory=LocalRunnerConfig) - prefect: IsPrefectEngineConfig = field(default_factory=PrefectEngineConfig) + local: IsLocalRunnerConfig = field(default_factory=LocalRunnerConfigEntryPublisher) + prefect: IsPrefectEngineConfig = field(default_factory=PrefectEngineConfigEntryPublisher) root_log: IsRootLogConfig = field(default_factory=RootLogConfigEntryPublisher) + def reset_to_defaults(self) -> None: + prev_back = self._back + self._back = None + + self.job = JobConfig() + self.data = DataConfig() + self.engine = EngineChoice.LOCAL + self.local = LocalRunnerConfigEntryPublisher() + self.prefect = PrefectEngineConfigEntryPublisher() + self.root_log = RootLogConfigEntryPublisher() + + self._back = prev_back + if self._back is not None: + self._back.reset_subscriptions() + @dataclass class RuntimeObjects(RuntimeEntryPublisher): job_creator: IsJobConfigHolder = field(default_factory=_job_creator_factory) + data_class_creator: IsDataConfigHolder = field(default_factory=_data_class_creator_factory) local: IsEngine = field(default_factory=LocalRunner) prefect: IsEngine = field(default_factory=PrefectEngine) registry: IsRunStateRegistry = field(default_factory=RunStateRegistry) @@ -59,10 +85,6 @@ class Runtime(DataPublisher): def __post_init__(self): super().__init__() - self.config._back = self - self.config.root_log._back = self - self.objects._back = self - self.reset_subscriptions() def reset_subscriptions(self): @@ -72,10 +94,14 @@ def reset_subscriptions(self): This function unsubscribes all existing subscriptions and then sets up new subscriptions for the `config` and `objects` members. """ + + self.reset_backlinks() + self.config.unsubscribe_all() self.objects.unsubscribe_all() self.config.subscribe('job', self.objects.job_creator.set_config) + self.config.subscribe('data', self.objects.data_class_creator.set_config) self.config.subscribe('local', self.objects.local.set_config) self.config.subscribe('prefect', self.objects.prefect.set_config) self.config.subscribe('root_log', self.objects.root_log.set_config) @@ -90,6 +116,13 @@ def reset_subscriptions(self): self.objects.subscribe('local', self._update_local_runner_config) self.objects.subscribe('prefect', self._update_prefect_engine_config) + def reset_backlinks(self): + self.config._back = self + self.config.local._back = self + self.config.prefect._back = self + self.config.root_log._back = self + self.objects._back = self + def _get_engine_config(self, engine_choice: EngineChoice): return getattr(self.config, engine_choice) diff --git a/src/omnipy/modules/pandas/models.py b/src/omnipy/modules/pandas/models.py index 03ed57ba..ebeaba08 100644 --- a/src/omnipy/modules/pandas/models.py +++ b/src/omnipy/modules/pandas/models.py @@ -32,10 +32,10 @@ def to_data(self) -> Any: return df.to_dict() def from_data(self, value: Iterable[Any]) -> None: - self._validate_and_set_contents(pd.DataFrame(value).convert_dtypes()) + self._validate_and_set_value(pd.DataFrame(value).convert_dtypes()) def from_json(self, value: str) -> None: - self._validate_and_set_contents(pd.read_json(value).convert_dtypes()) + self._validate_and_set_value(pd.read_json(value).convert_dtypes()) class PandasDataset(Dataset[PandasModel]): diff --git a/src/omnipy/modules/prefect/engine/prefect.py b/src/omnipy/modules/prefect/engine/prefect.py index b6d8fdaa..e4a82ec2 100644 --- a/src/omnipy/modules/prefect/engine/prefect.py +++ b/src/omnipy/modules/prefect/engine/prefect.py @@ -1,4 +1,5 @@ import asyncio +from dataclasses import dataclass from datetime import timedelta from typing import Any, Callable, Type @@ -9,11 +10,17 @@ FuncFlowRunnerEngine, LinearFlowRunnerEngine, TaskRunnerEngine) +from omnipy.hub.entry import RuntimeEntryPublisher from omnipy.util.helpers import resolve from .. import prefect_flow, prefect_task, PrefectFlow, PrefectTask, task_input_hash +@dataclass +class PrefectEngineConfigEntryPublisher(PrefectEngineConfig, RuntimeEntryPublisher): + ... + + class PrefectEngine(TaskRunnerEngine, LinearFlowRunnerEngine, DagFlowRunnerEngine, diff --git a/src/omnipy/util/helpers.py b/src/omnipy/util/helpers.py index 33c45abb..c89bedd9 100644 --- a/src/omnipy/util/helpers.py +++ b/src/omnipy/util/helpers.py @@ -1,11 +1,15 @@ +from collections import defaultdict, UserDict from collections.abc import Hashable, Iterable from copy import copy, deepcopy +from dataclasses import dataclass import functools +import gc import inspect from inspect import getmodule, isclass import locale as pkg_locale import operator import sys +import types from types import GenericAlias, ModuleType, NoneType, UnionType from typing import _AnnotatedAlias # type: ignore[attr-defined] from typing import _LiteralGenericAlias # type: ignore[attr-defined] @@ -14,18 +18,26 @@ from typing import (_SpecialForm, Annotated, Any, + Callable, cast, ClassVar, + Dict, ForwardRef, + Generic, get_args, get_origin, Mapping, NamedTuple, overload, Protocol, + Sequence, + Type, TypeVar, Union) +import weakref +from weakref import WeakKeyDictionary, WeakValueDictionary +from bidict import bidict from isort import place_module from isort.sections import STDLIB from pydantic import BaseModel, ValidationError @@ -33,9 +45,14 @@ from pydantic.typing import display_as_type from typing_inspect import get_generic_bases, is_generic_type +from omnipy.api.protocols.private.util import HasContents from omnipy.api.typedefs import LocaleType, TypeForm _KeyT = TypeVar('_KeyT', bound=Hashable) +_ObjT = TypeVar('_ObjT', bound=object) +_AnyKeyT = TypeVar('_AnyKeyT', bound=object) +_ValT = TypeVar('_ValT', bound=object) +_ContentT = TypeVar('_ContentT', bound=object) Dictable = Mapping[_KeyT, Any] | Iterable[tuple[_KeyT, Any]] @@ -60,7 +77,7 @@ def create_merged_dict(dictable_1: Dictable[_KeyT], def remove_none_vals(**kwargs: object) -> dict[object, object]: - return {key: val for key, val in kwargs.items() if val is not None} + return {key: obj for key, obj in kwargs.items() if obj is not None} def get_datetime_format(locale: LocaleType | None = None) -> str: @@ -73,8 +90,8 @@ def get_datetime_format(locale: LocaleType | None = None) -> str: return datetime_format -async def resolve(val): - return await val if inspect.isawaitable(val) else val +async def resolve(obj): + return await obj if inspect.isawaitable(obj) else obj def repr_max_len(data: object, max_len: int = 200): @@ -278,9 +295,308 @@ def generate_qualname(cls_name: str, model: Any) -> str: return f'{cls_name}[{fully_qual_model_name}]' -class Snapshot(NamedTuple): +# _deepcopy_dispatch: dict[type, Callable] +# d: dict[type, Callable] +# +# # Copied from stdlib (python3.10/copy.py:180), update for new CPython versions +# _deepcopy_dispatch = d = {} +# +# +# def _deepcopy_atomic(x, memo): +# return x +# +# +# d[type(None)] = _deepcopy_atomic +# d[type(Ellipsis)] = _deepcopy_atomic +# d[type(NotImplemented)] = _deepcopy_atomic +# d[int] = _deepcopy_atomic +# d[float] = _deepcopy_atomic +# d[bool] = _deepcopy_atomic +# d[complex] = _deepcopy_atomic +# d[bytes] = _deepcopy_atomic +# d[str] = _deepcopy_atomic +# d[types.CodeType] = _deepcopy_atomic +# d[type] = _deepcopy_atomic +# d[range] = _deepcopy_atomic +# d[types.BuiltinFunctionType] = _deepcopy_atomic +# d[types.FunctionType] = _deepcopy_atomic +# d[weakref.ref] = _deepcopy_atomic +# d[property] = _deepcopy_atomic + + +class RefCountMemoDict(UserDict[int, _ObjT], Generic[_ObjT]): + def __init__(self, dict=None, /, **kwargs) -> None: + super().__init__(dict, **kwargs) + self._key_2_obj_id = bidict[int, int]() + + # def __init__(self, dict=None, /, **kwargs) -> None: + # self._std_dict: dict[int, _ObjT] = {} + # self._weak_value_dict = WeakValueDictionary[int, _ObjT]() + # if dict is not None: + # self.update(dict) + # if kwargs: + # self.update(kwargs) + # + # def __setattr__(self, key, value): + # if key == 'data': + # self._std_dict.clear() + # self._weak_value_dict.clear() + # for k, v in value.items(): + # self[k] = v + # super().__setattr__(key, value) + # + # def __getattr__(self, item): + # if item == 'data': + # return self._std_dict | dict(self._weak_value_dict.items()) + # return super().__getattr__(item) + + def __setitem__(self, key, obj): + try: + print(f'{key},{id(obj)}: {obj} [{type(obj)}]') + except AttributeError: + print(f'{key},{id(obj)}: [{type(obj)}]') + # if isinstance(obj, list): + # print([id(x) for x in obj]) + # obj = [self.data[id(x)] if id(x) in self.data else x for x in obj] + # elif isinstance(obj, dict): + # print({k: id(v) for k, v in obj.items()}) + # obj = {k: self.data[id(v)] if id(v) in self.data else v for k, v in obj.items()} + # try: + # obj = weakref.ref(obj) + # except TypeError: + # pass + # print(f'{key}: {obj}') + if key != id(self): + self.data[key] = obj + self._key_2_obj_id[key] = id(obj) + # print(f'{orig_id_obj}: {obj}') + # self.data[orig_id_obj] = obj + # try: + # weakref.ref(obj) + # self._weak_value_dict[key] = obj + # except TypeError: + # self._std_dict[key] = obj + + # def __getitem__(self, item): + # print(f'Getting {item}') + # ret = super().__getitem__(item) + # # if isinstance(ret, weakref.ref): + # # return ret() + # # elif isinstance(ret, list): + # # return [x() if isinstance(x, weakref.ref) else x for x in ret] + # # elif isinstance(ret, dict): + # # return {k: v() if isinstance(v, weakref.ref) else v for k, v in ret.items()} + # return ret + + # def __delitem__(self, key): + # if key in self._weak_value_dict: + # del self._weak_value_dict[key] + # else: + # del self._std_dict[key] + + # def __copy__(self): + # inst = self.__class__.__new__(self.__class__) + # inst.__dict__.update(self.__dict__) + # # Create a copy and avoid triggering descriptors + # inst.__dict__["data"] = self.__dict__["data"].copy() + # return inst + + # def get(self, key, default=None): + # try: + # return self.__getitem__(key) + # except KeyError: + # return default + + def recursively_remove_deleted_objs( + self, + *keys: int, + known_references_callback: Callable[[int], int] | None = None, + ): + print(f'Recursively removing deleted objects for {keys}...') + + known_refcount_for_all_contained_objs_in_memo = defaultdict[int, int](lambda: 0) + self.get_known_refcount_for_all_contained_objs_in_memo( + *keys, + refcount_of_all_contained_objs_in_memo=known_refcount_for_all_contained_objs_in_memo, + known_refcount_callback=known_references_callback) + print(known_refcount_for_all_contained_objs_in_memo) + self._remove_deleted_objs(known_refcount_for_all_contained_objs_in_memo) + + def _remove_deleted_objs(self, known_refcount_for_all_contained_objs_in_memo): + while True: + # gc.collect() # Try to uncomment if memo dict is not cleared + to_remove = None + for key, known_ref_count in known_refcount_for_all_contained_objs_in_memo.items(): + obj = self.data[key] + ref_count = sys.getrefcount(obj) + print(f'{obj} has {ref_count} references') + for k, v in self.data.items(): + print(f'{k}: {v}') + ref_count_target = 4 if isinstance(obj, tuple) else 3 + # ref_count_target = 3 + ref_count_target += known_ref_count + refs = gc.get_referrers(obj) + for ref in refs: + print(f'{type(ref)}: {ref}') + try: + print( + f"ref['obj_copy'][1].__dict__: {ref['obj_copy'][1].__dict__}, id={id(ref['obj_copy'][1].__dict__)}" + ) + print( + f"ref['obj_copy'][1].data: {ref['obj_copy'][1].data}, id={id(ref['obj_copy'][1].data)}" + ) + except: + pass + # 3 references: the one in the memo dict, one in the gc, and obj + # +1 reference for tuples, as they are immutable + if ref_count <= ref_count_target: + print(f'Removing {self.data[key]}') + to_remove = key + break + if to_remove: + del self[to_remove] + del known_refcount_for_all_contained_objs_in_memo[to_remove] + else: + break + + # def _get_objects_in_memo_dict(self, *keys: int) -> dict[int, _ObjT]: + # + # elements_in_memo = {key: self.data[key] for key in keys if key in self.data} + # return { + # key: cast(_ObjT, obj()) if isinstance(obj, weakref.ref) else obj for key, + # obj in elements_in_memo.items() + # } + + def get_known_refcount_for_all_contained_objs_in_memo( + self, + *keys_of_objs_to_check: int, + refcount_of_all_contained_objs_in_memo: defaultdict[int, int], + known_refcount_callback: Callable[[int], int] | None = None, + ) -> None: + print(f'keys_of_objs_to_check: {keys_of_objs_to_check}') + dict_of_objs_to_check_in_memo = { + key: self.data[key] for key in keys_of_objs_to_check if key in self.data + } + print(f'dict_of_objs_to_check_in_memo: {dict_of_objs_to_check_in_memo}') + keys_of_contained_objs_also_in_memo: tuple[int, ...] = () + for key_to_check, obj_to_check in dict_of_objs_to_check_in_memo.items(): + contained_objs = {id(obj): obj for obj in gc.get_referents(obj_to_check)} + print(f'contained_objs: {contained_objs}') + ids_of_contained_objs = tuple( + id(contained_obj) for contained_obj in gc.get_referents(obj_to_check)) + print(f'ids_of_contained_objs: {ids_of_contained_objs}') + keys_of_contained_objs_also_in_memo += tuple( + self._key_2_obj_id.inverse[contained_obj_id] + for contained_obj_id in ids_of_contained_objs + if contained_obj_id in self._key_2_obj_id.inverse) + known_refcount = 0 + if known_refcount_callback: + known_refcount = known_refcount_callback(id(obj_to_check)) + for key in (key_to_check,) + keys_of_contained_objs_also_in_memo: + refcount_of_all_contained_objs_in_memo[key] += known_refcount + print('refcount_of_all_contained_objs_in_memo:', refcount_of_all_contained_objs_in_memo) + + if len(keys_of_contained_objs_also_in_memo) > 0: + return self.get_known_refcount_for_all_contained_objs_in_memo( + *keys_of_contained_objs_also_in_memo, + refcount_of_all_contained_objs_in_memo=refcount_of_all_contained_objs_in_memo, + known_refcount_callback=known_refcount_callback) + + +class KeyRef(list): + def __init__(self, obj: object) -> None: + super().__init__([id(obj)]) + + def __hash__(self) -> int: # type: ignore[override] + return self[0] + + +class WeakKeyRefContainer(Generic[_AnyKeyT, _ValT]): + def __init__(self) -> None: + self._key_dict: WeakValueDictionary[KeyRef, _AnyKeyT] = WeakValueDictionary() + self._value_dict: WeakKeyDictionary[KeyRef, _ValT] = WeakKeyDictionary() + + def __contains__(self, key: _AnyKeyT) -> bool: + return KeyRef(key) in self._value_dict + + def get(self, key: _AnyKeyT) -> _ValT | None: + key_ref = KeyRef(key) + if key_ref in self._value_dict: + return self._value_dict[key_ref] + else: + return None + + def __getitem__(self, key: _AnyKeyT) -> _ValT: + key_ref = KeyRef(key) + if key_ref in self._value_dict: + return self._value_dict[key_ref] + else: + raise KeyError(f'{key} is not in {self.__class__.__name__}') + + def __setitem__(self, key: _AnyKeyT, value: _ValT) -> None: + key_ref = KeyRef(key) + self._key_dict[key_ref] = key + self._value_dict[key_ref] = value + + def __len__(self) -> int: + return len(self._value_dict) + + +@dataclass +class Snapshot(Generic[_ObjT, _ContentT]): id: int - obj_copy: object + obj_copy: _ContentT + + def taken_of_same_obj(self, obj: _ObjT) -> bool: + return self.id == id(obj) + + def differs_from(self, obj: _ObjT) -> bool: + return not all_equals(self.obj_copy, obj) + + +class SnapshotHolder(WeakKeyRefContainer[HasContents[_ContentT], Snapshot[_ObjT, _ContentT]], + Generic[_ObjT, _ContentT]): + def __init__(self) -> None: + super().__init__() + self._deepcopy_memo = RefCountMemoDict[_ContentT]() + self._key_2_obj_copy_id = dict[int, int]() + self._obj_copy_id_keys = defaultdict[int, list[int]](list[int]) + + def __setitem__(self, obj: HasContents[_ContentT], value: Snapshot[_ObjT, _ContentT]) -> None: + raise TypeError(f"'{self.__class__.__name__}' object does not support item assignment") + + def take_snapshot(self, obj: HasContents[_ContentT]) -> None: + try: + obj_copy: _ContentT = deepcopy(obj.contents, + self._deepcopy_memo) # type: ignore[arg-type] + except (TypeError, ValueError, ValidationError) as exp: + print(exp) + obj_copy = copy(obj.contents) + + key = id(obj) + obj_copy_id = id(obj_copy) + + super().__setitem__(obj, Snapshot(key, obj_copy)) + + if key in self._key_2_obj_copy_id: + prev_obj_copy_id = self._key_2_obj_copy_id[key] + self._obj_copy_id_keys[prev_obj_copy_id].remove(key) + self._key_2_obj_copy_id[key] = obj_copy_id + self._obj_copy_id_keys[obj_copy_id].append(key) + + def recursively_remove_deleted_obj_from_deepcopy_memo(self, obj_copy_id: int) -> None: + def _known_snapshot_references(obj_copy_id: int) -> int: + if obj_copy_id in self._obj_copy_id_keys: + return len(self._obj_copy_id_keys[obj_copy_id]) + return 0 + + self._deepcopy_memo.recursively_remove_deleted_objs( + *self._obj_copy_id_keys[obj_copy_id], + known_references_callback=_known_snapshot_references, + ) + + +_memo: dict[int, object] = {} class RestorableContents: @@ -292,8 +608,8 @@ def has_snapshot(self) -> bool: def take_snapshot(self, obj: object): try: - snapshot_obj = deepcopy(obj) - except (TypeError, ValueError, ValidationError): + snapshot_obj = deepcopy(obj, _memo) + except (TypeError, ValueError, ValidationError) as exp: snapshot_obj = copy(obj) self._last_snapshot = Snapshot(id(obj), snapshot_obj) @@ -319,9 +635,9 @@ def recursive_module_import(module: ModuleType, imported_modules: list[ModuleTyp module_vars = vars(module) imported_modules.append(module) - for val in module_vars.values(): - if isclass(val): - for base_cls in val.__bases__: + for obj in module_vars.values(): + if isclass(obj): + for base_cls in obj.__bases__: base_cls_module = getmodule(base_cls) if base_cls_module and _is_internal_module(base_cls_module, imported_modules): module_vars = create_merged_dict( diff --git a/tests/compute/conftest.py b/tests/compute/conftest.py index ade7ffa0..250a0d40 100644 --- a/tests/compute/conftest.py +++ b/tests/compute/conftest.py @@ -21,12 +21,6 @@ def mock_job_classes() -> tuple[Type[JobTemplateMixin], Type[JobMixin]]: return MockJobTemplateSubclass, MockJobSubclass -@pytest.fixture(scope='function') -def teardown_reset_job_creator() -> None: - yield None - JobBaseMeta._job_creator_obj = JobCreator() - - @pytest.fixture(scope='function') def mock_local_runner( teardown_reset_job_creator: Annotated[None, pytest.fixture]) -> MockLocalRunner: diff --git a/tests/conftest.py b/tests/conftest.py index 6d193dce..03aa091f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,7 @@ from datetime import datetime import logging import os +from pathlib import Path import shutil import tempfile from typing import Annotated, Generator, Type @@ -10,6 +11,7 @@ from omnipy.api.protocols.public.hub import IsRuntime from omnipy.compute.job_creator import JobBaseMeta, JobCreator from omnipy.config.root_log import RootLogConfig +from omnipy.data.data_class_creator import DataClassBaseMeta, DataClassCreator @pytest.fixture(scope='function') @@ -32,15 +34,29 @@ def teardown_remove_root_log_handlers() -> Generator[None, None, None]: @pytest.fixture(scope='function') -def tmp_dir_path() -> Generator[str, None, None]: +def tmp_dir_path() -> Generator[Path, None, None]: with tempfile.TemporaryDirectory() as _tmp_dir_path: - yield _tmp_dir_path + yield Path(_tmp_dir_path) + + +@pytest.fixture(scope='function') +def teardown_reset_job_creator() -> Generator[None, None, None]: + yield None + JobBaseMeta._job_creator_obj = JobCreator() + + +@pytest.fixture(scope='function') +def teardown_reset_data_class_creator() -> Generator[None, None, None]: + yield None + DataClassBaseMeta._data_class_creator_obj = DataClassCreator() @pytest.fixture(scope='function') def runtime_cls( teardown_rm_root_log_dir: Annotated[None, pytest.fixture], teardown_remove_root_log_handlers: Annotated[None, pytest.fixture], + teardown_reset_job_creator: Annotated[None, pytest.fixture], + teardown_reset_data_class_creator: Annotated[None, pytest.fixture], ) -> Type[IsRuntime]: from omnipy.hub.runtime import Runtime return Runtime @@ -49,18 +65,16 @@ def runtime_cls( @pytest.fixture(scope='function') def runtime( runtime_cls: Annotated[Type[IsRuntime], pytest.fixture], - tmp_dir_path: Annotated[str, pytest.fixture], + tmp_dir_path: Annotated[Path, pytest.fixture], ) -> Generator[IsRuntime, None, None]: runtime = runtime_cls() - runtime.config.job.output_storage.local.persist_data_dir_path = os.path.join( - tmp_dir_path, 'outputs') - runtime.config.root_log.file_log_dir_path = os.path.join(tmp_dir_path, 'logs') + runtime.config.reset_to_defaults() + runtime.config.job.output_storage.local.persist_data_dir_path = str(tmp_dir_path / 'outputs') + runtime.config.root_log.file_log_dir_path = str(tmp_dir_path / 'logs') yield runtime - JobBaseMeta._job_creator_obj = JobCreator() - @pytest.fixture(scope='function') def mock_datetime() -> datetime: diff --git a/tests/data/helpers/mocks.py b/tests/data/helpers/mocks.py index a6f59141..6cfd40db 100644 --- a/tests/data/helpers/mocks.py +++ b/tests/data/helpers/mocks.py @@ -2,11 +2,20 @@ from typing import Any, IO, Type from omnipy.api.protocols.public.data import IsDataset +from omnipy.data.data_class_creator import DataClassBase from omnipy.data.dataset import Dataset from omnipy.data.model import Model from omnipy.data.serializer import Serializer, TarFileSerializer +class MockDataset(DataClassBase): + ... + + +class MockModel(DataClassBase): + ... + + class NumberDataset(Dataset[Model[int]]): ... diff --git a/tests/data/helpers/models.py b/tests/data/helpers/models.py index 3df894bc..c725f2d8 100644 --- a/tests/data/helpers/models.py +++ b/tests/data/helpers/models.py @@ -20,7 +20,7 @@ def to_data(self) -> float: return self.contents.int_part + self.contents.float_part def from_data(self, value: float): - self._validate_and_set_contents(MyFloatObject(int_part=floor(value), float_part=value % 1)) + self._validate_and_set_value(MyFloatObject(int_part=floor(value), float_part=value % 1)) class StringToLength(Model[str]): diff --git a/tests/data/test_data_class_creator.py b/tests/data/test_data_class_creator.py new file mode 100644 index 00000000..ac21edbb --- /dev/null +++ b/tests/data/test_data_class_creator.py @@ -0,0 +1,99 @@ +from typing import Annotated + +import pytest + +from data.helpers.mocks import MockDataset, MockModel +from omnipy.api.protocols.public.config import IsDataConfig +from omnipy.config.data import DataConfig +from omnipy.data.data_class_creator import DataClassBase, DataClassBaseMeta, DataClassCreator + + +def test_data_class_creator_init(): + with pytest.raises(TypeError): + DataClassCreator('something') # noqa + + data_class_creator_1 = DataClassCreator() + data_class_creator_2 = DataClassCreator() + + assert data_class_creator_1 != data_class_creator_2 + + +def test_data_class_creator_set_config() -> None: + data_class_creator = DataClassCreator() + assert data_class_creator.config == DataConfig() + + new_data_config = DataConfig(interactive_mode=False) + with pytest.raises(AttributeError): + data_class_creator.config = new_data_config + + data_class_creator.set_config(new_data_config) + assert data_class_creator.config == new_data_config + + +def test_data_class_creator_singular_mock( + teardown_reset_data_class_creator: Annotated[None, pytest.fixture]) -> None: + + assert isinstance(DataClassBase.data_class_creator, DataClassCreator) + + with pytest.raises(AttributeError): + DataClassBase.data_class_creator = DataClassCreator() # type: ignore[misc] + + with pytest.raises(AttributeError): + MockDataset.data_class_creator = DataClassCreator() # type: ignore[misc] + + with pytest.raises(AttributeError): + MockModel.data_class_creator = DataClassCreator() # type: ignore[misc] + + data_class_creator = DataClassBase.data_class_creator + assert MockDataset.data_class_creator is data_class_creator + assert MockModel.data_class_creator is data_class_creator + + dataset = MockDataset() + assert dataset.__class__.data_class_creator is data_class_creator + + model = MockModel() + assert model.__class__.data_class_creator is data_class_creator + + dataset_new = MockDataset() + assert dataset_new.__class__.data_class_creator is data_class_creator + + +def test_data_class_creator_properties_mock( + teardown_reset_data_class_creator: Annotated[None, pytest.fixture]) -> None: + data_classes = (MockDataset, MockModel, DataClassBase) + data_objects = (MockDataset(), MockModel()) + data_config = DataClassBase.data_class_creator.config + + _assert_config_property_in_classes(data_classes, val=data_config) + _assert_config_property_in_objects(data_objects, val=data_config) + + new_data_config = DataConfig(interactive_mode=False) + MockDataset.data_class_creator.set_config(new_data_config) + + _assert_config_property_in_classes(data_classes, val=new_data_config) + _assert_config_property_in_objects(data_objects, val=new_data_config) + + new_data_objects = (MockDataset(), MockModel()) + assert new_data_objects[0] is not data_objects[0] + assert new_data_objects[1] is not data_objects[1] + + _assert_config_property_in_objects(new_data_objects, val=new_data_config) + + assert not hasattr(MockDataset, 'set_config') + assert not hasattr(MockModel, 'set_config') + assert not hasattr(DataClassBase, 'set_config') + + +def _assert_config_property_in_classes(data_classes: tuple[DataClassBaseMeta, ...], + val: IsDataConfig) -> None: + for data_cls in data_classes: + assert getattr(data_cls.data_class_creator, 'config') is val + assert hasattr(data_cls, 'config') + + +def _assert_config_property_in_objects(data_objects: tuple[DataClassBase, ...], + val: IsDataConfig) -> None: + for data_obj in data_objects: + assert getattr(data_obj.__class__.data_class_creator, 'config') is val + assert getattr(data_obj, 'config') is val + assert hasattr(data_obj.__class__, 'config') diff --git a/tests/data/test_model.py b/tests/data/test_model.py index 6563002d..98894f3a 100644 --- a/tests/data/test_model.py +++ b/tests/data/test_model.py @@ -1198,6 +1198,134 @@ def _assert_model_or_val(dyn_convert: bool, _assert_val(model_or_val, target_type, contents) +def test_snapshot_deepcopy_reuse_objects(runtime: Annotated[IsRuntime, pytest.fixture],) -> None: + runtime.config.data.interactive_mode = True + + inner = Model[list[int]]([2, 4]) + middle = Model[list[int | Model[list[int]]]]([1, 3, inner]) + outer = Model[list[int | Model[list[int | Model[list[int]]]]]]([0, middle, 5]) + + assert type(outer[1][-1]) == type(middle[-1]) == type( # type: ignore[index] + inner) == Model[list[int]] + assert id(outer[1][-1]) == id(middle[-1]) == id(inner) # type: ignore[index] + + assert type(outer.snapshot[1][-1]) == type( # type: ignore[index] + middle.snapshot[-1]) == Model[list[int]] + assert id(outer.snapshot[1][-1]) == id(middle.snapshot[-1]) # type: ignore[index] + + assert type(outer[1][-1].contents) == type(middle[-1].contents) == type( # type: ignore[index] + inner.contents) == list + assert id(outer[1][-1].contents) == id(middle[-1].contents) == id( # type: ignore[index] + inner.contents) + + assert type(outer.snapshot[1][-1].contents) == type( # type: ignore[index] + middle.snapshot[-1].contents) == type(inner.snapshot) == list + assert id(outer.snapshot[1][-1].contents) == id( # type: ignore[index] + middle.snapshot[-1].contents) == id(inner.snapshot) + + assert type(outer[1].contents) == type(middle.contents) == list # type: ignore[index] + assert id(outer[1].contents) == id(middle.contents) # type: ignore[index] + + assert type(outer.snapshot[1].contents) == type( # type: ignore[attr-defined] + middle.snapshot) == list + assert id(outer.snapshot[1].contents) == id(middle.snapshot) # type: ignore[attr-defined] + + +def test_snapshot_with_mimic_special_method_and_interactive_mode( + runtime: Annotated[IsRuntime, pytest.fixture],) -> None: + # runtime.config.data.interactive_mode = True + + model = Model[list[int]]([123]) + + assert model.snapshot == model.contents == [123] + assert model.snapshot_taken_of_same_obj(model.contents) is True + assert model.contents_validated is True + + first_snapshot_id = id(model.snapshot) + assert first_snapshot_id != id(model.contents) # snapshot is copy of contents + + with pytest.raises(ValidationError): + model += ['abc'] # type: ignore[operator] + + assert model.snapshot == model.contents == [123] + assert model.snapshot_taken_of_same_obj(model.contents) is False + assert model.contents_validated is False + + second_snapshot_id = id(model.snapshot) + assert second_snapshot_id == first_snapshot_id + + model += [456] # type: ignore[operator] + + assert model.snapshot == model.contents == [123, 456] + assert model.snapshot_taken_of_same_obj(model.contents) is True + assert model.contents_validated is True + + third_snapshot_id = id(model.snapshot) + assert third_snapshot_id != second_snapshot_id + + model.validate_contents() + + assert model.snapshot == model.contents == [123, 456] + assert model.snapshot_taken_of_same_obj(model.contents) is True + assert model.contents_validated is True + + fourth_snapshot_id = id(model.snapshot) + assert fourth_snapshot_id != third_snapshot_id + + +def test_mimic_special_method_no_interactive_mode( + runtime: Annotated[IsRuntime, pytest.fixture],) -> None: + # runtime.config.data.interactive_mode = False + + model = Model[list[int]]([123]) + assert model.has_snapshot is False + with pytest.raises(AssertionError): + assert model.snapshot + + with pytest.raises(ValidationError): + model += ['abc'] # type: ignore[operator] + + _assert_model(model, list[int], [123, 'abc']) + assert model.snapshot is None + + with pytest.raises(ValidationError): + model += [456] # type: ignore[operator] + + _assert_model(model, list[int], [123, 'abc', 456]) + assert model.snapshot is None + + with pytest.raises(ValidationError): + model.validate_contents( + restore_snapshot_if_interactive_and_invalid=False, + take_snapshot_if_interactive_and_valid=True) + + _assert_model(model, list[int], [123, 'abc', 456]) + assert model.snapshot is None + + with pytest.raises(ValidationError): + model.validate_contents( + restore_snapshot_if_interactive_and_invalid=True, + take_snapshot_if_interactive_and_valid=True) + + _assert_model(model, list[int], [123, 'abc', 456]) + assert model.snapshot is None + + runtime.config.data.interactive_mode = True + + with pytest.raises(AssertionError): + model.validate_contents( + restore_snapshot_if_interactive_and_invalid=True, + take_snapshot_if_interactive_and_valid=True) + + del model[1] + + model.validate_contents( + restore_snapshot_if_interactive_and_invalid=True, + take_snapshot_if_interactive_and_valid=True) + assert model.snapshot.obj_copy == model.contents == [123, 456] + assert id(model.snapshot.obj_copy) != id(model.contents) + + @pytest.mark.parametrize('dyn_convert', [False, True]) def test_mimic_simple_list_operations( runtime: Annotated[IsRuntime, pytest.fixture], @@ -1218,7 +1346,7 @@ def test_mimic_simple_list_operations( assert len(model) == 4 _assert_model_or_val(dyn_convert, model[-1], int, 456) # type: ignore[index] - _assert_model_or_val(dyn_convert, model[1:-1], list[int], [234, 345]) # type: ignore[index] + _assert_model(model[1:-1], list[int], [234, 345]) # type: ignore[index] assert tuple(reversed(model)) == (456, 345, 234, 123) @@ -1236,7 +1364,7 @@ def test_mimic_simple_list_operations( model[1] /= 2 # type: ignore[index] _assert_model_or_val(dyn_convert, model[1], int, 117) # type: ignore[index] - _assert_model_or_val(dyn_convert, model, list[int], [123, 117, 432, 654]) + _assert_model(model, list[int], [123, 117, 432, 654]) assert model.index(432) == 2 @@ -1301,7 +1429,7 @@ def test_mimic_nested_list_operations( model[-1] = tuple(range(3)) - _assert_model_or_val(dyn_convert, model, list[int | list[int]], [123, 234, [0, 1, 2]]) + _assert_model(model, list[int | list[int]], [123, 234, [0, 1, 2]]) _assert_model_or_val(dyn_convert, model[0], int, 123) # type: ignore[index] _assert_model_or_val(dyn_convert, model[-1], list[int], [0, 1, 2]) # type: ignore[index] diff --git a/tests/hub/helpers/mocks.py b/tests/hub/helpers/mocks.py index b37d5984..40f2c771 100644 --- a/tests/hub/helpers/mocks.py +++ b/tests/hub/helpers/mocks.py @@ -13,7 +13,7 @@ from omnipy.api.protocols.public.engine import IsTaskRunnerEngine -class MockIsJobConfig(Protocol): +class IsMockJobConfig(Protocol): persist_outputs: bool = True restore_outputs: bool = False @@ -27,12 +27,12 @@ class MockJobConfig: class MockJobCreator: def __init__(self) -> None: self.engine: IsTaskRunnerEngine | None = None - self.config: MockIsJobConfig = MockJobConfig() + self.config: IsMockJobConfig = MockJobConfig() def set_engine(self, engine: IsTaskRunnerEngine) -> None: self.engine = engine - def set_config(self, config: MockIsJobConfig) -> None: + def set_config(self, config: IsMockJobConfig) -> None: self.config = config @@ -40,6 +40,27 @@ class MockJobCreator2(MockJobCreator): ... +class IsMockDataConfig(Protocol): + interactive_mode: bool = True + + +@dataclass +class MockDataConfig: + interactive_mode: bool = True + + +class MockDataClassCreator: + def __init__(self) -> None: + self.config: IsMockDataConfig = MockDataConfig() + + def set_config(self, config: IsMockDataConfig) -> None: + self.config = config + + +class MockDataClassCreator2(MockDataClassCreator): + ... + + class MockEngine(ABC): def __init__(self) -> None: config_cls = self.get_config_cls() diff --git a/tests/hub/test_root_log.py b/tests/hub/test_root_log.py index 242e046c..326035c0 100644 --- a/tests/hub/test_root_log.py +++ b/tests/hub/test_root_log.py @@ -20,7 +20,7 @@ from ..log.helpers.functions import assert_log_lines_from_stream -def _assert_root_log_config_default(root_log: RootLogConfig, dir_path: str): +def _assert_root_log_config_default(root_log: RootLogConfig, dir_path: Path): assert isinstance(root_log, RootLogConfig) assert isinstance(root_log.locale, (str, tuple)) @@ -31,7 +31,7 @@ def _assert_root_log_config_default(root_log: RootLogConfig, dir_path: str): assert root_log.stdout_log_min_level == logging.INFO assert root_log.stderr_log_min_level == logging.ERROR assert root_log.file_log_min_level == logging.WARNING - assert root_log.file_log_dir_path == os.path.join(dir_path, 'logs') + assert root_log.file_log_dir_path == str(dir_path / 'logs') def _log_record_for_level(level: int, datetime_obj: datetime | None = None): @@ -130,7 +130,7 @@ def _assert_root_log_objects( def test_root_log_config_default(teardown_rm_root_log_dir: Annotated[None, pytest.fixture]) -> None: - _assert_root_log_config_default(RootLogConfig(), str(Path.cwd())) + _assert_root_log_config_default(RootLogConfig(), Path.cwd()) def test_root_log_objects_default( @@ -140,7 +140,7 @@ def test_root_log_objects_default( def test_runtime_root_log_config( runtime: Annotated[IsRuntime, pytest.fixture], - tmp_dir_path: Annotated[str, pytest.fixture], + tmp_dir_path: Annotated[Path, pytest.fixture], ) -> None: assert isinstance(runtime.config.root_log, RootLogConfig) assert isinstance(runtime.objects.root_log, RootLogObjects) @@ -150,7 +150,7 @@ def test_runtime_root_log_config( def test_root_log_config_dependencies(runtime: Annotated[IsRuntime, pytest.fixture], - tmp_dir_path: Annotated[str, pytest.fixture]) -> None: + tmp_dir_path: Annotated[Path, pytest.fixture]) -> None: runtime.config.root_log.log_to_stdout = False runtime.config.root_log.log_to_stderr = False @@ -189,7 +189,7 @@ def test_root_log_config_dependencies(runtime: Annotated[IsRuntime, pytest.fixtu runtime.config.root_log.log_to_file = True runtime.config.root_log.file_log_min_level = logging.INFO - runtime.config.root_log.file_log_dir_path = str(Path(tmp_dir_path).joinpath('extra_level')) + runtime.config.root_log.file_log_dir_path = str(tmp_dir_path / 'extra_level') _assert_root_file_handler(runtime.objects.root_log.file_handler, runtime.config.root_log) diff --git a/tests/hub/test_runtime.py b/tests/hub/test_runtime.py index ec94019f..ef8aff22 100644 --- a/tests/hub/test_runtime.py +++ b/tests/hub/test_runtime.py @@ -8,12 +8,15 @@ ConfigPersistOutputsOptions, ConfigRestoreOutputsOptions, EngineChoice) -from omnipy.api.protocols.public.hub import IsRuntime +from omnipy.api.protocols.public.hub import IsRuntime, IsRuntimeConfig from omnipy.config.data import DataConfig -from omnipy.data.serializer import SerializerRegistry +from omnipy.data.data_class_creator import DataClassBase from omnipy.hub.runtime import RuntimeConfig, RuntimeObjects -from .helpers.mocks import (MockJobConfig, +from .helpers.mocks import (MockDataClassCreator, + MockDataClassCreator2, + MockDataConfig, + MockJobConfig, MockJobCreator, MockJobCreator2, MockLocalRunner, @@ -31,7 +34,7 @@ MockRunStateRegistry2) -def _assert_runtime_config_default(config: RuntimeConfig, dir_path: str): +def _assert_runtime_config_default(config: IsRuntimeConfig, dir_path: Path): from omnipy.config.engine import LocalRunnerConfig, PrefectEngineConfig from omnipy.config.job import JobConfig @@ -46,53 +49,78 @@ def _assert_runtime_config_default(config: RuntimeConfig, dir_path: str): ConfigRestoreOutputsOptions.DISABLED assert config.job.output_storage.protocol == \ ConfigOutputStorageProtocolOptions.LOCAL - assert config.job.output_storage.local.persist_data_dir_path == \ - os.path.join(dir_path, 'outputs') + assert config.job.output_storage.local.persist_data_dir_path == str(dir_path / 'outputs') assert config.job.output_storage.s3.persist_data_dir_path == os.path.join('omnipy', 'outputs') assert config.job.output_storage.s3.endpoint_url == '' assert config.job.output_storage.s3.bucket_name == '' assert config.job.output_storage.s3.access_key == '' assert config.job.output_storage.s3.secret_key == '' assert config.data.interactive_mode is True + assert config.data.dynamically_convert_elements_to_models is False assert config.data.terminal_size_columns == 80 assert config.data.terminal_size_lines == 24 assert config.engine == EngineChoice.LOCAL assert config.prefect.use_cached_results is False -def _assert_runtime_objects_default(objects: RuntimeObjects, config: RuntimeConfig): +def _assert_runtime_objects_default(objects: RuntimeObjects): from omnipy.compute.job import JobBase from omnipy.compute.job_creator import JobCreator + from omnipy.data.data_class_creator import DataClassBase, DataClassCreator + from omnipy.data.serializer import SerializerRegistry from omnipy.engine.local import LocalRunner + from omnipy.hub.root_log import RootLogObjects from omnipy.log.registry import RunStateRegistry from omnipy.modules.prefect.engine.prefect import PrefectEngine assert isinstance(objects.job_creator, JobCreator) assert objects.job_creator is JobBase.job_creator - # TODO: add level "objects.engine" ? + assert isinstance(objects.data_class_creator, DataClassCreator) + assert objects.data_class_creator is DataClassBase.data_class_creator + assert isinstance(objects.local, LocalRunner) assert isinstance(objects.prefect, PrefectEngine) - assert isinstance(objects.registry, RunStateRegistry) assert isinstance(objects.serializers, SerializerRegistry) + assert isinstance(objects.root_log, RootLogObjects) + assert isinstance(objects.waiting_for_terminal_repr, bool) + def test_config_default(teardown_rm_root_log_dir: Annotated[None, pytest.fixture]) -> None: - _assert_runtime_config_default(RuntimeConfig(), str(Path.cwd())) + _assert_runtime_config_default(RuntimeConfig(), Path.cwd()) def test_objects_default(teardown_rm_root_log_dir: Annotated[None, pytest.fixture]) -> None: - _assert_runtime_objects_default(RuntimeObjects(), RuntimeConfig()) + _assert_runtime_objects_default(RuntimeObjects()) -def test_default_config(runtime: Annotated[IsRuntime, pytest.fixture], - tmp_dir_path: Annotated[str, pytest.fixture]) -> None: +def test_default_runtime(runtime: Annotated[IsRuntime, pytest.fixture], + tmp_dir_path: Annotated[Path, pytest.fixture]) -> None: assert isinstance(runtime.config, RuntimeConfig) assert isinstance(runtime.objects, RuntimeObjects) _assert_runtime_config_default(runtime.config, tmp_dir_path) - _assert_runtime_objects_default(runtime.objects, runtime.config) + _assert_runtime_objects_default(runtime.objects) + + +def test_runtime_config_after_data_class_creator( + runtime_cls: Annotated[Type[IsRuntime], pytest.fixture]) -> None: + DataClassBase.data_class_creator.config.dynamically_convert_elements_to_models = True + DataClassBase.data_class_creator.config.terminal_size_columns = 100 + + runtime = runtime_cls() + + assert runtime.config.data.dynamically_convert_elements_to_models is True + assert runtime.config.data.terminal_size_columns == 100 + + runtime.config.reset_to_defaults() + + _assert_runtime_config_default(runtime.config, Path.cwd()) + + assert DataClassBase.data_class_creator.config.dynamically_convert_elements_to_models is False + assert DataClassBase.data_class_creator.config.terminal_size_columns == 80 def test_engines_subscribe_to_registry( @@ -286,3 +314,31 @@ def test_job_creator_subscribe_to_job_config( runtime.objects.job_creator = MockJobCreator2() assert runtime.config.job is mock_job_config_2 + + +def test_data_class_creator_subscribe_to_data_config( + runtime_cls: Annotated[Type[IsRuntime], pytest.fixture]) -> None: + mock_data_class_creator = MockDataClassCreator() + mock_data_config = MockDataConfig(interactive_mode=False) + runtime = runtime_cls( + objects=RuntimeObjects(data_class_creator=mock_data_class_creator), + config=RuntimeConfig(data=mock_data_config), + ) + assert runtime.objects.data_class_creator.config is runtime.config.data is mock_data_config + assert runtime.objects.data_class_creator.config.interactive_mode is False + + runtime.config.data.interactive_mode = True + assert runtime.objects.data_class_creator.config.interactive_mode is True + + mock_data_config_2 = MockDataConfig(interactive_mode=False) + assert mock_data_config_2 is not mock_data_config + runtime.config.data = mock_data_config_2 + assert runtime.objects.data_class_creator.config is runtime.config.data is mock_data_config_2 + assert runtime.objects.data_class_creator.config.interactive_mode is False + + runtime.objects.data_class_creator = MockDataClassCreator() + assert runtime.objects.data_class_creator.config is runtime.config.data is mock_data_config_2 + assert runtime.objects.data_class_creator.config.interactive_mode is False + + runtime.objects.data_class_creator = MockDataClassCreator2() + assert runtime.config.data is mock_data_config_2 diff --git a/tests/util/test_helpers.py b/tests/util/test_helpers.py index 87a63288..aa2acb5d 100644 --- a/tests/util/test_helpers.py +++ b/tests/util/test_helpers.py @@ -1,4 +1,6 @@ -from copy import copy +from collections import UserDict, UserList +from copy import copy, deepcopy +import gc from types import MethodType, NoneType, UnionType from typing import (Annotated, Any, @@ -16,6 +18,7 @@ import pytest from typing_inspect import get_generic_type +from omnipy.api.protocols.private.util import HasContents from omnipy.data.dataset import Dataset from omnipy.data.model import Model from omnipy.util.helpers import (all_type_variants, @@ -33,12 +36,16 @@ is_pure_pydantic_model, is_strict_subclass, is_union, + RefCountMemoDict, remove_annotated_plus_optional_if_present, - RestorableContents, - transfer_generic_args_to_cls) + Snapshot, + SnapshotHolder, + transfer_generic_args_to_cls, + WeakKeyRefContainer) T = TypeVar('T') U = TypeVar('U') +_ContentT = TypeVar('_ContentT', covariant=True, bound=object) class MyGenericDict(dict[T, U], Generic[T, U]): @@ -327,7 +334,7 @@ def test_is_optional() -> None: assert is_optional(Union[Union[str, None], int]) is True assert is_optional(Union[str, int] | None) is True # type: ignore[operator] - assert is_optional(Union[str, int] | NoneType) is True + assert is_optional(Union[str, int] | NoneType) is True # type: ignore[operator] assert is_optional(Union[str, NoneType] | int) is True assert is_optional(Union[str, None] | int) is True @@ -467,60 +474,326 @@ def test_remove_annotated_optional_if_present() -> None: 'something']) == Union[str, list[int]] -def test_restorable_contents(): - contents = RestorableContents() +def _assert_values_in_memo(memo: RefCountMemoDict, + all_ids: tuple[int, ...], + contained: tuple[bool, ...]) -> None: + assert len(memo) == sum(1 for _ in contained if _ is True) + for id_, is_contained in zip(all_ids, contained): + assert (id_ in memo) == is_contained - my_list = [1, 3, 5] - my_dict = {1: 2, 3: 4} - my_other_list = [my_list] - assert contents.has_snapshot() is False +def test_ref_count_memo_dict() -> None: + # Note: WeakValueDictionary cannot be used here, as most basic types do not support weak refs. - with pytest.raises(AssertionError): - contents.get_last_snapshot() + ref_count_memo_dict: RefCountMemoDict = RefCountMemoDict() - with pytest.raises(AssertionError): - contents.last_snapshot_taken_of_same_obj(my_list) + class MyClass: + def __init__(self, contents: object) -> None: + self.contents = contents + + def __del__(self) -> None: + print(f'__del__() called for {self}') + print(ref_count_memo_dict) + + self_id = id(self) + contents_id = id(self.contents) + del self + ref_count_memo_dict.recursively_remove_deleted_objs(self_id, contents_id) + + def __repr__(self) -> str: + return f'MyClass({self.contents})' + + def _outer_test_count_memo_dict() -> tuple: + def _inner_test_count_memo_dict() -> tuple[MyClass, tuple]: + a_list = [1, 2, 3] + b_tuple = ('I want my...', 'I want my...', 'I want my MTV') + c_dict = {1: 2, 3: a_list} + d_obj = MyClass({2: 4, 6: a_list}) + e_obj = MyClass({1: a_list, 2: d_obj}) + + id_a = id(a_list) + id_b = id(b_tuple) + id_c = id(c_dict) + id_d = id(d_obj) + id_d_c = id(d_obj.contents) + id_e = id(e_obj) + id_e_c = id(e_obj.contents) + all_ids = (id_a, id_b, id_c, id_d, id_d_c, id_e, id_e_c) + + ref_count_memo_dict[id_a] = a_list + ref_count_memo_dict[id_b] = b_tuple + ref_count_memo_dict[id_c] = c_dict + ref_count_memo_dict[id_d] = d_obj + ref_count_memo_dict[id_d_c] = d_obj.contents + ref_count_memo_dict[id_e] = e_obj + ref_count_memo_dict[id_e_c] = e_obj.contents + + assert ref_count_memo_dict[id_a] == a_list + assert ref_count_memo_dict[id_b] == b_tuple + assert ref_count_memo_dict[id_c] == c_dict + assert ref_count_memo_dict[id_d] == d_obj + assert ref_count_memo_dict[id_d_c] == d_obj.contents + assert ref_count_memo_dict[id_e] == e_obj + assert ref_count_memo_dict[id_e_c] == e_obj.contents + + _assert_values_in_memo(ref_count_memo_dict, + all_ids, (True, True, True, True, True, True, True)) + + del a_list + del b_tuple + + # a_list was deleted, but is still linked from c_dict and d_obj + ref_count_memo_dict.recursively_remove_deleted_objs(id_a) + _assert_values_in_memo(ref_count_memo_dict, + all_ids, (True, True, True, True, True, True, True)) + + print('Returning from _inner_test_count_memo_dict()') + + return e_obj, all_ids + + e_obj, all_ids = _inner_test_count_memo_dict() + id_a, id_b, id_c, id_d, id_d_c, id_e, id_e_c = all_ids + + # a_list is still linked from d_obj + # b_tuple was already deleted, but not checked for ref count until now + # c_dict was deleted when out of scope of_inner_test_count_memo_dict() and not linked from + # anywhere else + # d_obj was deleted when out of scope of_inner_test_count_memo_dict(), but e_obj still + # references it. + ref_count_memo_dict.recursively_remove_deleted_objs(id_b, id_c) + _assert_values_in_memo(ref_count_memo_dict, + all_ids, (True, False, False, True, True, True, True)) + + print('Returning from _outer_test_count_memo_dict()') + return all_ids + + all_ids = _outer_test_count_memo_dict() + + # e_obj is deleted when out of scope of _outer_test_count_memo_dict(), and d_obj.__del__() + # method calls recursively_remove_deleted_objs(id_e). No more references left to a, b, c, d, and e. + # d_obj was fully deleted when e_obj is deleted. + _assert_values_in_memo(ref_count_memo_dict, + all_ids, (False, False, False, False, False, False, False)) + + +class HasContentsMixin(Generic[_ContentT]): + @property + def contents(self) -> _ContentT: + return self.data # type: ignore[attr-defined] + + # + # def __deepcopy__(self, memo=None): + # print(f'__deepcopy__() called for {id(self)}: {self}') + # ret = self.__class__(deepcopy(self.data, memo)) + # # memo[id(self)] = ret + # print(f' memo: {memo}, {type(memo)}') + # return ret + + def __repr__(self) -> str: + return f'{self.__class__.__name__}({self.contents})' + + +class MyList(HasContentsMixin[list], UserList): + ... - with pytest.raises(AssertionError): - contents.differs_from_last_snapshot(my_list) - contents.take_snapshot(my_list) - assert contents.has_snapshot() is True - assert contents.last_snapshot_taken_of_same_obj(my_list) is True - assert contents.differs_from_last_snapshot(my_list) is False +class MyDict(HasContentsMixin[dict], UserDict): + ... + + +def test_weak_key_ref_container() -> None: + class SomeObject: + def __init__(self, name: str) -> None: + self.name = name + + weak_key_ref_container = WeakKeyRefContainer[MyList | MyDict, SomeObject]() + + a_list = MyList([1, 3, 5]) + + assert a_list not in weak_key_ref_container + assert weak_key_ref_container.get(a_list) is None + + with pytest.raises(KeyError): + weak_key_ref_container[a_list] + + assert len(weak_key_ref_container) == 0 + + weak_key_ref_container[a_list] = SomeObject('a_list') + + assert a_list in weak_key_ref_container + assert weak_key_ref_container.get(a_list) is weak_key_ref_container[a_list] + assert isinstance(weak_key_ref_container[a_list], SomeObject) + some_object_a = weak_key_ref_container[a_list] + assert some_object_a.name == 'a_list' + + b_list = MyList([a_list]) + a_list.append(7) + assert a_list == MyList([1, 3, 5, 7]) + assert b_list == MyList([[1, 3, 5, 7]]) + + b_dict = MyDict({1: 2, 3: 4}) + + assert b_dict not in weak_key_ref_container + weak_key_ref_container[b_dict] = SomeObject('b_dict') + assert isinstance(weak_key_ref_container[b_dict], SomeObject) + some_object_b = weak_key_ref_container[b_dict] + assert some_object_b.name == 'b_dict' - assert contents.last_snapshot_taken_of_same_obj(copy(my_list)) is False - assert contents.differs_from_last_snapshot(copy(my_list)) is False + assert len(weak_key_ref_container) == 2 + del b_dict + assert len(weak_key_ref_container) == 1 + + assert a_list in weak_key_ref_container + del a_list + assert len(weak_key_ref_container) == 1 + + a_list = b_list[0] + assert a_list in weak_key_ref_container + + del a_list + del b_list + assert len(weak_key_ref_container) == 0 + + +def test_snapshots() -> None: + snapshot_holder = SnapshotHolder[MyList | MyDict, list | dict]() + + my_list = MyList([1, 3, 5]) + assert my_list.contents == [1, 3, 5] + + assert my_list not in snapshot_holder + assert snapshot_holder.get(my_list) is None + + with pytest.raises(KeyError): + snapshot_holder[my_list] + + with pytest.raises(TypeError): + snapshot_holder[my_list] = my_list # type: ignore[assignment] + + assert len(snapshot_holder) == 0 + + snapshot_holder.take_snapshot(my_list) + + assert my_list in snapshot_holder + assert snapshot_holder.get(my_list) is snapshot_holder[my_list] + assert isinstance(snapshot_holder[my_list], Snapshot) + + snapshot = snapshot_holder[my_list] + assert snapshot.id == id(my_list) + assert snapshot.taken_of_same_obj(my_list) is True + + assert snapshot.obj_copy == my_list.contents + assert snapshot.differs_from(my_list) is False + + assert id(snapshot.obj_copy) != id(my_list) + + assert snapshot.taken_of_same_obj(copy(my_list)) is False + assert snapshot.differs_from(copy(my_list)) is False + + my_other_list = MyList([my_list]) my_list.append(7) - assert my_list == [1, 3, 5, 7] - assert my_other_list == [[1, 3, 5, 7]] - assert contents.last_snapshot_taken_of_same_obj(my_list) is True - assert contents.differs_from_last_snapshot(my_list) is True + assert snapshot.taken_of_same_obj(my_list) is True + assert snapshot.differs_from(my_list) is True + assert my_list == MyList([1, 3, 5, 7]) + assert my_other_list == MyList([[1, 3, 5, 7]]) + + my_list_from_snapshot = snapshot.obj_copy + assert my_list_from_snapshot == [1, 3, 5] - my_old_list = my_list - my_list = contents.get_last_snapshot() - assert my_list == [1, 3, 5] - assert my_old_list == [1, 3, 5, 7] - assert my_other_list == [[1, 3, 5, 7]] + # the snapshot is a (preferably deep) copy of the old object + assert my_list_from_snapshot is not my_list.contents - assert my_list is not my_old_list # the snapshot is a (preferably deep) copy of the old object - assert contents.last_snapshot_taken_of_same_obj(my_list) is False - assert contents.differs_from_last_snapshot(my_list) is False + my_dict = MyDict({1: 2, 3: 4}) - my_dict[5] = my_list - assert my_dict == {1: 2, 3: 4, 5: [1, 3, 5]} + my_dict[5] = my_list_from_snapshot + assert my_dict == MyDict({1: 2, 3: 4, 5: [1, 3, 5]}) - assert contents.last_snapshot_taken_of_same_obj(my_dict) is False - contents.take_snapshot(my_dict) - assert contents.last_snapshot_taken_of_same_obj(my_dict) is True - assert contents.differs_from_last_snapshot(my_dict) is False + assert my_dict not in snapshot_holder + snapshot_holder.take_snapshot(my_dict) + assert snapshot_holder[my_dict].taken_of_same_obj(my_dict) is True + assert snapshot_holder[my_dict].differs_from(my_dict) is False del my_dict[5] - assert my_dict == {1: 2, 3: 4} - assert contents.last_snapshot_taken_of_same_obj(my_dict) is True - assert contents.differs_from_last_snapshot(my_dict) is True + assert my_dict == MyDict({1: 2, 3: 4}) + assert snapshot_holder[my_dict].taken_of_same_obj(my_dict) is True + assert snapshot_holder[my_dict].differs_from(my_dict) is True + + assert len(snapshot_holder) == 2 + + del my_dict + assert len(snapshot_holder) == 1 + + del my_list + assert len(snapshot_holder) == 1 + + my_list = my_other_list[0] + assert my_list in snapshot_holder + + del my_list + del my_other_list + assert len(snapshot_holder) == 0 + + +def test_snapshot_deepcopy_reuse_objects() -> None: + snapshot_holder = SnapshotHolder['MyMemoDeletingList', list]() + + class MyMemoDeletingList(MyList): + def __del__(self) -> None: + if snapshot_holder is not None: + content_id = id(self.contents) + self.data = [] + try: + snapshot_holder.recursively_remove_deleted_obj_from_deepcopy_memo(content_id) + except (AttributeError) as exp: + print(exp) + print(snapshot_holder._deepcopy_memo) + + def _inner_test_snapshot_deepcopy_reuse_objects( + snapshot_holder: SnapshotHolder[MyMemoDeletingList, list]) -> None: + + inner = MyMemoDeletingList([2, 4]) + middle = MyMemoDeletingList([1, 3, inner]) + outer = MyMemoDeletingList([0, middle, 5]) + + snapshot_holder.take_snapshot(outer) + snapshot_holder.take_snapshot(middle) + snapshot_holder.take_snapshot(inner) + + assert type(outer[1][-1]) == type(middle[-1]) == type(inner) == MyMemoDeletingList + assert id(outer[1][-1]) == id(middle[-1]) == id(inner) + + assert type(snapshot_holder[outer].obj_copy[1][-1]) \ + == type(snapshot_holder[middle].obj_copy[-1]) \ + == MyMemoDeletingList + assert id(snapshot_holder[outer].obj_copy[1][-1]) \ + == id(snapshot_holder[middle].obj_copy[-1]) + + assert type(outer[1][-1].contents) == type(middle[-1].contents) == type( + inner.contents) == list + assert id(outer[1][-1].contents) == id(middle[-1].contents) == id(inner.contents) + + assert type(snapshot_holder[outer].obj_copy[1][-1].contents) \ + == type(snapshot_holder[middle].obj_copy[-1].contents) \ + == type(snapshot_holder[inner].obj_copy) \ + == list + assert id(snapshot_holder[outer].obj_copy[1][-1].contents) \ + == id(snapshot_holder[middle].obj_copy[-1].contents) \ + == id(snapshot_holder[inner].obj_copy) + + assert type(outer[1].contents) == type(middle.contents) == list + assert id(outer[1].contents) == id(middle.contents) + + assert type(snapshot_holder[outer].obj_copy[1].contents) \ + == type(snapshot_holder[middle].obj_copy) \ + == list + assert id(snapshot_holder[outer].obj_copy[1].contents) \ + == id(snapshot_holder[middle].obj_copy) + + # snapshot_holder = SnapshotHolder[MyMemoDeletingList, list]() + _inner_test_snapshot_deepcopy_reuse_objects(snapshot_holder) + gc.collect() + assert len(snapshot_holder._deepcopy_memo) == 0 def test_get_calling_module_name() -> None: