fix(api): refactor and fix failed tests (#1646)

antarest/study/common/studystorage.py

@@ -256,3 +256,15 @@ def unarchive_study_output(self, study: T, output_id: str, keep_src_zip: bool) -
 
     def unarchive(self, study: T) -> None:
         raise NotImplementedError()
+
+    # def export_study_flat(self, **kwargs) -> None:
+    #     raise NotImplementedError()
+    def export_study_flat(
+        self,
+        path_study: Path,
+        dst_path: Path,
+        outputs: bool = True,
+        output_src_path: Optional[Path] = None,
+        output_list_filter: Optional[List[str]] = None,
+    ) -> None:
+        raise NotImplementedError()

antarest/study/service.py

@@ -1033,26 +1033,26 @@ def export_study_flat(
         assert_permission(params.user, study, StudyPermissionType.READ)
         self._assert_study_unarchived(study)
         path_study = Path(study.path)
+        storage = self.storage_service.get_storage(study)
         if isinstance(study, RawStudy):
             if study.archived:
-                self.storage_service.get_storage(study).unarchive(study)
+                storage.unarchive(study)
             try:
-                return export_study_flat(
+                return storage.export_study_flat(
                     path_study=path_study,
-                    study_factory=study,
-                    dest=dest,
+                    dst_path=dest,
                     outputs=len(output_list or []) > 0,
                     output_list_filter=output_list,
+                    output_src_path=None,
                 )
             finally:
                 if study.archived:
                     shutil.rmtree(study.path)
         snapshot_path = path_study / "snapshot"
         output_src_path = path_study / "output"
-        export_study_flat(
+        return storage.export_study_flat(
             path_study=snapshot_path,
-            study_factory=study,
-            dest=dest,
+            dst_path=dest,
             outputs=len(output_list or []) > 0,
             output_list_filter=output_list,
             output_src_path=output_src_path,

antarest/study/storage/abstract_storage_service.py

@@ -39,7 +39,6 @@
 def export_study_flat(
     path_study: Path,
     dest: Path,
-    study_factory: StudyFactory,
     outputs: bool = True,
     output_list_filter: Optional[List[str]] = None,
     output_src_path: Optional[Path] = None,
@@ -91,11 +90,6 @@ def export_study_flat(
     duration = "{:.3f}".format(stop_time - start_time)
     logger.info(f"Study {path_study} exported (flat mode) in {duration}s")
 
-    study = study_factory.create_from_fs(dest, "", use_cache=False)
-    study.tree.denormalize()
-    duration = "{:.3f}".format(time.time() - stop_time)
-    logger.info(f"Study {path_study} denormalized in {duration}s")
-
 
 class AbstractStorageService(IStudyStorageService[T], ABC):
     def __init__(
@@ -295,19 +289,25 @@ def export_study(self, metadata: T, target: Path, outputs: bool = True) -> Path:
         with tempfile.TemporaryDirectory(dir=self.config.storage.tmp_dir) as tmpdir:
             logger.info(f"Exporting study {metadata.id} to temporary path {tmpdir}")
             tmp_study_path = Path(tmpdir) / "tmp_copy"
-            if not isinstance(metadata, RawStudy):
+            if isinstance(metadata, RawStudy):
+                export_study_flat(
+                    path_study=path_study,
+                    dest=tmp_study_path,
+                    outputs=outputs,
+                )
+            else:
                 snapshot_path = path_study / "snapshot"
                 output_src_path = path_study / "output"
                 export_study_flat(
                     path_study=snapshot_path,
-                    study_factory=self.study_factory,
                     dest=tmp_study_path,
                     outputs=outputs,
                     output_src_path=output_src_path,
                 )
-            export_study_flat(
-                path_study, tmp_study_path, self.study_factory, outputs
+            study = self.study_factory.create_from_fs(
+                tmp_study_path, "", use_cache=False
             )
+            study.tree.denormalize()
             stopwatch = StopWatch()
             zip_dir(tmp_study_path, target)
             stopwatch.log_elapsed(lambda x: logger.info(f"Study {path_study} exported (zipped mode) in {x}s"))

antarest/study/storage/rawstudy/raw_study_service.py

@@ -16,7 +16,7 @@
 from antarest.core.requests import RequestParameters
 from antarest.core.utils.utils import extract_zip
 from antarest.study.model import DEFAULT_WORKSPACE_NAME, Patch, RawStudy, Study, StudyAdditionalData
-from antarest.study.storage.abstract_storage_service import AbstractStorageService
+from antarest.study.storage.abstract_storage_service import AbstractStorageService, export_study_flat
 from antarest.study.storage.patch_service import PatchService
 from antarest.study.storage.rawstudy.model.filesystem.config.model import FileStudyTreeConfig, FileStudyTreeConfigDTO
 from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy, StudyFactory
@@ -322,6 +322,26 @@ def import_study(self, metadata: RawStudy, stream: IO[bytes]) -> Study:
         metadata.path = str(path_study)
         return metadata
 
+    def export_study_flat(
+        self,
+        path_study: Path,
+        dst_path: Path,
+        outputs: bool = True,
+        output_src_path: Optional[Path] = None,
+        output_list_filter: Optional[List[str]] = None,
+    ) -> None:
+        export_study_flat(
+            path_study=path_study,
+            dest=dst_path,
+            outputs=outputs,
+            output_list_filter=output_list_filter,
+            output_src_path=output_src_path,
+        )
+        study = self.study_factory.create_from_fs(
+            dst_path, "", use_cache=False
+        )
+        study.tree.denormalize()
+
     def check_errors(
         self,
         metadata: RawStudy,

antarest/study/storage/variantstudy/variant_study_service.py

@@ -719,14 +719,28 @@ def _generate(
 
         if last_executed_command_index is None:
             if isinstance(parent_study, VariantStudy):
+                #     self._safe_generation(parent_study)
+                #     self.export_study_flat(
+                #         metadata=parent_study,
+                #         dst_path=dst_path,
+                #         outputs=False,
+                #         denormalize=False,
+                #     )
+                # else:
+                #     self.raw_study_service.export_study_flat(
+                #         metadata=parent_study,
+                #         dst_path=dst_path,
+                #         outputs=False,
+                #         denormalize=False,
+                #     )
+
                 self._safe_generation(parent_study)
                 path_study = Path(parent_study.path)
                 snapshot_path = path_study / SNAPSHOT_RELATIVE_PATH
                 output_src_path = path_study / "output"
-                export_study_flat(
-                    snapshot_path,
-                    dst_path,
-                    self.study_factory,
+                self.export_study_flat(
+                    path_study=snapshot_path,
+                    dst_path=dst_path,
                     outputs=False,
                     output_src_path=output_src_path,
                 )
@@ -735,10 +749,9 @@ def _generate(
                 if parent_study.archived:
                     self.raw_study_service.unarchive(parent_study)
                 try:
-                    export_study_flat(
+                    self.raw_study_service.export_study_flat(
                         path_study=path_study,
-                        dest=dst_path,
-                        study_factory=self.study_factory,
+                        dst_path=dst_path,
                         outputs=False,
                     )
                 finally:
@@ -1077,6 +1090,26 @@ def get_study_path(self, metadata: Study) -> Path:
         """
         return Path(metadata.path) / SNAPSHOT_RELATIVE_PATH
 
+    def export_study_flat(
+        self,
+        path_study: Path,
+        dst_path: Path,
+        outputs: bool = True,
+        output_src_path: Optional[Path] = None,
+        output_list_filter: Optional[List[str]] = None,
+    ) -> None:
+        export_study_flat(
+            path_study=path_study,
+            dest=dst_path,
+            outputs=outputs,
+            output_src_path=output_src_path,
+            output_list_filter=output_list_filter,
+        )
+        study = self.study_factory.create_from_fs(
+            dst_path, "", use_cache=False
+        )
+        study.tree.denormalize()
+
     def get_synthesis(
         self,
         metadata: VariantStudy,

tests/storage/business/test_export.py

@@ -5,10 +5,10 @@
 import pytest
 from checksumdir import dirhash
 
-from antarest.core.config import Config, StorageConfig
+from antarest.core.config import Config
 from antarest.study.model import DEFAULT_WORKSPACE_NAME, RawStudy
-from antarest.study.storage.rawstudy.raw_study_service import RawStudyService
 from antarest.study.storage.abstract_storage_service import export_study_flat
+from antarest.study.storage.rawstudy.raw_study_service import RawStudyService
 
 
 @pytest.mark.unit_test
@@ -92,29 +92,15 @@ def test_export_flat(tmp_path: Path):
     root_hash = dirhash(root, "md5")
     root_without_output_hash = dirhash(root_without_output, "md5")
 
-    study_factory = Mock()
-
-    study_service = RawStudyService(
-        config=Config(storage=StorageConfig(tmp_dir=tmp_path)),
-        study_factory=study_factory,
-        path_resources=Mock(),
-        patch_service=Mock(),
-        cache=Mock(),
-    )
-    study_tree = Mock()
-    study_factory.create_from_fs.return_value = study_tree
-
-    study = RawStudy(id="id", path=root)
-    path_study = Path(study.path)
+    path_study = root
 
-    export_study_flat(path_study, tmp_path / "copy_with_output", study_factory, outputs=True)
+    export_study_flat(path_study, tmp_path / "copy_with_output", outputs=True)
     copy_with_output_hash = dirhash(tmp_path / "copy_with_output", "md5")
     assert root_hash == copy_with_output_hash
 
     export_study_flat(
         path_study,
         tmp_path / "copy_without_output",
-        study_factory,
         outputs=False,
     )
     copy_without_output_hash = dirhash(tmp_path / "copy_without_output", "md5")

tests/storage/integration/test_exporter.py

@@ -16,7 +16,7 @@
 from antarest.matrixstore.service import MatrixService
 from antarest.study.main import build_study_service
 from antarest.study.model import DEFAULT_WORKSPACE_NAME, RawStudy
-from antarest.study.storage.utils import export_study_flat
+from antarest.study.storage.abstract_storage_service import export_study_flat
 from antarest.study.storage.variantstudy.business.matrix_constants_generator import GeneratorMatrixConstants
 from tests.storage.conftest import SimpleFileTransferManager, SimpleSyncTaskService
 
@@ -118,7 +118,6 @@ def test_export_flat(
     export_study_flat(
         path_studies / "STA-mini",
         export_path / "STA-mini-export",
-        Mock(),
         outputs,
         output_list,
     )

tests/variantstudy/model/test_variant_model.py

@@ -1,6 +1,6 @@
 import datetime
 from pathlib import Path
-from unittest.mock import ANY, Mock, patch
+from unittest.mock import ANY, Mock
 
 from sqlalchemy import create_engine
 
@@ -13,7 +13,6 @@
 from antarest.core.utils.fastapi_sqlalchemy import DBSessionMiddleware, db
 from antarest.study.model import DEFAULT_WORKSPACE_NAME, RawStudy, StudyAdditionalData
 from antarest.study.storage.variantstudy.command_factory import CommandFactory
-from antarest.study.storage.variantstudy.model.dbmodel import VariantStudy
 from antarest.study.storage.variantstudy.model.model import CommandDTO, GenerationResultInfoDTO
 from antarest.study.storage.variantstudy.repository import VariantStudyRepository
 from antarest.study.storage.variantstudy.variant_study_service import SNAPSHOT_RELATIVE_PATH, VariantStudyService
@@ -29,12 +28,7 @@
 )
 
 
-@patch(
-    "antarest.study.storage.variantstudy.variant_study_service.export_study_flat"
-)
-def test_commands_service(
-    mock_export, tmp_path: Path, command_factory: CommandFactory
-):
+def test_commands_service(tmp_path: Path, command_factory: CommandFactory):
     engine = create_engine(
         "sqlite:///:memory:",
         echo=False,
@@ -47,7 +41,6 @@ def test_commands_service(
         custom_engine=engine,
         session_args={"autocommit": False, "autoflush": False},
     )
-
     repository = VariantStudyRepository(LocalCache())
     service = VariantStudyService(
         raw_study_service=Mock(),
@@ -145,8 +138,7 @@ def test_commands_service(
         assert study.snapshot.id == study.id
 
 
-@patch("antarest.study.storage.variantstudy.variant_study_service.export_study_flat")
-def test_smart_generation(mock_export, tmp_path: Path, command_factory: CommandFactory) -> None:
+def test_smart_generation(tmp_path: Path, command_factory: CommandFactory) -> None:
     engine = create_engine(
         "sqlite:///:memory:",
         echo=False,
@@ -180,18 +172,13 @@ def test_smart_generation(mock_export, tmp_path: Path, command_factory: CommandF
     ]
 
     # noinspection PyUnusedLocal
-    def export_flat(
-        path_study: Path,
-        dest: Path,
-        study_factory: VariantStudy,
-        outputs: bool = True,
-        denormalize: bool = True,
-    ) -> None:
-        dest.mkdir(parents=True)
-        (dest / "user").mkdir()
-        (dest / "user" / "some_unmanaged_config").touch()
-
-    mock_export.side_effect = export_flat
+    def export_flat(path_study: Path, dst_path: Path, outputs: bool = True) -> None:
+        dst_path.mkdir(parents=True)
+        (dst_path / "user").mkdir()
+        (dst_path / "user" / "some_unmanaged_config").touch()
+
+    service.raw_study_service.export_study_flat.side_effect = export_flat
+
     with db():
         origin_id = "base-study"
         # noinspection PyArgumentList