From b4209aae7fe626cc12c78437cdd4654dac57e68c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20=C5=98ehounek?= Date: Tue, 13 Aug 2024 11:42:25 +0200 Subject: [PATCH] Remove progress, make continuous logs visible --- .../commands/migrate/helpers.py | 22 ++-- .../commands/migrate/hooks.py | 15 +-- .../commands/migrate/migrate.py | 104 +++++++++--------- .../commands/migrate/organization.py | 10 +- .../commands/migrate/schemas.py | 5 - .../commands/migrate/upload_helpers.py | 62 +++++++---- .../commands/migrate/workspaces.py | 26 ++--- 7 files changed, 112 insertions(+), 132 deletions(-) diff --git a/project_rossum_deploy/commands/migrate/helpers.py b/project_rossum_deploy/commands/migrate/helpers.py index f495c40..bf4c354 100644 --- a/project_rossum_deploy/commands/migrate/helpers.py +++ b/project_rossum_deploy/commands/migrate/helpers.py @@ -1,7 +1,6 @@ import asyncio import copy from typing import Callable -from rich.progress import Progress from rossum_api import ElisAPIClient from rossum_api.api_client import Resource @@ -11,7 +10,6 @@ MAPPING_SELECTED_ATTRIBUTE, ) from project_rossum_deploy.utils.functions import ( - PauseProgress, extract_id_from_url, ) from project_rossum_deploy.utils.functions import find_object_by_id @@ -52,7 +50,6 @@ def replace_dependency_url( dependency: str, source_id_target_pairs: dict[int, list], target_object: dict = None, - progress: Progress = None, ): if isinstance(object[dependency], list): replace_list_of_dependency_urls( @@ -62,7 +59,6 @@ def replace_dependency_url( dependency=dependency, source_id_target_pairs=source_id_target_pairs, target_object=target_object, - progress=progress, ) else: source_dependency_url = object[dependency] @@ -87,10 +83,9 @@ def replace_dependency_url( if settings.IS_PROJECT_IN_SAME_ORG and dependency == "organization": return - with PauseProgress(progress): - display_warning( - f'Dependency "{dependency}" for object "{object.get('id', 'no-ID')}" was not modified. Source and target objects share the dependency. This can happen if you did not {settings.MIGRATE_COMMAND_NAME} the dependency and no target equivalent exists.' - ) + display_warning( + f'Dependency "{dependency}" for object "{object.get('id', 'no-ID')}" was not modified. Source and target objects share the dependency. This can happen if you did not {settings.MIGRATE_COMMAND_NAME} the dependency and no target equivalent exists.' + ) # TODO: refactor to use the same replace URL function @@ -101,7 +96,6 @@ def replace_list_of_dependency_urls( dependency: str, source_id_target_pairs: dict[int, list], target_object: dict = None, - progress: Progress = None, ): new_urls = [] for source_index, source_dependency_url in enumerate(object[dependency]): @@ -121,10 +115,9 @@ def replace_list_of_dependency_urls( new_urls.append(new_url) if source_id_str == target_id_str: - with PauseProgress(progress): - display_warning( - f'Dependency "{dependency}"[{source_index}] for object "{object.get('id', 'no-ID')}" was not modified. Source and target objects share the dependency. This can happen if you did not {settings.MIGRATE_COMMAND_NAME} the dependency and no target equivalent exists.' - ) + display_warning( + f'Dependency "{dependency}"[{source_index}] for object "{object.get('id', 'no-ID')}" was not modified. Source and target objects share the dependency. This can happen if you did not {settings.MIGRATE_COMMAND_NAME} the dependency and no target equivalent exists.' + ) # Target queues can have 'dangling' hooks that exist only on target, these should not be overwritten. if target_object: @@ -207,6 +200,9 @@ async def migrate_object_to_multiple_targets( return list(filter(lambda x: x, results)) +# TODO: add separate function for ignoring that will log and return + + # Extra args are there to accomodate all upload function signatures async def simulate_migrate_object( client: ElisAPIClient, diff --git a/project_rossum_deploy/commands/migrate/hooks.py b/project_rossum_deploy/commands/migrate/hooks.py index 9b22153..733f6b5 100644 --- a/project_rossum_deploy/commands/migrate/hooks.py +++ b/project_rossum_deploy/commands/migrate/hooks.py @@ -1,6 +1,5 @@ import asyncio import functools -from rich.progress import Progress from anyio import Path from rossum_api import ElisAPIClient from rossum_api.api_client import Resource @@ -23,7 +22,6 @@ ) from project_rossum_deploy.commands.migrate.upload_helpers import upload_hook from project_rossum_deploy.utils.functions import ( - PauseProgress, detemplatize_name_id, extract_id_from_url, ) @@ -35,7 +33,6 @@ async def migrate_hooks( mapping: dict, source_id_target_pairs: dict[int, list], sources_by_source_id_map: dict, - progress: Progress, plan_only: bool = False, selected_only: bool = False, target_objects: list[dict] = [], @@ -43,22 +40,19 @@ async def migrate_hooks( force: bool = False, ): hook_paths = [hook_path async for hook_path in (source_path / "hooks").iterdir()] - task = progress.add_task("Releasing hooks.", total=len(hook_paths)) target_token_owner_id = "" if not settings.IS_PROJECT_IN_SAME_ORG: target_org_token_owner = await get_token_owner(client) if not target_org_token_owner: - with PauseProgress(progress): - target_token_owner_id = Prompt.ask( - "Please input user ID of the hook token owner (e.g., 938382)" - ) + target_token_owner_id = Prompt.ask( + "Please input user ID of the hook token owner (e.g., 938382)" + ) else: target_token_owner_id = target_org_token_owner.id async def migrate_hook(hook_path: Path): if hook_path.suffix != ".json": - progress.update(task, advance=1) return try: @@ -96,7 +90,6 @@ async def migrate_hook(hook_path: Path): client=client, hook=hook, hook_mapping=hook_mapping, - progress=progress, target_objects=target_objects, errors=errors, force=force, @@ -113,8 +106,6 @@ async def migrate_hook(hook_path: Path): plan_only=plan_only, ) source_id_target_pairs[id].extend(results) - - progress.update(task, advance=1) except PrdVersionException as e: raise e except Exception as e: diff --git a/project_rossum_deploy/commands/migrate/migrate.py b/project_rossum_deploy/commands/migrate/migrate.py index ef8686f..4677e96 100644 --- a/project_rossum_deploy/commands/migrate/migrate.py +++ b/project_rossum_deploy/commands/migrate/migrate.py @@ -2,7 +2,6 @@ from anyio import Path from rich import print from rich.panel import Panel -from rich.progress import Progress import click from rossum_api import ElisAPIClient from project_rossum_deploy.commands.download.download import ( @@ -167,61 +166,56 @@ async def migrate_project( target_paths = await find_all_object_paths(org_path / settings.TARGET_DIRNAME) target_objects = [await read_json(path) for path in target_paths] - with Progress() as progress: - await migrate_organization( - source_path=source_path, - client=client, - mapping=mapping, - source_id_target_pairs=source_id_target_pairs, - sources_by_source_id_map=sources_by_source_id_map, - target_organization_id=target_organization_id, - progress=progress, - plan_only=plan_only, - selected_only=selected_only, - target_objects=target_objects, - errors=errors_by_target_id, - force=force, - ) + await migrate_organization( + source_path=source_path, + client=client, + mapping=mapping, + source_id_target_pairs=source_id_target_pairs, + sources_by_source_id_map=sources_by_source_id_map, + target_organization_id=target_organization_id, + plan_only=plan_only, + selected_only=selected_only, + target_objects=target_objects, + errors=errors_by_target_id, + force=force, + ) - await migrate_schemas( - source_path=source_path, - client=client, - mapping=mapping, - source_id_target_pairs=source_id_target_pairs, - sources_by_source_id_map=sources_by_source_id_map, - progress=progress, - plan_only=plan_only, - selected_only=selected_only, - target_objects=target_objects, - errors=errors_by_target_id, - force=force, - ) - await migrate_hooks( - source_path=source_path, - client=client, - mapping=mapping, - source_id_target_pairs=source_id_target_pairs, - sources_by_source_id_map=sources_by_source_id_map, - progress=progress, - plan_only=plan_only, - selected_only=selected_only, - target_objects=target_objects, - errors=errors_by_target_id, - force=force, - ) - await migrate_workspaces( - source_path=source_path, - client=client, - mapping=mapping, - source_id_target_pairs=source_id_target_pairs, - sources_by_source_id_map=sources_by_source_id_map, - progress=progress, - plan_only=plan_only, - selected_only=selected_only, - target_objects=target_objects, - errors=errors_by_target_id, - force=force, - ) + await migrate_schemas( + source_path=source_path, + client=client, + mapping=mapping, + source_id_target_pairs=source_id_target_pairs, + sources_by_source_id_map=sources_by_source_id_map, + plan_only=plan_only, + selected_only=selected_only, + target_objects=target_objects, + errors=errors_by_target_id, + force=force, + ) + await migrate_hooks( + source_path=source_path, + client=client, + mapping=mapping, + source_id_target_pairs=source_id_target_pairs, + sources_by_source_id_map=sources_by_source_id_map, + plan_only=plan_only, + selected_only=selected_only, + target_objects=target_objects, + errors=errors_by_target_id, + force=force, + ) + await migrate_workspaces( + source_path=source_path, + client=client, + mapping=mapping, + source_id_target_pairs=source_id_target_pairs, + sources_by_source_id_map=sources_by_source_id_map, + plan_only=plan_only, + selected_only=selected_only, + target_objects=target_objects, + errors=errors_by_target_id, + force=force, + ) if not plan_only: # Update the mapping with right hand sides (targets) created during migration diff --git a/project_rossum_deploy/commands/migrate/organization.py b/project_rossum_deploy/commands/migrate/organization.py index 045c290..d1e083d 100644 --- a/project_rossum_deploy/commands/migrate/organization.py +++ b/project_rossum_deploy/commands/migrate/organization.py @@ -1,4 +1,3 @@ -from rich.progress import Progress from anyio import Path from rossum_api import ElisAPIClient from rich import print @@ -29,15 +28,12 @@ async def migrate_organization( source_id_target_pairs: dict[int, list], sources_by_source_id_map: dict, target_organization_id: int, - progress: Progress, plan_only: bool = False, selected_only: bool = False, target_objects: list[dict] = [], errors: dict = {}, force: bool = False, ): - task = progress.add_task("Releasing organization.", total=1) - try: organization = await read_json(source_path / "organization.json") sources_by_source_id_map[organization["id"]] = organization @@ -58,7 +54,6 @@ async def migrate_organization( ] if selected_only and not check_if_selected(mapping["organization"]): - progress.update(task, advance=1) return if plan_only: @@ -77,7 +72,10 @@ async def migrate_organization( force=force, ) ] - progress.update(task, advance=1) + + print( + f'Released organization "{organization['id']}" -> "{target_organization_id}".' + ) except PrdVersionException as e: raise e except MissingTargetOrganizationException as e: diff --git a/project_rossum_deploy/commands/migrate/schemas.py b/project_rossum_deploy/commands/migrate/schemas.py index a0a2e20..8f351bd 100644 --- a/project_rossum_deploy/commands/migrate/schemas.py +++ b/project_rossum_deploy/commands/migrate/schemas.py @@ -4,7 +4,6 @@ from anyio import Path from rossum_api import ElisAPIClient -from rich.progress import Progress from rich.panel import Panel from rich import print from rossum_api.api_client import Resource @@ -34,7 +33,6 @@ async def migrate_schemas( mapping: dict, source_id_target_pairs: dict[int, list], sources_by_source_id_map: dict, - progress: Progress, plan_only: bool = False, selected_only: bool = False, target_objects: list[dict] = [], @@ -44,11 +42,9 @@ async def migrate_schemas( schema_paths = [ schema_path async for schema_path in (source_path / "schemas").iterdir() ] - task = progress.add_task("Releasing schemas.", total=len(schema_paths)) async def migrate_schema(schema_path: Path): if schema_path.suffix != ".json": - progress.update(task, advance=1) return try: @@ -98,7 +94,6 @@ async def migrate_schema(schema_path: Path): ) source_id_target_pairs[id].extend(results) - progress.update(task, advance=1) except PrdVersionException as e: raise e except Exception as e: diff --git a/project_rossum_deploy/commands/migrate/upload_helpers.py b/project_rossum_deploy/commands/migrate/upload_helpers.py index 5f43dd6..5c1d796 100644 --- a/project_rossum_deploy/commands/migrate/upload_helpers.py +++ b/project_rossum_deploy/commands/migrate/upload_helpers.py @@ -1,7 +1,6 @@ from rossum_api import ElisAPIClient from rossum_api.api_client import Resource from rich import print -from rich.progress import Progress from rich.panel import Panel from rich.prompt import Prompt @@ -15,7 +14,6 @@ settings, ) from project_rossum_deploy.utils.functions import ( - PauseProgress, extract_id_from_url, ) @@ -82,11 +80,15 @@ async def upload_workspace( errors[target_id] = (Resource.Workspace, local_object.get("name", "")) return local_object - return await client._http_client.update( + result = await client._http_client.update( Resource.Workspace, id_=target_id, data=workspace ) + print(f'Released (updated) workspace "{workspace['id']}" -> "{target_id}".') + return result else: - return await client._http_client.create(Resource.Workspace, workspace) + result = await client._http_client.create(Resource.Workspace, workspace) + print(f'Released (created) workspace "{workspace['id']}" -> "{result['id']}".') + return result async def upload_queue( @@ -111,11 +113,15 @@ async def upload_queue( errors[target_id] = (Resource.Queue, local_object.get("name", "")) return local_object - return await client._http_client.update( + result = await client._http_client.update( Resource.Queue, id_=target_id, data=queue ) + print(f'Released (updated) queue "{queue['id']}" -> "{target_id}".') + return result else: - return await client._http_client.create(Resource.Queue, queue) + result = await client._http_client.create(Resource.Queue, queue) + print(f'Released (created) queue "{queue['id']}" -> "{result['id']}".') + return result async def upload_inbox( @@ -140,11 +146,15 @@ async def upload_inbox( errors[target_id] = (Resource.Inbox, local_object.get("name", "")) return local_object - return await client._http_client.update( + result = await client._http_client.update( Resource.Inbox, id_=target_id, data=inbox ) + print(f'Released (updated) inbox "{inbox['id']}" -> "{target_id}".') + return result else: - return await client._http_client.create(Resource.Inbox, inbox) + result = await client._http_client.create(Resource.Inbox, inbox) + print(f'Released (created) inbox "{inbox['id']}" -> "{result['id']}".') + return result async def upload_schema( @@ -169,11 +179,15 @@ async def upload_schema( errors[target_id] = (Resource.Schema, local_object.get("name", "")) return local_object - return await client._http_client.update( + result = await client._http_client.update( Resource.Schema, id_=target_id, data=schema ) + print(f'Released (updated) schema "{schema['id']}" -> "{target_id}".') + return result else: - return await client._http_client.create(Resource.Schema, schema) + result = await client._http_client.create(Resource.Schema, schema) + print(f'Released (created) schema "{schema['id']}" -> "{result['id']}".') + return result async def upload_hook( @@ -181,7 +195,6 @@ async def upload_hook( hook: dict, hook_mapping: dict, target_id: int, - progress: Progress, target_objects=[], errors={}, force=False, @@ -200,13 +213,19 @@ async def upload_hook( errors[target_id] = (Resource.Hook, local_object.get("name", "")) return local_object - return await client._http_client.update(Resource.Hook, id_=target_id, data=hook) + result = await client._http_client.update( + Resource.Hook, id_=target_id, data=hook + ) + print(f'Released (updated) hook "{hook['id']}" -> "{target_id}".') + return result + else: created_hook = await create_hook_based_on_template(hook=hook, client=client) if not created_hook: created_hook = await create_hook_without_template( - hook=hook, client=client, hook_mapping=hook_mapping, progress=progress + hook=hook, client=client, hook_mapping=hook_mapping ) + print(f'Released (created) hook "{hook['id']}" -> "{created_hook['id']}".') return created_hook @@ -269,7 +288,7 @@ async def create_hook_based_on_template(hook: dict, client: ElisAPIClient): async def create_hook_without_template( - hook: dict, hook_mapping: dict, client: ElisAPIClient, progress: Progress + hook: dict, hook_mapping: dict, client: ElisAPIClient ): # Use the dummy URL only for newly-created private hooks # And only if attribute override does not specify the url @@ -279,14 +298,11 @@ async def create_hook_without_template( and hook_mapping.get("attribute_override", {}).get("config", {}).get("path", "") != "url" ): - with PauseProgress(progress): - private_hook_url = Prompt.ask( - f"Please provide hook url (target base_url is '{client._http_client.base_url}') for '{hook['name']}'" - ) - hook["config"]["url"] = ( - private_hook_url - if private_hook_url - else settings.PRIVATE_HOOK_DUMMY_URL - ) + private_hook_url = Prompt.ask( + f"Please provide hook url (target base_url is '{client._http_client.base_url}') for '{hook['name']}'" + ) + hook["config"]["url"] = ( + private_hook_url if private_hook_url else settings.PRIVATE_HOOK_DUMMY_URL + ) return await client._http_client.create(Resource.Hook, hook) diff --git a/project_rossum_deploy/commands/migrate/workspaces.py b/project_rossum_deploy/commands/migrate/workspaces.py index b7dee06..07f6546 100644 --- a/project_rossum_deploy/commands/migrate/workspaces.py +++ b/project_rossum_deploy/commands/migrate/workspaces.py @@ -4,7 +4,6 @@ import logging from anyio import Path from rich import print -from rich.progress import Progress from rich.panel import Panel from rossum_api import ElisAPIClient @@ -40,7 +39,6 @@ async def migrate_workspaces( mapping: dict, source_id_target_pairs: dict[int, list], sources_by_source_id_map: dict, - progress: Progress, plan_only: bool = False, selected_only: bool = False, target_objects: list[dict] = [], @@ -51,7 +49,6 @@ async def migrate_workspaces( workspace_path async for workspace_path in (source_path / "workspaces").iterdir() ] - task = progress.add_task("Releasing workspaces.", total=len(workspace_paths)) async def migrate_workspace(ws_path: Path): try: @@ -67,16 +64,20 @@ async def migrate_workspace(ws_path: Path): 1, "organization", source_id_target_pairs, - progress=progress, ) workspace_mapping = find_mapping_of_object( mapping["organization"]["workspaces"], id ) - skip_migration = workspace_mapping.get("ignore", None) or ( - selected_only and not check_if_selected(workspace_mapping) - ) + # Ignoring WS should be hierarchical - queues and inboxes should get ignored as well + if workspace_mapping.get("ignore", None): + print( + f'Ignored workspace "{workspace['id']}" including its queues and inboxes.' + ) + return + + skip_migration = selected_only and not check_if_selected(workspace_mapping) if plan_only or skip_migration: partial_upload_workspace = functools.partial( @@ -119,10 +120,8 @@ async def migrate_workspace(ws_path: Path): target_objects=target_objects, errors=errors, force=force, - progress=progress, ) - progress.update(task, advance=1) except PrdVersionException as e: raise e except Exception as e: @@ -149,7 +148,6 @@ async def migrate_queues_and_inboxes( target_objects: list[dict] = [], errors: dict = {}, force: bool = False, - progress: Progress = None, ): if not (await (ws_path / "queues").exists()): return @@ -187,7 +185,6 @@ async def migrate_queue_and_inbox(queue_path: Path): target_objects=target_objects, errors=errors, force=force, - progress=progress, ) source_id_target_pairs[id] = [] @@ -273,7 +270,6 @@ async def prepare_queue_upload( target_objects: list[dict] = [], errors: dict = {}, force: bool = False, - progress: Progress = None, ): queue = deepcopy(queue) target_object = ( @@ -290,7 +286,6 @@ async def prepare_queue_upload( dependency="workspace", source_id_target_pairs=source_id_target_pairs, target_object=target_object, - progress=progress, ) if previous_workspace_url == queue["workspace"] and not target_id: @@ -306,7 +301,6 @@ async def prepare_queue_upload( dependency="schema", source_id_target_pairs=source_id_target_pairs, target_object=target_object, - progress=progress, ) # Both should be updated, otherwise Elis API uses 'webhooks' in case of a mismatch even though it is deprecated replace_dependency_url( @@ -316,7 +310,6 @@ async def prepare_queue_upload( dependency="hooks", source_id_target_pairs=source_id_target_pairs, target_object=target_object, - progress=progress, ) replace_dependency_url( object=queue, @@ -325,7 +318,6 @@ async def prepare_queue_upload( dependency="webhooks", source_id_target_pairs=source_id_target_pairs, target_object=target_object, - progress=progress, ) queue.pop("inbox", None) @@ -349,7 +341,6 @@ async def prepare_inbox_upload( target_objects: list[dict] = [], errors: dict = {}, force: bool = False, - progress: Progress = None, ): inbox = deepcopy(inbox) @@ -360,7 +351,6 @@ async def prepare_inbox_upload( target_index=target_index, target_objects_count=target_objects_count, source_id_target_pairs=source_id_target_pairs, - progress=progress, ) if previous_queue_urls == inbox["queues"] and not target_id: