From cd4c0f551e8a8366bf8c5ff2eed8aebc5acc7dda Mon Sep 17 00:00:00 2001 From: Ruth Netser Date: Wed, 6 Sep 2023 13:40:22 +0300 Subject: [PATCH] Set default timeout to 60m and restructure code (#145) * arrange code * arrange code * arrange code --- README.md | 2 +- openshift_cli_installer/cli.py | 30 +++-- .../libs/destroy_clusters.py | 6 +- .../libs/managed_clusters/__init__.py | 0 .../libs/managed_clusters/helpers.py | 33 +++++ .../{ => managed_clusters}/osd_clusters.py | 2 +- .../{ => managed_clusters}/rosa_clusters.py | 43 +----- .../libs/unmanaged_clusters/__init__.py | 0 .../aws_ipi_clusters.py | 18 +-- .../tests/test_cluster_version.py | 6 +- .../utils/{helpers.py => clusters.py} | 121 ----------------- openshift_cli_installer/utils/const.py | 1 + openshift_cli_installer/utils/general.py | 124 ++++++++++++++++++ 13 files changed, 196 insertions(+), 190 deletions(-) create mode 100644 openshift_cli_installer/libs/managed_clusters/__init__.py create mode 100644 openshift_cli_installer/libs/managed_clusters/helpers.py rename openshift_cli_installer/libs/{ => managed_clusters}/osd_clusters.py (97%) rename openshift_cli_installer/libs/{ => managed_clusters}/rosa_clusters.py (87%) create mode 100644 openshift_cli_installer/libs/unmanaged_clusters/__init__.py rename openshift_cli_installer/libs/{ => unmanaged_clusters}/aws_ipi_clusters.py (95%) rename openshift_cli_installer/utils/{helpers.py => clusters.py} (63%) create mode 100644 openshift_cli_installer/utils/general.py diff --git a/README.md b/README.md index 3745d99e..18c96f77 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ Every call to the openshift installer cli must have at least one `--cluster` opt * region: The region to deploy the cluster * Optional parameters: * Parameter names should be separated by semicolons (`;`) - * To set cluster create / destroy timeout, pass `--cluster ...timeout=1h'`; default is 30 minutes. + * To set cluster create / destroy timeout (not applicable for AWS IPI clusters), pass `--cluster ...timeout=1h'`; default is 60 minutes. * `timeout` and `expiration-time` format examples: `1h`, `30m`, `3600s` * `ocm-env`: OCM environment to deploy the cluster; available options: `stage` or `production` (defaults to `stage`). AWS-IPI clusters only use `production`. * AWS IPI: diff --git a/openshift_cli_installer/cli.py b/openshift_cli_installer/cli.py index 2e630577..a058788e 100644 --- a/openshift_cli_installer/cli.py +++ b/openshift_cli_installer/cli.py @@ -6,23 +6,30 @@ import rosa.cli from clouds.aws.aws_utils import set_and_verify_aws_credentials -from openshift_cli_installer.libs.aws_ipi_clusters import ( - create_install_config_file, - create_or_destroy_aws_ipi_cluster, - download_openshift_install_binary, - update_aws_clusters_versions, -) from openshift_cli_installer.libs.destroy_clusters import destroy_clusters -from openshift_cli_installer.libs.osd_clusters import ( +from openshift_cli_installer.libs.managed_clusters.helpers import ( + prepare_managed_clusters_data, +) +from openshift_cli_installer.libs.managed_clusters.osd_clusters import ( osd_create_cluster, osd_delete_cluster, ) -from openshift_cli_installer.libs.rosa_clusters import ( - prepare_managed_clusters_data, +from openshift_cli_installer.libs.managed_clusters.rosa_clusters import ( rosa_create_cluster, rosa_delete_cluster, ) +from openshift_cli_installer.libs.unmanaged_clusters.aws_ipi_clusters import ( + create_install_config_file, + create_or_destroy_aws_ipi_cluster, + download_openshift_install_binary, + update_aws_clusters_versions, +) from openshift_cli_installer.utils.click_dict_type import DictParamType +from openshift_cli_installer.utils.clusters import ( + add_ocm_client_to_cluster_dict, + check_existing_clusters, + update_rosa_osd_clusters_versions, +) from openshift_cli_installer.utils.const import ( AWS_OSD_STR, AWS_STR, @@ -34,11 +41,6 @@ ROSA_STR, STAGE_STR, ) -from openshift_cli_installer.utils.helpers import ( - add_ocm_client_to_cluster_dict, - check_existing_clusters, - update_rosa_osd_clusters_versions, -) def get_clusters_by_type(clusters): diff --git a/openshift_cli_installer/libs/destroy_clusters.py b/openshift_cli_installer/libs/destroy_clusters.py index 01131871..726a0c74 100644 --- a/openshift_cli_installer/libs/destroy_clusters.py +++ b/openshift_cli_installer/libs/destroy_clusters.py @@ -7,11 +7,13 @@ import yaml from clouds.aws.session_clients import s3_client -from openshift_cli_installer.libs.aws_ipi_clusters import ( +from openshift_cli_installer.libs.managed_clusters.rosa_clusters import ( + rosa_delete_cluster, +) +from openshift_cli_installer.libs.unmanaged_clusters.aws_ipi_clusters import ( create_or_destroy_aws_ipi_cluster, download_openshift_install_binary, ) -from openshift_cli_installer.libs.rosa_clusters import rosa_delete_cluster from openshift_cli_installer.utils.const import ( AWS_STR, CLUSTER_DATA_YAML_FILENAME, diff --git a/openshift_cli_installer/libs/managed_clusters/__init__.py b/openshift_cli_installer/libs/managed_clusters/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/openshift_cli_installer/libs/managed_clusters/helpers.py b/openshift_cli_installer/libs/managed_clusters/helpers.py new file mode 100644 index 00000000..546a74e9 --- /dev/null +++ b/openshift_cli_installer/libs/managed_clusters/helpers.py @@ -0,0 +1,33 @@ +from datetime import datetime, timedelta + +from openshift_cli_installer.utils.const import HYPERSHIFT_STR, TIMEOUT_60MIN +from openshift_cli_installer.utils.general import tts + + +def prepare_managed_clusters_data( + clusters, + aws_account_id, + aws_secret_access_key, + aws_access_key_id, +): + for _cluster in clusters: + _cluster["cluster-name"] = _cluster["name"] + _cluster["timeout"] = tts(ts=_cluster.get("timeout", TIMEOUT_60MIN)) + _cluster["channel-group"] = _cluster.get("channel-group", "stable") + _cluster["aws-access-key-id"] = aws_access_key_id + _cluster["aws-secret-access-key"] = aws_secret_access_key + _cluster["aws-account-id"] = aws_account_id + _cluster["multi-az"] = _cluster.get("multi-az", False) + if _cluster["platform"] == HYPERSHIFT_STR: + _cluster["hosted-cp"] = "true" + _cluster["tags"] = "dns:external" + _cluster["machine-cidr"] = _cluster.get("cidr", "10.0.0.0/16") + + expiration_time = _cluster.get("expiration-time") + if expiration_time: + _expiration_time = tts(ts=expiration_time) + _cluster["expiration-time"] = ( + f"{(datetime.now() + timedelta(seconds=_expiration_time)).isoformat()}Z" + ) + + return clusters diff --git a/openshift_cli_installer/libs/osd_clusters.py b/openshift_cli_installer/libs/managed_clusters/osd_clusters.py similarity index 97% rename from openshift_cli_installer/libs/osd_clusters.py rename to openshift_cli_installer/libs/managed_clusters/osd_clusters.py index 617cae2f..a3e2249f 100644 --- a/openshift_cli_installer/libs/osd_clusters.py +++ b/openshift_cli_installer/libs/managed_clusters/osd_clusters.py @@ -1,7 +1,7 @@ import click from ocm_python_wrapper.cluster import Cluster -from openshift_cli_installer.utils.helpers import ( +from openshift_cli_installer.utils.clusters import ( add_cluster_info_to_cluster_data, dump_cluster_data_to_file, set_cluster_auth, diff --git a/openshift_cli_installer/libs/rosa_clusters.py b/openshift_cli_installer/libs/managed_clusters/rosa_clusters.py similarity index 87% rename from openshift_cli_installer/libs/rosa_clusters.py rename to openshift_cli_installer/libs/managed_clusters/rosa_clusters.py index 94162626..c9d1875a 100644 --- a/openshift_cli_installer/libs/rosa_clusters.py +++ b/openshift_cli_installer/libs/managed_clusters/rosa_clusters.py @@ -1,7 +1,6 @@ import os import re import shutil -from datetime import datetime, timedelta import click import rosa.cli @@ -9,18 +8,19 @@ from ocm_python_wrapper.cluster import Cluster from python_terraform import IsNotFlagged, Terraform, TerraformCommandError +from openshift_cli_installer.utils.clusters import ( + add_cluster_info_to_cluster_data, + cluster_shortuuid, + dump_cluster_data_to_file, + set_cluster_auth, +) from openshift_cli_installer.utils.const import ( CLUSTER_DATA_YAML_FILENAME, HYPERSHIFT_STR, ) -from openshift_cli_installer.utils.helpers import ( - add_cluster_info_to_cluster_data, +from openshift_cli_installer.utils.general import ( bucket_object_name, - cluster_shortuuid, - dump_cluster_data_to_file, get_manifests_path, - set_cluster_auth, - tts, zip_and_upload_to_s3, ) @@ -138,35 +138,6 @@ def prepare_hypershift_vpc(cluster_data): raise -def prepare_managed_clusters_data( - clusters, - aws_account_id, - aws_secret_access_key, - aws_access_key_id, -): - for _cluster in clusters: - _cluster["cluster-name"] = _cluster["name"] - _cluster["timeout"] = tts(ts=_cluster.get("timeout", "30m")) - _cluster["channel-group"] = _cluster.get("channel-group", "stable") - _cluster["aws-access-key-id"] = aws_access_key_id - _cluster["aws-secret-access-key"] = aws_secret_access_key - _cluster["aws-account-id"] = aws_account_id - _cluster["multi-az"] = _cluster.get("multi-az", False) - if _cluster["platform"] == HYPERSHIFT_STR: - _cluster["hosted-cp"] = "true" - _cluster["tags"] = "dns:external" - _cluster["machine-cidr"] = _cluster.get("cidr", "10.0.0.0/16") - - expiration_time = _cluster.get("expiration-time") - if expiration_time: - _expiration_time = tts(ts=expiration_time) - _cluster["expiration-time"] = ( - f"{(datetime.now() + timedelta(seconds=_expiration_time)).isoformat()}Z" - ) - - return clusters - - def rosa_create_cluster(cluster_data, s3_bucket_name=None, s3_bucket_path=None): hosted_cp_arg = "--hosted-cp" _platform = cluster_data["platform"] diff --git a/openshift_cli_installer/libs/unmanaged_clusters/__init__.py b/openshift_cli_installer/libs/unmanaged_clusters/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/openshift_cli_installer/libs/aws_ipi_clusters.py b/openshift_cli_installer/libs/unmanaged_clusters/aws_ipi_clusters.py similarity index 95% rename from openshift_cli_installer/libs/aws_ipi_clusters.py rename to openshift_cli_installer/libs/unmanaged_clusters/aws_ipi_clusters.py index fd972f0a..69ebbdc0 100644 --- a/openshift_cli_installer/libs/aws_ipi_clusters.py +++ b/openshift_cli_installer/libs/unmanaged_clusters/aws_ipi_clusters.py @@ -9,14 +9,15 @@ from ocp_utilities.utils import run_command from openshift_cli_installer.utils.cluster_versions import set_clusters_versions -from openshift_cli_installer.utils.const import CREATE_STR, DESTROY_STR -from openshift_cli_installer.utils.helpers import ( +from openshift_cli_installer.utils.clusters import ( add_cluster_info_to_cluster_data, - bucket_object_name, cluster_shortuuid, dump_cluster_data_to_file, +) +from openshift_cli_installer.utils.const import CREATE_STR, DESTROY_STR +from openshift_cli_installer.utils.general import ( + bucket_object_name, get_manifests_path, - get_ocm_client, zip_and_upload_to_s3, ) @@ -229,12 +230,3 @@ def get_all_versions(_test=None): base_available_versions = get_aws_versions() return base_available_versions - - -def prepare_base_aws_cluster_data(aws_ipi_clusters, ocm_token): - for _cluster in aws_ipi_clusters: - _cluster["ocm-client"] = get_ocm_client( - ocm_token=ocm_token, ocm_env=_cluster["ocm_env"] - ) - - return aws_ipi_clusters diff --git a/openshift_cli_installer/tests/test_cluster_version.py b/openshift_cli_installer/tests/test_cluster_version.py index 1ca5eab5..0dfd964a 100644 --- a/openshift_cli_installer/tests/test_cluster_version.py +++ b/openshift_cli_installer/tests/test_cluster_version.py @@ -1,14 +1,16 @@ import click import pytest -from openshift_cli_installer.libs.aws_ipi_clusters import update_aws_clusters_versions +from openshift_cli_installer.libs.unmanaged_clusters.aws_ipi_clusters import ( + update_aws_clusters_versions, +) from openshift_cli_installer.tests.all_osd_versions import ( BASE_AVAILABLE_OSD_VERSIONS_DICT, ) from openshift_cli_installer.tests.all_rosa_versions import ( BASE_AVAILABLE_ROSA_VERSIONS_DICT, ) -from openshift_cli_installer.utils.helpers import update_rosa_osd_clusters_versions +from openshift_cli_installer.utils.clusters import update_rosa_osd_clusters_versions @pytest.mark.parametrize( diff --git a/openshift_cli_installer/utils/helpers.py b/openshift_cli_installer/utils/clusters.py similarity index 63% rename from openshift_cli_installer/utils/helpers.py rename to openshift_cli_installer/utils/clusters.py index 63c6df45..0c2594f7 100644 --- a/openshift_cli_installer/utils/helpers.py +++ b/openshift_cli_installer/utils/clusters.py @@ -1,17 +1,11 @@ import copy import os -import re -import shutil -from functools import wraps -from importlib.util import find_spec from pathlib import Path -from time import sleep import click import rosa.cli import shortuuid import yaml -from clouds.aws.session_clients import s3_client from ocm_python_wrapper.cluster import Cluster from ocm_python_wrapper.ocm_client import OCMPythonClient from ocm_python_wrapper.versions import Versions @@ -30,44 +24,6 @@ ) -# TODO: Move to own repository. -def ignore_exceptions(logger=None, retry=None): - def wrapper(func): - @wraps(func) - def inner(*args, **kwargs): - try: - return func(*args, **kwargs) - except Exception as ex: - if retry: - for _ in range(0, retry): - try: - return func(*args, **kwargs) - except Exception: - sleep(1) - - if logger: - logger.info(ex) - return None - - return inner - - return wrapper - - -def remove_terraform_folder_from_install_dir(install_dir): - """ - .terraform folder created when call terraform.init() and it's take more space. - """ - folders_to_remove = [] - for root, dirs, files in os.walk(install_dir): - for _dir in dirs: - if _dir == ".terraform": - folders_to_remove.append(os.path.join(root, _dir)) - - for folder in folders_to_remove: - shutil.rmtree(folder) - - def get_ocm_client(ocm_token, ocm_env): return OCMPythonClient( token=ocm_token, @@ -81,33 +37,6 @@ def cluster_shortuuid(): return shortuuid.uuid() -@ignore_exceptions() -def zip_and_upload_to_s3( - install_dir, - s3_bucket_name, - uuid, - s3_bucket_path=None, -): - remove_terraform_folder_from_install_dir(install_dir=install_dir) - - _base_name = f"{install_dir}-{uuid}" - - zip_file = shutil.make_archive( - base_name=_base_name, - format="zip", - root_dir=install_dir, - ) - bucket_key = os.path.join(s3_bucket_path or "", os.path.split(zip_file)[-1]) - click.echo(f"Upload {zip_file} file to S3 {s3_bucket_name}, path {bucket_key}") - s3_client().upload_file( - Filename=zip_file, - Bucket=s3_bucket_name, - Key=bucket_key, - ) - - return _base_name - - def dump_cluster_data_to_file(cluster_data): _cluster_data = copy.copy(cluster_data) _cluster_data.pop("ocm-client", "") @@ -117,22 +46,6 @@ def dump_cluster_data_to_file(cluster_data): fd.write(yaml.dump(_cluster_data)) -def bucket_object_name(cluster_data, _shortuuid, s3_bucket_path=None): - return ( - f"{f'{s3_bucket_path}/' if s3_bucket_path else ''}{cluster_data['name']}-{_shortuuid}.zip" - ) - - -def get_manifests_path(): - manifests_path = os.path.join("openshift_cli_installer", "manifests") - if not os.path.isdir(manifests_path): - manifests_path = os.path.join( - find_spec("openshift_cli_installer").submodule_search_locations[0], - "manifests", - ) - return manifests_path - - def update_rosa_osd_clusters_versions(clusters, _test=False, _test_versions_dict=None): if _test: base_available_versions_dict = _test_versions_dict @@ -196,40 +109,6 @@ def add_cluster_info_to_cluster_data(cluster_data, cluster_object=None): return cluster_data -def tts(ts): - """ - Convert time string to seconds. - - Args: - ts (str): time string to convert, can be and int followed by s/m/h - if only numbers was sent return int(ts) - - Example: - >>> tts(ts="1h") - 3600 - >>> tts(ts="3600") - 3600 - - Returns: - int: Time in seconds - """ - try: - time_and_unit = re.match(r"(?P