From a3ff222d90944ca464edb09f60769fb07bf29fa3 Mon Sep 17 00:00:00 2001 From: Chris Date: Wed, 23 Oct 2024 15:39:48 +0800 Subject: [PATCH] test(robot): pull backup from another longhorn ref: longhorn/longhorn#9646 Signed-off-by: Chris --- e2e/keywords/longhorn.resource | 8 ++ e2e/keywords/volume.resource | 9 ++ e2e/libs/backup/rest.py | 9 +- e2e/libs/keywords/longhorn_deploy_keywords.py | 8 +- e2e/libs/longhorn_deploy/base.py | 23 +++-- e2e/libs/longhorn_deploy/longhorn_deploy.py | 8 +- .../longhorn_deploy/longhorn_helm_chart.py | 6 +- e2e/libs/longhorn_deploy/longhorn_kubectl.py | 11 ++- e2e/libs/utility/constant.py | 1 + .../pull_backup_from_another_longhorn.robot | 98 +++++++++++++++++++ e2e/utilities/longhorn-install.sh | 35 ++++++- pipelines/utilities/run_longhorn_e2e_test.sh | 2 + 12 files changed, 192 insertions(+), 26 deletions(-) create mode 100644 e2e/tests/negative/pull_backup_from_another_longhorn.robot diff --git a/e2e/keywords/longhorn.resource b/e2e/keywords/longhorn.resource index f413066528..af4475b20a 100644 --- a/e2e/keywords/longhorn.resource +++ b/e2e/keywords/longhorn.resource @@ -82,3 +82,11 @@ Delete instance-manager of deployment ${deployment_id} volume Wait for Longhorn components all running wait_for_namespace_pods_running longhorn-system + +Install Longhorn stable version + install_longhorn_system is_stable_version=True + +Uninstall Longhorn stable version + ${backups_before_uninstall} = list_all_backups + uninstall_longhorn_system is_stable_version=True + Set Test Variable ${backups_before_uninstall} diff --git a/e2e/keywords/volume.resource b/e2e/keywords/volume.resource index b322b369e0..0d5c8620b4 100644 --- a/e2e/keywords/volume.resource +++ b/e2e/keywords/volume.resource @@ -91,6 +91,10 @@ Write data ${data_id} to volume ${volume_id} ${volume_name} = generate_name_with_suffix volume ${volume_id} write_volume_random_data ${volume_name} 2048 ${data_id} +Write data ${data_id} ${size} MB to volume ${volume_id} + ${volume_name} = generate_name_with_suffix volume ${volume_id} + write_volume_random_data ${volume_name} ${size} ${data_id} + Keep writing data to volume ${volume_id} ${volume_name} = generate_name_with_suffix volume ${volume_id} keep_writing_data ${volume_name} @@ -292,6 +296,11 @@ Check volume ${volume_id} data is backup ${backup_id} created in another cluster ${backup_data} = get_backup_data_from_backup_list ${backups_before_uninstall} ${backup_id} Should Be Equal ${current_checksum} ${backup_data} +Create volume ${volume_id} from backup ${backup_id} in another cluster + ${volume_name} = generate_name_with_suffix volume ${volume_id} + ${backup_url} = get_backup_url_from_backup_list ${backups_before_uninstall} ${backup_id} + create_volume ${volume_name} fromBackup=${backup_url} + Create DR volume ${volume_id} from backup ${backup_id} in another cluster ${volume_name} = generate_name_with_suffix volume ${volume_id} ${backup_url} = get_backup_url_from_backup_list ${backups_before_uninstall} ${backup_id} diff --git a/e2e/libs/backup/rest.py b/e2e/libs/backup/rest.py index 977c13d60f..7fe9fa3dfe 100644 --- a/e2e/libs/backup/rest.py +++ b/e2e/libs/backup/rest.py @@ -53,8 +53,13 @@ def get(self, backup_id, volume_name): def get_from_list(self, backup_list, backup_id): for backup in backup_list["items"]: - if backup['metadata']['annotations']['test.longhorn.io/backup-id'] == backup_id: - return backup + try: + if backup['metadata']['annotations']['test.longhorn.io/backup-id'] == backup_id: + return backup + except KeyError as e: + logging(f"Missing key in backup metadata: {str(e)} for backup {backup['metadata']['name']}") + except Exception as e: + logging(f"Unexpected error accessing backup {backup['metadata']['name']}: {str(e)}") return None def get_by_snapshot(self, volume_name, snapshot_name): diff --git a/e2e/libs/keywords/longhorn_deploy_keywords.py b/e2e/libs/keywords/longhorn_deploy_keywords.py index ac170f4d5f..df755d41ea 100644 --- a/e2e/libs/keywords/longhorn_deploy_keywords.py +++ b/e2e/libs/keywords/longhorn_deploy_keywords.py @@ -4,11 +4,11 @@ class longhorn_deploy_keywords: def __init__(self): self.longhorn = LonghornDeploy() - def uninstall_longhorn_system(self): - self.longhorn.uninstall() + def uninstall_longhorn_system(self, is_stable_version=False): + self.longhorn.uninstall(is_stable_version) def check_longhorn_crd_removed(self): self.longhorn.check_longhorn_crd_removed() - def install_longhorn_system(self): - self.longhorn.install() + def install_longhorn_system(self, is_stable_version=False): + self.longhorn.install(is_stable_version) diff --git a/e2e/libs/longhorn_deploy/base.py b/e2e/libs/longhorn_deploy/base.py index 1ba6468cbf..4159ac89e9 100644 --- a/e2e/libs/longhorn_deploy/base.py +++ b/e2e/libs/longhorn_deploy/base.py @@ -4,6 +4,7 @@ from utility.constant import LONGHORN_UNINSTALL_JOB_LABEL from utility.constant import LONGHORN_INSTALL_SCRIPT_PATH from utility.constant import LONGHORN_INSTALL_TIMEOUT +from utility.constant import LONGHORN_INSTALL_STABLE_SHELL_FUNCTION import subprocess import os from utility.utility import get_retry_count_and_interval @@ -19,7 +20,7 @@ def install(self): return NotImplemented @abstractmethod - def uninstall(self, longhorn_branch=None): + def uninstall(self, is_stable_version=False): return NotImplemented def check_longhorn_crd_removed(self): @@ -29,17 +30,27 @@ def check_longhorn_crd_removed(self): def check_longhorn_uninstall_pod_log(self): logs = k8s.get_pod_logs(LONGHORN_NAMESPACE, LONGHORN_UNINSTALL_JOB_LABEL) - assert "error" not in logs - assert "level=fatal" not in logs + assert "level=error" not in logs, f"find string 'level=error' in uninstall log {logs}" + assert "level=fatal" not in logs, f"find string 'level=fatal' in uninstall log {logs}" - def install_longhorn(self): + def install_longhorn(self, is_stable_version=False): current_path=os.getcwd() full_path = os.path.join(current_path, LONGHORN_INSTALL_SCRIPT_PATH) + if is_stable_version is True: + cmd = ['bash', full_path, LONGHORN_INSTALL_STABLE_SHELL_FUNCTION] + else: + cmd = ['bash', full_path] + try: - output = subprocess.check_output(['bash', full_path], timeout=LONGHORN_INSTALL_TIMEOUT) + output = subprocess.check_output(cmd, timeout=LONGHORN_INSTALL_TIMEOUT) logging(output) except subprocess.CalledProcessError as e: - logging(f"Error: {e.stderr}") + logging(f"Command failed with exit code {e.returncode}") + logging(f"stdout: {e.output}") + logging(f"stderr: {e.stderr}") + raise except subprocess.TimeoutExpired as e: logging(f"Command timed out after {e.timeout} seconds") + logging(f"stdout: {e.output}") + raise diff --git a/e2e/libs/longhorn_deploy/longhorn_deploy.py b/e2e/libs/longhorn_deploy/longhorn_deploy.py index 2023e7d73d..47fed5fadb 100644 --- a/e2e/libs/longhorn_deploy/longhorn_deploy.py +++ b/e2e/libs/longhorn_deploy/longhorn_deploy.py @@ -14,11 +14,11 @@ def __init__(self): elif self._method == "helm": self.longhorn = LonghornHelmChart() - def uninstall(self): - return self.longhorn.uninstall() + def uninstall(self, is_stable_version=False): + return self.longhorn.uninstall(is_stable_version) def check_longhorn_crd_removed(self): return self.longhorn.check_longhorn_crd_removed() - def install(self): - return self.longhorn.install() + def install(self, is_stable_version=False): + return self.longhorn.install(is_stable_version) diff --git a/e2e/libs/longhorn_deploy/longhorn_helm_chart.py b/e2e/libs/longhorn_deploy/longhorn_helm_chart.py index 47f3cd345e..67193bc1ee 100644 --- a/e2e/libs/longhorn_deploy/longhorn_helm_chart.py +++ b/e2e/libs/longhorn_deploy/longhorn_helm_chart.py @@ -8,7 +8,7 @@ class LonghornHelmChart(Base): - def uninstall(self): + def uninstall(self, is_stable_version=False): control_plane_nodes = Node.list_node_names_by_role(self, role="control-plane") control_plane_node = control_plane_nodes[0] @@ -19,5 +19,5 @@ def uninstall(self): k8s.delete_namespace(namespace=LONGHORN_NAMESPACE) k8s.wait_namespace_terminated(namespace=LONGHORN_NAMESPACE) - def install(self): - self.install_longhorn() + def install(self, is_stable_version=False): + self.install_longhorn(is_stable_version) diff --git a/e2e/libs/longhorn_deploy/longhorn_kubectl.py b/e2e/libs/longhorn_deploy/longhorn_kubectl.py index 960088ed99..d915bae2e7 100644 --- a/e2e/libs/longhorn_deploy/longhorn_kubectl.py +++ b/e2e/libs/longhorn_deploy/longhorn_kubectl.py @@ -9,8 +9,11 @@ class LonghornKubectl(Base): - def uninstall(self): - longhorn_branch = os.getenv("LONGHORN_REPO_BRANCH") + def uninstall(self, is_stable_version=False): + env_var = "LONGHORN_STABLE_VERSION" if is_stable_version else "LONGHORN_REPO_BRANCH" + longhorn_branch = os.getenv(env_var) + if not longhorn_branch: + raise ValueError(f"Required environment variable {env_var} is not set") control_plane_nodes = Node.list_node_names_by_role(self, role="control-plane") control_plane_node = control_plane_nodes[0] @@ -30,5 +33,5 @@ def uninstall(self): assert res, "delete uninstallation components failed" k8s.wait_namespace_terminated(namespace=LONGHORN_NAMESPACE) - def install(self): - self.install_longhorn() + def install(self, is_stable_version=False): + self.install_longhorn(is_stable_version) diff --git a/e2e/libs/utility/constant.py b/e2e/libs/utility/constant.py index ac1c6e49ea..dc3ab00994 100644 --- a/e2e/libs/utility/constant.py +++ b/e2e/libs/utility/constant.py @@ -22,3 +22,4 @@ LONGHORN_UNINSTALL_JOB_LABEL="job-name=longhorn-uninstall" LONGHORN_INSTALL_SCRIPT_PATH="utilities/longhorn-install.sh" LONGHORN_INSTALL_TIMEOUT = 600 +LONGHORN_INSTALL_STABLE_SHELL_FUNCTION = "install_longhorn_stable_version" diff --git a/e2e/tests/negative/pull_backup_from_another_longhorn.robot b/e2e/tests/negative/pull_backup_from_another_longhorn.robot new file mode 100644 index 0000000000..819350ad68 --- /dev/null +++ b/e2e/tests/negative/pull_backup_from_another_longhorn.robot @@ -0,0 +1,98 @@ +*** Settings *** +Documentation Uninstallation Checks + +Test Tags negative + +Resource ../keywords/common.resource +Resource ../keywords/setting.resource +Resource ../keywords/volume.resource +Resource ../keywords/persistentvolume.resource +Resource ../keywords/persistentvolumeclaim.resource +Resource ../keywords/workload.resource +Resource ../keywords/backup.resource +Resource ../keywords/snapshot.resource +Resource ../keywords/backupstore.resource +Resource ../keywords/longhorn.resource +Library ../libs/keywords/setting_keywords.py + +Test Setup Set test environment +Test Teardown Cleanup test resources + +*** Variables *** +${LOOP_COUNT} 1 +${RETRY_COUNT} 300 +${RETRY_INTERVAL} 1 +${DATA_ENGINE} v1 + +*** Test Cases *** +Pull backup created by another Longhorn system + [Documentation] Pull backup created by another Longhorn system + ... 1. Install test version of Longhorn. + ... 2. Create volume, write data, and take backup. + ... 3. Uninstall Longhorn. + ... 4. Install test version of Longhorn. + ... 5. Restore the backup create in step 2 and verify the data. + ... 6. Uninstall Longhorn. + ... 7. Install previous version of Longhorn. + ... 8. Create volume, write data, and take backup. + ... 9. Uninstall Longhorn. + ... 10. Install test version of Longhorn. + ... 11. Restore the backup create in step 8 and verify the data. + ... + ... Important + ... - This test case need have set environment variable manually first if not run on Jenkins + ... - LONGHORN_INSTALL_METHOD : helm or manifest + ... - LONGHORN_REPO_BRANCH (ex:master) + ... - CUSTOM_LONGHORN_MANAGER_IMAGE (if not using master-head) + ... - CUSTOM_LONGHORN_ENGINE_IMAGE (if not using master-head) + ... - CUSTOM_LONGHORN_INSTANCE_MANAGER_IMAGE (if not using master-head) + ... - CUSTOM_LONGHORN_SHARE_MANAGER_IMAGE (if not using master-head) + ... - CUSTOM_LONGHORN_BACKING_IMAGE_MANAGER_IMAGE (if not using master-head) + ... - LONGHORN_STABLE_VERSION (ex:v1.6.3) + Given Set setting deleting-confirmation-flag to true + And Create volume 0 with dataEngine=${DATA_ENGINE} + And Attach volume 0 + And Wait for volume 0 healthy + And Write data 0 300 MB to volume 0 + When Create backup 0 for volume 0 + Then Verify backup list contains no error for volume 0 + And Verify backup list contains backup 0 of volume 0 + Then Uninstall Longhorn + And Check Longhorn CRD removed + + # Install current version then pull backup and verify data + Then Install Longhorn + And Set setting deleting-confirmation-flag to true + And Set backupstore + And Check backup synced from backupstore + And Create volume 1 from backup 0 in another cluster + And Wait for volume 1 detached + And Attach volume 1 + And Wait for volume 1 healthy + Then Check volume 1 data is backup 0 created in another cluster + Then Uninstall Longhorn + And Check Longhorn CRD removed + + # Install previous version and create backup + Then Install Longhorn stable version + And Set setting deleting-confirmation-flag to true + And Set backupstore + And Create volume 2 with dataEngine=${DATA_ENGINE} + And Attach volume 2 + And Wait for volume 2 healthy + And Write data 1 300 MB to volume 2 + When Create backup 1 for volume 2 + Then Verify backup list contains no error for volume 2 + And Verify backup list contains backup 1 of volume 2 + Then Uninstall Longhorn stable version + And Check Longhorn CRD removed + + # Install current version then pull backup and verify data + Then Install Longhorn + And Set backupstore + And Check backup synced from backupstore + And Create volume 3 from backup 1 in another cluster + And Wait for volume 3 detached + And Attach volume 3 + And Wait for volume 3 healthy + Then Check volume 3 data is backup 1 created in another cluster diff --git a/e2e/utilities/longhorn-install.sh b/e2e/utilities/longhorn-install.sh index 9cd0d428c2..459166e22a 100644 --- a/e2e/utilities/longhorn-install.sh +++ b/e2e/utilities/longhorn-install.sh @@ -13,6 +13,8 @@ source ../pipelines/utilities/longhorn_manifest.sh # create and clean tmpdir TMPDIR="/tmp/longhorn" LONGHORN_NAMESPACE="longhorn-system" +LONGHORN_REPO_DIR="${TMPDIR}/longhorn" +LONGHORN_REPO_URI=${LONGHORN_REPO_URI:-"https://github.com/longhorn/longhorn.git"} mkdir -p ${TMPDIR} rm -rf "${TMPDIR}/" @@ -23,19 +25,46 @@ install_longhorn_by_chart(){ wait_longhorn_status_running } +install_longhorn_stable_by_chart(){ + git clone --single-branch \ + --branch "${LONGHORN_STABLE_VERSION}" \ + "${LONGHORN_REPO_URI}" \ + "${LONGHORN_REPO_DIR}" + helm upgrade --install longhorn "${LONGHORN_REPO_DIR}/chart/" --namespace "${LONGHORN_NAMESPACE}" + wait_longhorn_status_running +} + +install_longhorn_stable_by_manifest(){ + LONGHORN_STABLE_VERSION=${LONGHORN_STABLE_VERSION} + LONGHORN_STABLE_MANIFEST_URL="https://raw.githubusercontent.com/longhorn/longhorn/${LONGHORN_STABLE_VERSION}/deploy/longhorn.yaml" + install_longhorn_by_manifest "${LONGHORN_STABLE_MANIFEST_URL}" +} + install_longhorn(){ create_longhorn_namespace install_backupstores if [[ "${LONGHORN_INSTALL_METHOD}" == "helm" ]]; then - LONGHORN_REPO_URI=${LONGHORN_REPO_URI:-"https://github.com/longhorn/longhorn.git"} - LONGHORN_REPO_DIR="${TMPDIR}/longhorn" install_longhorn_by_chart elif [[ "${LONGHORN_INSTALL_METHOD}" == "manifest" ]]; then generate_longhorn_yaml_manifest "${TF_VAR_tf_workspace}" install_longhorn_by_manifest "${TF_VAR_tf_workspace}/longhorn.yaml" fi setup_longhorn_ui_nodeport +} +install_longhorn_stable_version(){ + create_longhorn_namespace + install_backupstores + if [[ "${LONGHORN_INSTALL_METHOD}" == "helm" ]]; then + install_longhorn_stable_by_chart + elif [[ "${LONGHORN_INSTALL_METHOD}" == "manifest" ]]; then + install_longhorn_stable_by_manifest + fi + setup_longhorn_ui_nodeport } -install_longhorn +if [[ $# -gt 0 ]]; then + $1 # Run the function passed as the first argument +else + install_longhorn +fi diff --git a/pipelines/utilities/run_longhorn_e2e_test.sh b/pipelines/utilities/run_longhorn_e2e_test.sh index e75b7c3732..8cd6606fe8 100755 --- a/pipelines/utilities/run_longhorn_e2e_test.sh +++ b/pipelines/utilities/run_longhorn_e2e_test.sh @@ -42,6 +42,7 @@ run_longhorn_e2e_test(){ yq e -i 'select(.spec.containers[0] != null).spec.containers[0].env += {"name": "CUSTOM_LONGHORN_SHARE_MANAGER_IMAGE", "value": "'${CUSTOM_LONGHORN_SHARE_MANAGER_IMAGE}'"}' "${LONGHORN_TESTS_MANIFEST_FILE_PATH}" yq e -i 'select(.spec.containers[0] != null).spec.containers[0].env += {"name": "CUSTOM_LONGHORN_BACKING_IMAGE_MANAGER_IMAGE", "value": "'${CUSTOM_LONGHORN_BACKING_IMAGE_MANAGER_IMAGE}'"}' "${LONGHORN_TESTS_MANIFEST_FILE_PATH}" yq e -i 'select(.spec.containers[0] != null).spec.containers[0].env += {"name": "LONGHORN_INSTALL_METHOD", "value": "'${LONGHORN_INSTALL_METHOD}'"}' "${LONGHORN_TESTS_MANIFEST_FILE_PATH}" + yq e -i 'select(.spec.containers[0] != null).spec.containers[0].env += {"name": "LONGHORN_STABLE_VERSION", "value": "'${LONGHORN_STABLE_VERSION}'"}' "${LONGHORN_TESTS_MANIFEST_FILE_PATH}" LONGHORN_TEST_POD_NAME=`yq e 'select(.spec.containers[0] != null).metadata.name' ${LONGHORN_TESTS_MANIFEST_FILE_PATH}` @@ -106,6 +107,7 @@ run_longhorn_e2e_test_out_of_cluster(){ -e CUSTOM_LONGHORN_SHARE_MANAGER_IMAGE="${CUSTOM_LONGHORN_SHARE_MANAGER_IMAGE}"\ -e CUSTOM_LONGHORN_BACKING_IMAGE_MANAGER_IMAGE="${CUSTOM_LONGHORN_BACKING_IMAGE_MANAGER_IMAGE}"\ -e LONGHORN_INSTALL_METHOD="${LONGHORN_INSTALL_METHOD}"\ + -e LONGHORN_STABLE_VERSION="${LONGHORN_STABLE_VERSION}"\ --mount source="vol-${IMAGE_NAME}",target=/tmp \ "${LONGHORN_TESTS_CUSTOM_IMAGE}" "${ROBOT_COMMAND_ARGS[@]}" docker stop "${CONTAINER_NAME}"