diff --git a/e2e/deploy/test.yaml b/e2e/deploy/test.yaml index c23daa8929..157910b1bb 100644 --- a/e2e/deploy/test.yaml +++ b/e2e/deploy/test.yaml @@ -67,21 +67,52 @@ spec: - name: LONGHORN_JUNIT_REPORT_PATH value: /tmp/test-report/longhorn-test-junit-report.xml - name: LONGHORN_BACKUPSTORE - value: "s3://backupbucket@us-east-1/backupstore$minio-secret, nfs://longhorn-test-nfs-svc.default:/opt/backupstore" + value: "s3://backupbucket@us-east-1/backupstore$minio-secret" - name: LONGHORN_BACKUPSTORE_POLL_INTERVAL value: "30" - name: LONGHORN_DISK_TYPE value: "ssd" - name: LONGHORN_UPGRADE_TYPE value: "from_stable" - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - name: MANAGED_K8S_CLUSTER value: "false" - name: HOST_PROVIDER value: "aws" + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: host-provider-cred-secret + key: AWS_ACCESS_KEY_ID + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: host-provider-cred-secret + key: AWS_SECRET_ACCESS_KEY + - name: AWS_DEFAULT_REGION + valueFrom: + secretKeyRef: + name: host-provider-cred-secret + key: AWS_DEFAULT_REGION + - name: LAB_URL + valueFrom: + secretKeyRef: + name: host-provider-cred-secret + key: LAB_URL + - name: LAB_ACCESS_KEY + valueFrom: + secretKeyRef: + name: host-provider-cred-secret + key: LAB_ACCESS_KEY + - name: LAB_SECRET_KEY + valueFrom: + secretKeyRef: + name: host-provider-cred-secret + key: LAB_SECRET_KEY + - name: LAB_CLUSTER_ID + valueFrom: + secretKeyRef: + name: host-provider-cred-secret + key: LAB_CLUSTER_ID volumeMounts: - name: dev mountPath: /dev diff --git a/pipelines/e2e/scripts/longhorn-setup.sh b/pipelines/e2e/scripts/longhorn-setup.sh index 091da9f15b..4e78acf4a2 100755 --- a/pipelines/e2e/scripts/longhorn-setup.sh +++ b/pipelines/e2e/scripts/longhorn-setup.sh @@ -6,11 +6,11 @@ source pipelines/utilities/kubeconfig.sh source pipelines/utilities/selinux_workaround.sh source pipelines/utilities/install_csi_snapshotter.sh source pipelines/utilities/create_aws_secret.sh +source pipelines/utilities/create_harvester_secret.sh source pipelines/utilities/install_backupstores.sh source pipelines/utilities/create_longhorn_namespace.sh source pipelines/utilities/longhorn_manifest.sh source pipelines/utilities/longhorn_ui.sh -source pipelines/utilities/install_litmus.sh source pipelines/utilities/run_longhorn_e2e_test.sh # create and clean tmpdir @@ -46,7 +46,7 @@ main(){ # DON'T REMOVE! set +x create_aws_secret - create_cloud_secret + create_harvester_secret set -x create_instance_mapping_configmap diff --git a/pipelines/gke/scripts/longhorn-setup.sh b/pipelines/gke/scripts/longhorn-setup.sh index 10e7652bde..da1e40b8d8 100755 --- a/pipelines/gke/scripts/longhorn-setup.sh +++ b/pipelines/gke/scripts/longhorn-setup.sh @@ -146,20 +146,6 @@ install_backupstores(){ } -create_aws_secret(){ - AWS_ACCESS_KEY_ID_BASE64=`echo -n "${TF_VAR_lh_aws_access_key}" | base64` - AWS_SECRET_ACCESS_KEY_BASE64=`echo -n "${TF_VAR_lh_aws_secret_key}" | base64` - AWS_DEFAULT_REGION_BASE64=`echo -n "${TF_VAR_aws_region}" | base64` - - yq e -i '.data.AWS_ACCESS_KEY_ID |= "'${AWS_ACCESS_KEY_ID_BASE64}'"' "${TF_VAR_tf_workspace}/templates/aws_cred_secrets.yml" - yq e -i '.data.AWS_SECRET_ACCESS_KEY |= "'${AWS_SECRET_ACCESS_KEY_BASE64}'"' "${TF_VAR_tf_workspace}/templates/aws_cred_secrets.yml" - yq e -i '.data.AWS_DEFAULT_REGION |= "'${AWS_DEFAULT_REGION_BASE64}'"' "${TF_VAR_tf_workspace}/templates/aws_cred_secrets.yml" - - kubectl apply -f "${TF_VAR_tf_workspace}/templates/aws_cred_secrets.yml" - kubectl apply -f "${TF_VAR_tf_workspace}/templates/aws_cred_secrets.yml" -n kube-system -} - - run_longhorn_upgrade_test(){ LONGHORN_TESTS_CUSTOM_IMAGE=${LONGHORN_TESTS_CUSTOM_IMAGE:-"longhornio/longhorn-manager-test:master-head"} @@ -245,13 +231,8 @@ run_longhorn_tests(){ # set MANAGED_K8S_CLUSTER to true yq e -i 'select(.spec.containers[0] != null).spec.containers[0].env[6].value="true"' ${LONGHORN_TESTS_MANIFEST_FILE_PATH} - set +x - ## inject aws cloudprovider and credentials env variables from created secret - yq e -i 'select(.spec.containers[0].env != null).spec.containers[0].env += {"name": "CLOUDPROVIDER", "value": "aws"}' "${LONGHORN_TESTS_MANIFEST_FILE_PATH}" - yq e -i 'select(.spec.containers[0].env != null).spec.containers[0].env += {"name": "AWS_ACCESS_KEY_ID", "valueFrom": {"secretKeyRef": {"name": "aws-cred-secret", "key": "AWS_ACCESS_KEY_ID"}}}' "${LONGHORN_TESTS_MANIFEST_FILE_PATH}" - yq e -i 'select(.spec.containers[0].env != null).spec.containers[0].env += {"name": "AWS_SECRET_ACCESS_KEY", "valueFrom": {"secretKeyRef": {"name": "aws-cred-secret", "key": "AWS_SECRET_ACCESS_KEY"}}}' "${LONGHORN_TESTS_MANIFEST_FILE_PATH}" - yq e -i 'select(.spec.containers[0].env != null).spec.containers[0].env += {"name": "AWS_DEFAULT_REGION", "valueFrom": {"secretKeyRef": {"name": "aws-cred-secret", "key": "AWS_DEFAULT_REGION"}}}' "${LONGHORN_TESTS_MANIFEST_FILE_PATH}" - set -x + ## inject cloudprovider + yq e -i 'select(.spec.containers[0].env != null).spec.containers[0].env += {"name": "CLOUDPROVIDER", "value": "gke"}' "${LONGHORN_TESTS_MANIFEST_FILE_PATH}" LONGHORN_TEST_POD_NAME=`yq e 'select(.spec.containers[0] != null).metadata.name' ${LONGHORN_TESTS_MANIFEST_FILE_PATH}` @@ -280,12 +261,6 @@ run_longhorn_tests(){ main(){ set_kubeconfig_envvar - # set debugging mode off to avoid leaking aws secrets to the logs. - # DON'T REMOVE! - set +x - create_aws_secret - set -x - create_longhorn_namespace if [[ "${TF_VAR_distro}" == "COS_CONTAINERD" ]]; then diff --git a/pipelines/templates/aws_cred_secrets.yml b/pipelines/templates/aws_cred_secrets.yml deleted file mode 100644 index 563584477b..0000000000 --- a/pipelines/templates/aws_cred_secrets.yml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: aws-cred-secret -type: Opaque -data: - AWS_ACCESS_KEY_ID: "set aws-access-key-id base64 encoded" - AWS_SECRET_ACCESS_KEY: "set aws-secret-key base64 encoded" - AWS_DEFAULT_REGION: "set aws-default-region base64 encoded" diff --git a/pipelines/templates/cloud_secret.yml b/pipelines/templates/cloud_secret.yml deleted file mode 100644 index c0009336e9..0000000000 --- a/pipelines/templates/cloud_secret.yml +++ /dev/null @@ -1,12 +0,0 @@ -# Provide credentials to litmus to make it have AWS access -apiVersion: v1 -kind: Secret -metadata: - name: cloud-secret -type: Opaque -stringData: - cloud_config.yml: |- - # Add the cloud AWS credentials respectively - [default] - aws_access_key_id = - aws_secret_access_key = \ No newline at end of file diff --git a/pipelines/templates/host_provider_cred_secrets.yml b/pipelines/templates/host_provider_cred_secrets.yml new file mode 100644 index 0000000000..48cb48568d --- /dev/null +++ b/pipelines/templates/host_provider_cred_secrets.yml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Secret +metadata: + name: host-provider-cred-secret +type: Opaque +data: + AWS_ACCESS_KEY_ID: QVdTX0FDQ0VTU19LRVlfSUQ= # set aws-access-key-id base64 encoded + AWS_SECRET_ACCESS_KEY: QVdTX1NFQ1JFVF9BQ0NFU1NfS0VZ # set aws-secret-key base64 encoded + AWS_DEFAULT_REGION: QVdTX0RFRkFVTFRfUkVHSU9O # set aws-default-region base64 encoded + LAB_URL: TEFCX1VSTA== # set LAB_URL base64 encoded + LAB_ACCESS_KEY: TEFCX0FDQ0VTU19LRVk= # set LAB_ACCESS_KEY base64 encoded + LAB_SECRET_KEY: TEFCX1NFQ1JFVF9LRVk= # set LAB_SECRET_KEY base64 encoded + LAB_CLUSTER_ID: TEFCX0NMVVNURVJfSUQ= # set LAB_CLUSTER_ID base64 encoded diff --git a/pipelines/utilities/create_aws_secret.sh b/pipelines/utilities/create_aws_secret.sh index fa015b8b05..2b74387fd6 100755 --- a/pipelines/utilities/create_aws_secret.sh +++ b/pipelines/utilities/create_aws_secret.sh @@ -3,10 +3,9 @@ create_aws_secret(){ AWS_SECRET_ACCESS_KEY_BASE64=`echo -n "${TF_VAR_lh_aws_secret_key}" | base64` AWS_DEFAULT_REGION_BASE64=`echo -n "${TF_VAR_aws_region}" | base64` - yq e -i '.data.AWS_ACCESS_KEY_ID |= "'${AWS_ACCESS_KEY_ID_BASE64}'"' "pipelines/templates/aws_cred_secrets.yml" - yq e -i '.data.AWS_SECRET_ACCESS_KEY |= "'${AWS_SECRET_ACCESS_KEY_BASE64}'"' "pipelines/templates/aws_cred_secrets.yml" - yq e -i '.data.AWS_DEFAULT_REGION |= "'${AWS_DEFAULT_REGION_BASE64}'"' "pipelines/templates/aws_cred_secrets.yml" + yq e -i '.data.AWS_ACCESS_KEY_ID |= "'${AWS_ACCESS_KEY_ID_BASE64}'"' "pipelines/templates/host_provider_cred_secrets.yml" + yq e -i '.data.AWS_SECRET_ACCESS_KEY |= "'${AWS_SECRET_ACCESS_KEY_BASE64}'"' "pipelines/templates/host_provider_cred_secrets.yml" + yq e -i '.data.AWS_DEFAULT_REGION |= "'${AWS_DEFAULT_REGION_BASE64}'"' "pipelines/templates/host_provider_cred_secrets.yml" - kubectl apply -f "pipelines/templates/aws_cred_secrets.yml" - kubectl apply -f "pipelines/templates/aws_cred_secrets.yml" -n kube-system + kubectl apply -f "pipelines/templates/host_provider_cred_secrets.yml" } \ No newline at end of file diff --git a/pipelines/utilities/create_harvester_secret.sh b/pipelines/utilities/create_harvester_secret.sh new file mode 100644 index 0000000000..e54b2cab2e --- /dev/null +++ b/pipelines/utilities/create_harvester_secret.sh @@ -0,0 +1,13 @@ +create_harvester_secret(){ + LAB_URL_BASE64=`echo -n "${TF_VAR_lab_url}" | base64` + LAB_ACCESS_KEY_BASE64=`echo -n "${TF_VAR_lab_access_key}" | base64` + LAB_SECRET_KEY_BASE64=`echo -n "${TF_VAR_lab_secret_key}" | base64` + LAB_CLUSTER_ID_BASE64=`echo -n "$(cat /tmp/cluster_id)" | base64` + + yq e -i '.data.LAB_URL |= "'${LAB_URL_BASE64}'"' "pipelines/templates/host_provider_cred_secrets.yml" + yq e -i '.data.LAB_ACCESS_KEY |= "'${LAB_ACCESS_KEY_BASE64}'"' "pipelines/templates/host_provider_cred_secrets.yml" + yq e -i '.data.LAB_SECRET_KEY |= "'${LAB_SECRET_KEY_BASE64}'"' "pipelines/templates/host_provider_cred_secrets.yml" + yq e -i '.data.LAB_CLUSTER_ID |= "'${LAB_CLUSTER_ID_BASE64}'"' "pipelines/templates/host_provider_cred_secrets.yml" + + kubectl apply -f "pipelines/templates/host_provider_cred_secrets.yml" +} \ No newline at end of file diff --git a/pipelines/utilities/run_longhorn_e2e_test.sh b/pipelines/utilities/run_longhorn_e2e_test.sh index 33107ea335..bf01ec5a0a 100755 --- a/pipelines/utilities/run_longhorn_e2e_test.sh +++ b/pipelines/utilities/run_longhorn_e2e_test.sh @@ -27,24 +27,10 @@ run_longhorn_e2e_test(){ fi if [[ "${TF_VAR_k8s_distro_name}" == "eks" ]] || [[ "${TF_VAR_k8s_distro_name}" == "aks" ]]; then - yq e -i 'select(.spec.containers[0] != null).spec.containers[0].env[6].value="true"' ${LONGHORN_TESTS_MANIFEST_FILE_PATH} + yq e -i 'select(.spec.containers[0] != null).spec.containers[0].env[5].value="true"' ${LONGHORN_TESTS_MANIFEST_FILE_PATH} fi - yq e -i 'select(.spec.containers[0] != null).spec.containers[0].env[7].value="'${LONGHORN_TEST_CLOUDPROVIDER}'"' ${LONGHORN_TESTS_MANIFEST_FILE_PATH} - - set +x - if [[ "${LONGHORN_TEST_CLOUDPROVIDER}" == "aws" ]]; then - ## inject aws cloudprovider and credentials env variables from created secret - yq e -i 'select(.spec.containers[0].env != null).spec.containers[0].env += {"name": "AWS_ACCESS_KEY_ID", "valueFrom": {"secretKeyRef": {"name": "aws-cred-secret", "key": "AWS_ACCESS_KEY_ID"}}}' "${LONGHORN_TESTS_MANIFEST_FILE_PATH}" - yq e -i 'select(.spec.containers[0].env != null).spec.containers[0].env += {"name": "AWS_SECRET_ACCESS_KEY", "valueFrom": {"secretKeyRef": {"name": "aws-cred-secret", "key": "AWS_SECRET_ACCESS_KEY"}}}' "${LONGHORN_TESTS_MANIFEST_FILE_PATH}" - yq e -i 'select(.spec.containers[0].env != null).spec.containers[0].env += {"name": "AWS_DEFAULT_REGION", "valueFrom": {"secretKeyRef": {"name": "aws-cred-secret", "key": "AWS_DEFAULT_REGION"}}}' "${LONGHORN_TESTS_MANIFEST_FILE_PATH}" - elif [[ "${LONGHORN_TEST_CLOUDPROVIDER}" == "harvester" ]]; then - yq e -i 'select(.spec.containers[0].env != null).spec.containers[0].env += {"name": "LAB_URL", "value": "'${TF_VAR_lab_url}'"}' "${LONGHORN_TESTS_MANIFEST_FILE_PATH}" - yq e -i 'select(.spec.containers[0].env != null).spec.containers[0].env += {"name": "LAB_ACCESS_KEY", "value": "'${TF_VAR_lab_access_key}'"}' "${LONGHORN_TESTS_MANIFEST_FILE_PATH}" - yq e -i 'select(.spec.containers[0].env != null).spec.containers[0].env += {"name": "LAB_SECRET_KEY", "value": "'${TF_VAR_lab_secret_key}'"}' "${LONGHORN_TESTS_MANIFEST_FILE_PATH}" - yq e -i 'select(.spec.containers[0].env != null).spec.containers[0].env += {"name": "LAB_CLUSTER_ID", "value": "'$(cat /tmp/cluster_id)'"}' "${LONGHORN_TESTS_MANIFEST_FILE_PATH}" - fi - set -x + yq e -i 'select(.spec.containers[0] != null).spec.containers[0].env[6].value="'${LONGHORN_TEST_CLOUDPROVIDER}'"' ${LONGHORN_TESTS_MANIFEST_FILE_PATH} LONGHORN_TEST_POD_NAME=`yq e 'select(.spec.containers[0] != null).metadata.name' ${LONGHORN_TESTS_MANIFEST_FILE_PATH}`