qa-saas #378
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # qa-saas workflow is for scale testing and/or validation against the Nexodus qa or prod SaaS deployments | |
| name: qa-saas | |
| concurrency: | |
| group: qa-scale | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| deployment_size: | |
| description: 'deployment size: small | medium | large | xlarge' | |
| required: true | |
| default: 'small' | |
| type: string | |
| pr_or_branch: | |
| description: 'pull request number or branch name' | |
| required: true | |
| default: 'main' | |
| debug_pause: | |
| description: 'time in minutes to pause before tearing down the infra for debugging' | |
| required: false | |
| default: '0' | |
| controller_address: | |
| description: 'options: qa.nexodus.io | try.nexodus.io' | |
| required: true | |
| default: 'qa.nexodus.io' | |
| type: string | |
| schedule: | |
| - cron: '0 17 * * *' | |
| jobs: | |
| deploy-qa: | |
| name: deploy-qa-ec2 | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| env: | |
| AWS_REGION: "us-east-1" | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| ANSIBLE_PRIVATE_KEY_FILE: "nexodus.pem" | |
| ANSIBLE_HOST_KEY_CHECKING: "false" | |
| # Set default values for scheduled runs | |
| DEPLOYMENT_SIZE: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.deployment_size || 'small' }} | |
| PR_OR_BRANCH: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.pr_or_branch || 'main' }} | |
| DEBUG_PAUSE: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.debug_pause || '0' }} | |
| CONTROLLER_ADDRESS: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.controller_address || 'qa.nexodus.io' }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Determine if pr_or_branch is a PR number | |
| id: check_pr | |
| run: | | |
| if [[ "${{ github.event.inputs.pr_or_branch }}" =~ ^[0-9]+$ ]]; then | |
| echo "is_pr=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "is_pr=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Fetch and checkout PR | |
| if: steps.check_pr.outputs.is_pr == 'true' | |
| run: | | |
| git fetch origin pull/${{ github.event.inputs.pr_or_branch }}/head:pr-${{ github.event.inputs.pr_or_branch }} | |
| git checkout pr-${{ github.event.inputs.pr_or_branch }} | |
| - name: Checkout branch | |
| if: steps.check_pr.outputs.is_pr == 'false' | |
| run: git checkout ${{ github.event.inputs.pr_or_branch }} | |
| - name: Setup Go | |
| uses: ./.github/actions/setup-go-env | |
| - name: Build | |
| run: | | |
| make dist/nexd-linux-amd64 | |
| make dist/nexctl-linux-amd64 | |
| - name: Configure aws credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| role-to-assume: ${{ secrets.AWS_ROLE }} | |
| role-session-name: nexodus-ci-deploy | |
| aws-region: us-east-1 | |
| - name: Copy binaries to s3 | |
| run: | | |
| aws s3 cp ./dist/nexd-linux-amd64 s3://nexodus-io/ec2-e2e/qa/ | |
| aws s3 cp ./dist/nexctl-linux-amd64 s3://nexodus-io/ec2-e2e/qa/ | |
| - name: Set Controller and Keycloak Credentials | |
| run: | | |
| # Set controller address to default if not provided in event inputs | |
| controller_address=${{ github.event.inputs.controller_address || 'qa.nexodus.io' }} | |
| echo "controller_address=${controller_address}" >> "$GITHUB_ENV" | |
| controller_auth_address="auth.$controller_address" | |
| echo "CONTROLLER_AUTH_ADDRESS=$controller_auth_address" >> "$GITHUB_ENV" | |
| # Set Keycloak credentials based on controller address | |
| if [ "$controller_address" = "qa.nexodus.io" ]; then | |
| echo "KC_USERNAME=${{ secrets.KC_QA_USERNAME }}" >> "$GITHUB_ENV" | |
| echo "KC_PASSWORD=${{ secrets.KC_QA_PASSWORD }}" >> "$GITHUB_ENV" | |
| elif [ "$controller_address" = "try.nexodus.io" ]; then | |
| echo "KC_USERNAME=${{ secrets.KC_PROD_USERNAME }}" >> "$GITHUB_ENV" | |
| echo "KC_PASSWORD=${{ secrets.KC_PROD_PASSWORD }}" >> "$GITHUB_ENV" | |
| else | |
| # Default to qa.nexodus.io credentials if controller_address is not specified | |
| echo "KC_USERNAME=${{ secrets.KC_QA_USERNAME }}" >> "$GITHUB_ENV" | |
| echo "KC_PASSWORD=${{ secrets.KC_QA_PASSWORD }}" >> "$GITHUB_ENV" | |
| fi | |
| - name: Build Keycloak Tool | |
| run: | | |
| go build -o ./ ./hack/e2e-scripts/kctool/ | |
| - name: Create a Keycloak User | |
| id: kc-user | |
| run: | | |
| output=$(./kctool --create-user \ | |
| -ku "${{ env.KC_USERNAME }}" \ | |
| -kp "${{ env.KC_PASSWORD }}" \ | |
| -u qa \ | |
| -p "${{ secrets.QA_USER_PASSWORD }}" \ | |
| "https://${{ env.CONTROLLER_AUTH_ADDRESS }}") | |
| echo "USER=$output" >> "$GITHUB_OUTPUT" | |
| - name: User results from Keycloak | |
| run: echo "User is ${{ steps.kc-user.outputs.USER }}" | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.10' | |
| - name: Install Ansible and Dependencies | |
| run: pip3.10 install boto boto3 ansible-vault ansible-core | |
| - name: Install amazon.aws Ansible library | |
| run: ansible-galaxy collection install amazon.aws | |
| - name: Set Default Deployment Size if Not Provided | |
| if: github.event_name == 'schedule' || github.event.inputs.deployment_size == '' | |
| run: | | |
| echo "${{ secrets.ANSIBLE_VARS_SMALL_QA }}" > ./ops/ansible/aws/vars.yml | |
| - name: Set Deployment Size to Small | |
| if: github.event.inputs.deployment_size == 'small' | |
| run: | | |
| echo "${{ secrets.ANSIBLE_VARS_SMALL_QA }}" > ./ops/ansible/aws/vars.yml | |
| - name: Set Deployment Size to Medium | |
| if: github.event.inputs.deployment_size == 'medium' | |
| run: | | |
| echo "${{ secrets.ANSIBLE_VARS_MEDIUM_QA }}" > ./ops/ansible/aws/vars.yml | |
| - name: Set Deployment Size to Large | |
| if: github.event.inputs.deployment_size == 'large' | |
| run: | | |
| echo "${{ secrets.ANSIBLE_VARS_LARGE_QA }}" > ./ops/ansible/aws/vars.yml | |
| - name: Set Deployment Size to XLarge | |
| if: github.event.inputs.deployment_size == 'xlarge' | |
| run: | | |
| echo "${{ secrets.ANSIBLE_VARS_XLARGE_QA }}" > ./ops/ansible/aws/vars.yml | |
| - name: Create Ansible Secrets | |
| run: | | |
| echo "${{ secrets.ANSIBLE_SSH_KEY }}" > nexodus.pem | |
| chmod 0400 nexodus.pem | |
| echo "${{ secrets.ANSIBLE_VAULT_PASSWORD }}" > vault-secret.txt | |
| chmod 0400 vault-secret.txt | |
| - name: Deploy EC2 Agent Nodes | |
| run: | | |
| ansible-playbook -vv ./ops/ansible/aws/deploy-ec2-qa.yml \ | |
| -i ./ops/ansible/aws/inventory.txt \ | |
| --private-key nexodus.pem \ | |
| --vault-password-file vault-secret.txt \ | |
| --extra-vars "nexodus_auth_uid=${{ steps.kc-user.outputs.USER }}" \ | |
| --extra-vars "nexodus_auth_password=${{ secrets.QA_USER_PASSWORD }}" \ | |
| --extra-vars "debug_pause=0" \ | |
| --extra-vars "controller_address=$controller_address" | |
| - name: Mesh Connectivity Results | |
| run: | | |
| set -e | |
| /bin/sh -c 'cat ./ops/ansible/aws/*-connectivity-results.txt > ./ops/ansible/aws/mesh-connectivity-results.txt' | |
| cat ./ops/ansible/aws/mesh-connectivity-results.txt | |
| if grep -iq 'Unreachable' ./ops/ansible/aws/mesh-connectivity-results.txt || grep -iq 'Failed' ./ops/ansible/aws/mesh-connectivity-results.txt; then | |
| echo "Connectivity results contain 'Unreachable or Failed' nodes, check the connectivity results and artifacts for details. Failing the job" | |
| exit 1 | |
| else | |
| echo "Connectivity results do not contain any 'Unreachable' nodes" | |
| fi | |
| - name: Terminate EC2 Instances | |
| if: always() | |
| run: | | |
| ansible-playbook -vv ./ops/ansible/aws/terminate-instances.yml \ | |
| --private-key nexodus.pem \ | |
| --vault-password-file vault-secret.txt | |
| - name: Upload nexd and api-server Logs to Artifacts | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: dev-ec2-artifacts | |
| path: ./ops/ansible/aws/nexd-logs/ | |
| retention-days: 10 |