deploy-iac.yml

# This workflow will deploy all the Azure services, including AKS, KV & secrets, MYSQL (eventually with a Firewall rule to allow your workstation IP)
# eventually if DEPLOY_TO_VNET is set to true : also VNet, AKS deployed to VNet, private DNS-Zone, client VM deployed to the VNet

name: Deploy IaC with Azure Bicep

env:

  # ==== Versions ====

  DEPLOYMENT_VERSION: 2.6.13
  AZ_CLI_VERSION: 2.45.0
  JAVA_VERSION: 11

  # ==== General settings  ====

  APP_NAME: tap42
  LOCATION: westeurope # francecentral
  RG_KV: rg-kv-tanzu101 # RG where to deploy KV
  RG_APP: rg-aks-tap-apps # RG where to deploy the other Azure services: AKS, TAP, ACR, MySQL, etc.

  # DNS  
  DNS_ZONE: cloudapp.azure.com
  APP_DNS_ZONE: tap.westeurope.cloudapp.azure.com
  CUSTOM_DNS: appinnohandsonlab.com # set here your own domain name
  AZURE_DNS_LABEL_NAME: petclinic

  TAP_NAMESPACE: tanzu
  PETCLINIC_NAMESPACE: petclinic
  AKS_CLUSTER_NAME: aks-tap42
  VNET_NAME: vnet-aks

  DNS_PREFIX: tanzu-tap42 # customize this param
  ING_NS: ingress # Namespace to use for the Ingress Controller
  AKS_IDENTITY_NAME: id-aks-tap42-cluster-dev-westeurope-101 # customize this , MUST BE 'id-aks-${appName}-cluster-dev-${location}-101'

  # ==== Identities ====:
  CUSTOMERS_SVC_APP_ID_NAME: id-aks-tap42-petclinic-customers-service-dev-westeurope-101 # customize this, MUST BE 'id-aks-${appName}-petclinic-customers-service-dev-${location}-101'
  VETS_SVC_APP_ID_NAME: id-aks-tap42-petclinic-vets-service-dev-westeurope-101 # customize this, MUST BE 'id-aks-${appName}-petclinic-vets-service-dev-${location}-101'
  VISITS_SVC_APP_ID_NAME: id-aks-tap42-petclinic-visits-service-dev-westeurope-101 # customize this, MUST BE 'id-aks-${appName}-petclinic-visits-service-dev-${location}-101'
  CONFIG_SERVER_APP_ID_NAME: id-aks-tap42-petclinic-config-server-dev-westeurope-101 # customize this, MUST BE  'id-aks-${appName}-petclinic-config-server-dev-${location}-101'
 
   # MySQL
  MYSQL_SERVER_NAME: tap42
  MYSQL_DB_NAME: petclinic
  MYSQL_ADM_USR: mys_adm
  MYSQL_TIME_ZONE: Europe/Paris
  MYSQL_CHARACTER_SET: utf8
  MYSQL_COLLATION: utf8_general_ci 
  MYSQL_PORT: 3306
  MYSQL_VERSION: "5.7" # "8.0.21"

   # SKU common for MySQL & PG
  DB_SKU_NAME: Standard_B1ms
  DB_SKU_TIER : Burstable

  # PG
  PG_SERVER_NAME: tap42
  PG_DB_NAME: tap
  PG_ADM_USR: pgs_adm
  PG_TIME_ZONE: Europe/Paris
  PG_CHARACTER_SET: utf8
  PG_COLLATION: fr_FR.utf8 # select * from pg_collation ;
  PG_PORT: 5432
  PG_VERSION: "13"

  ################################## DO NOT CHANGE params below   ##################################

  # APPLICATION INSIGHTS
  APPLICATIONINSIGHTS_CONFIGURATION_FILE: BOOT-INF/classes/applicationinsights.json
  
  # ==== APPS ====
  
  CLOUD_PROVIDER_ENV: azure
  PRJ_PREFIX: spring-petclinic

  API_GATEWAY: api-gateway
  ADMIN_SERVER: admin-server
  CUSTOMERS_SERVICE: customers-service
  VETS_SERVICE: vets-service
  VISITS_SERVICE: visits-service

  CONFIG_SERVER: config-server
  DISCOVERY_SERVER: discovery-server

  # https://github.com/Azure/actions-workflow-samples/blob/master/assets/create-secrets-for-GitHub-workflows.md#consume-secrets-in-your-workflow
  # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#example-using-secrets

  # ==== Secrets ====

  # https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions
  # Never use structured data as a secret
  # Structured data can cause secret redaction within logs to fail, because redaction largely relies on finding an exact match for 
  # the specific secret value. For example, do not use a blob of JSON, XML, or YAML (or similar) to encapsulate a secret value, 
  # as this significantly reduces the probability the secrets will be properly redacted. Instead, create individual secrets for each sensitive value.

  SPRING_DATASOURCE_PASSWORD: ${{ secrets.SPRING_DATASOURCE_PASSWORD }}
  SPRING_CLOUD_AZURE_TENANT_ID: ${{ secrets.SPRING_CLOUD_AZURE_TENANT_ID }}
  
  VM_ADMIN_PASSWORD: ${{ secrets.VM_ADMIN_PASSWORD }}

  SSH_PRV_KEY: ${{ secrets.SSH_PRV_KEY }}
  SSH_PUB_KEY: ${{ secrets.SSH_PUB_KEY }}
  SSH_KEY: aksadm

  credentials: ${{ secrets.AZURE_CREDENTIALS }}
  AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
  AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
  AAD_ADM_GRP: ${{ secrets.AAD_ADM_GRP }}

on:
  workflow_dispatch:

jobs:
 
  call-pre-req-workflow:
    name: Trigger Pre-Req
    uses: ./.github/workflows/deploy-iac-pre-req.yml
    secrets: inherit

  deploy-iac:
    needs: call-pre-req-workflow
    runs-on: ubuntu-latest
            
    steps:

    - name: Login with GHA Runner SP
      uses: azure/login@v1 # https://github.com/marketplace/actions/azure-login
      with:
        creds: ${{ secrets.AZURE_CREDENTIALS }} # ${{ env.credentials }}

    - name: Set Base environment variables
      run: |
        
        echo "ADMIN_SERVER_FOLDER=${{ env.PRJ_PREFIX }}-${{ env.ADMIN_SERVER }}" >> $GITHUB_ENV
        echo "DISCOVERY_SERVER_FOLDER=${{ env.PRJ_PREFIX }}-${{ env.DISCOVERY_SERVER }}" >> $GITHUB_ENV

        echo "API_GATEWAY_FOLDER=${{ env.PRJ_PREFIX }}-${{ env.API_GATEWAY }}" >> $GITHUB_ENV
        echo "CONFIG_SERVER_FOLDER=${{ env.PRJ_PREFIX }}-${{ env.CONFIG_SERVER }}" >> $GITHUB_ENV
        echo "CUSTOMERS_SERVICE_FOLDER=${{ env.PRJ_PREFIX }}-${{ env.CUSTOMERS_SERVICE }}" >> $GITHUB_ENV
        echo "VETS_SERVICE_FOLDER=${{ env.PRJ_PREFIX }}-${{ env.VETS_SERVICE }}" >> $GITHUB_ENV
        echo "VISITS_SERVICE_FOLDER=${{ env.PRJ_PREFIX }}-${{ env.VISITS_SERVICE }}" >> $GITHUB_ENV

        storage_name=$(az deployment group show --name storage -g ${{ env.RG_APP }} --query properties.outputs.azurestorageName.value -o tsv)
        echo "storage_name=" $storage_name
        echo "AZ_STORAGE_NAME="$storage_name >> $GITHUB_ENV

        blobContainerName=$(az deployment group show --name storage -g ${{ env.RG_APP }} --query properties.outputs.blobcontainerName.value -o tsv)
        echo "blobContainerName=" $blobContainerName
        echo "AZ_BLOB_CONTAINER_NAME="$blobContainerName >> $GITHUB_ENV

        MYSQL_SERVER_NAME=$(az deployment group show --name mysqldb -g ${{ env.RG_APP }} --query properties.outputs.mySQLServerName.value -o tsv)
        echo "MYSQL_SERVER_NAME="$MYSQL_SERVER_NAME
        echo "MYSQL_SERVER_NAME="$MYSQL_SERVER_NAME >> $GITHUB_ENV

        PG_SERVER_NAME=$(az deployment group show --name postgresqldb -g ${{ env.RG_APP }} --query properties.outputs.postgreSQLServerName.value -o tsv)
        echo "PG_SERVER_NAME="$PG_SERVER_NAME
        echo "PG_SERVER_NAME="$PG_SERVER_NAME >> $GITHUB_ENV

        REGISTRY_URL=$(az deployment group show --name acr -g ${{ env.RG_APP }} --query properties.outputs.acrRegistryUrl.value -o tsv)
        echo "REGISTRY_URL="$REGISTRY_URL
        echo "REGISTRY_URL="$REGISTRY_URL >> $GITHUB_ENV

        AZURE_CONTAINER_REGISTRY=$(az deployment group show --name acr -g ${{ env.RG_APP }} --query properties.outputs.acrName.value -o tsv)
        echo "AZURE_CONTAINER_REGISTRY="$AZURE_CONTAINER_REGISTRY
        echo "AZURE_CONTAINER_REGISTRY="$AZURE_CONTAINER_REGISTRY >> $GITHUB_ENV

        KV_NAME=$(az deployment group show --name aks-tap-kv -g ${{ env.RG_KV }} --query properties.outputs.keyVaultName.value -o tsv)
        echo "KV_NAME=$KV_NAME" >> $GITHUB_ENV 
        echo "KV_NAME=$KV_NAME"

        SPRING_CLOUD_AZURE_KEY_VAULT_ENDPOINT=$(az deployment group show --name aks-tap-kv -g ${{ env.RG_KV }} --query properties.outputs.keyVaultURI.value -o tsv)
        echo "SPRING_CLOUD_AZURE_KEY_VAULT_ENDPOINT=$SPRING_CLOUD_AZURE_KEY_VAULT_ENDPOINT" >> $GITHUB_ENV 
        echo "SPRING_CLOUD_AZURE_KEY_VAULT_ENDPOINT=$SPRING_CLOUD_AZURE_KEY_VAULT_ENDPOINT"

        echo "GH_WORKSPACE=${{ github.workspace }}" >> $GITHUB_ENV # "/github/workspace"
        echo "LOCAL_IP=$(curl whatismyip.akamai.com)" >> $GITHUB_ENV
        
      shell: bash

    - name: Display environment variables
      run: |

        echo "Checking GITHUB_ENV"
        
        echo "ADMIN_SERVER_FOLDER=$ADMIN_SERVER_FOLDER"
        echo "DISCOVERY_SERVER_FOLDER=$DISCOVERY_SERVER_FOLDER"

        echo "API_GATEWAY_FOLDER=$API_GATEWAY_FOLDER"
        echo "CONFIG_SERVER_FOLDER=$CONFIG_SERVER_FOLDER"
        echo "CUSTOMERS_SERVICE_FOLDER=$CUSTOMERS_SERVICE_FOLDER"
        echo "VETS_SERVICE_FOLDER=$VETS_SERVICE_FOLDER"
        echo "VISITS_SERVICE_FOLDER=$VISITS_SERVICE_FOLDER"

        echo "AZ_STORAGE_NAME="$AZ_STORAGE_NAME
        echo "AZ_BLOB_CONTAINER_NAME=$AZ_BLOB_CONTAINER_NAME
        echo "MYSQL_SERVER_NAME=$MYSQL_SERVER_NAME
        echo "PG_SERVER_NAME="$PG_SERVER_NAME
        echo "REGISTRY_URL="$REGISTRY_URL
        echo "AZURE_CONTAINER_REGISTRY="$AZURE_CONTAINER_REGISTRY
        echo "KV_NAME=$KV_NAME"
        echo "SPRING_CLOUD_AZURE_KEY_VAULT_ENDPOINT=$SPRING_CLOUD_AZURE_KEY_VAULT_ENDPOINT"

        echo "GH_WORKSPACE=$GH_WORKSPACE"
        echo "LOCAL_IP=$LOCAL_IP"

      shell: bash

    - name: Checkout
      uses: actions/checkout@v3 # https://github.com/actions/checkout

    - name: Create AKS cluster
      run: |

          echo "****************************************************************************************"
          echo "*                                                                                      *"
          echo "*                                                                                      *"
          echo "*About to create AKS cluster                                                           *"
          echo "*                                                                                      *"         
          echo "*                                                                                      *"
          echo "****************************************************************************************"

          az deployment group create --name aks-main -f iac/bicep/main.bicep -g ${{ env.RG_APP }} \
            -p appName=${{ env.APP_NAME }} \
            -p location=${{ env.LOCATION }} \
            -p vnetName=${{ env.VNET_NAME }} \
            -p dnsPrefix=${{ env.DNS_PREFIX }} \
            -p clusterName=${{ env.AKS_CLUSTER_NAME }} \
            -p aksIdentityName=${{ env.AKS_IDENTITY_NAME }} \
            -p adminGroupObjectIDs=${{ env.AAD_ADM_GRP }} \
            -p sshPublicKey="${{ env.SSH_PUB_KEY }}"
 
            CLUSTER_RESOURCE_GROUP=$(az aks show --resource-group ${{ env.RG_APP }} --name ${{ env.AKS_CLUSTER_NAME }} --query nodeResourceGroup -o tsv)
            echo "CLUSTER_RESOURCE_GROUP:" $CLUSTER_RESOURCE_GROUP
            echo "CLUSTER_RESOURCE_GROUP=$CLUSTER_RESOURCE_GROUP" >> $GITHUB_ENV

      shell: bash

    - name: Whitelist AKS OutboundIP
      run: |

            az config set extension.use_dynamic_install=yes_without_prompt

            echo "About to Whitelist AKS OutboundIP in KV & MySQL"

            vNetRules=$(az deployment group show --name vnet-aks -g ${{ env.RG_APP }} --query properties.outputs.aksSubnetId.value)

            aksOutboundIPResourceId=$(az deployment group show --name aks-main -g ${{ env.RG_APP }} \
              --query properties.outputs.aksEffectiveOutboundIPs.value[0] | jq -r .id)
            
            aksOutboundIP=$(az network public-ip show --ids $aksOutboundIPResourceId --query ipAddress)
            echo "aksOutboundIP:" $aksOutboundIP

            ipRules=[$aksOutboundIP]

            az deployment group create --name aks-kv-db-set-ip-rules -f iac/bicep/set-ip-rules.bicep -g ${{ env.RG_APP }} \
              -p appName=${{ env.APP_NAME }} \
              -p location=${{ env.LOCATION }} \
              -p kvName=${{ env.KV_NAME }} \
              -p kvRGName=${{ env.RG_KV }} \
              -p databaseSkuName=${{ env.DB_SKU_NAME }} \
              -p databaseSkuTier=${{ env.DB_SKU_TIER }} \
              -p mySqlDbName=${{ env.MYSQL_DB_NAME }} \
              -p mySqlVersion="${{ env.MYSQL_VERSION }}" \
              -p mySqlCharset=${{ env.MYSQL_CHARACTER_SET }} \
              -p mySqlCollation=${{ env.MYSQL_COLLATION }} \
              -p postgreSQLadministratorLogin=${{ env.PG_ADM_USR }} \
              -p pgDbName=${{ env.PG_DB_NAME }} \
              -p pgCharset=${{ env.PG_CHARACTER_SET }} \
              -p pgCollation=${{ env.PG_COLLATION }} \
              -p postgreSQLVersion=${{ env.PG_VERSION }} \
              -p azureStorageName=${{ env.AZ_STORAGE_NAME }} \
              -p blobContainerName=${{ env.AZ_BLOB_CONTAINER_NAME }} \
              -p vNetRules=[$vNetRules] \
              -p ipRules=$ipRules

      shell: bash

    - name: Get Apps Identity Id
      run: |

          # vetsServicePrincipalId=$(az deployment group show --name aks-identities -g ${{ env.RG_APP }} --query properties.outputs.vetsServicePrincipalId.value)
          vetsServiceClientId=$(az deployment group show --name aks-identities -g ${{ env.RG_APP }} --query properties.outputs.vetsServiceClientId.value)

          # visitsServicePrincipalId=$(az deployment group show --name aks-identities -g ${{ env.RG_APP }} --query properties.outputs.visitsServicePrincipalId.value)
          visitsServiceClientId=$(az deployment group show --name aks-identities -g ${{ env.RG_APP }} --query properties.outputs.visitsServiceClientId.value)

          # configServerPrincipalId=$(az deployment group show --name aks-identities -g ${{ env.RG_APP }} --query properties.outputs.configServerPrincipalId.value)
          configServerClientId=$(az deployment group show --name aks-identities -g ${{ env.RG_APP }} --query properties.outputs.configServerClientId.value)

          # customersServicePrincipalId=$(az deployment group show --name aks-identities -g ${{ env.RG_APP }} --query properties.outputs.customersServicePrincipalId.value)
          customersServiceClientId=$(az deployment group show --name aks-identities -g ${{ env.RG_APP }} --query properties.outputs.customersServiceClientId.value)

          # aksClusterPrincipalId=$(az deployment group show --name aks-identities -g ${{ env.RG_APP }} --query properties.outputs.aksIdentityPrincipalId.value)
          aksClusterClientId=$(az deployment group show --name aks-identities -g ${{ env.RG_APP }} --query properties.outputs.aksIdentityClientId.value)

          echo "vetsServiceClientId=$vetsServiceClientId" >> $GITHUB_ENV
          echo "visitsServiceClientId=$visitsServiceClientId" >> $GITHUB_ENV
          echo "customersServiceClientId=$customersServiceClientId" >> $GITHUB_ENV
          echo "configServerClientId=$configServerClientId" >> $GITHUB_ENV
          echo "aksClusterClientId=$aksClusterClientId" >> $GITHUB_ENV

    # https://github.com/Azure/aks-set-context/tree/releases/v1
    - name: AKS Set Context
      uses: azure/aks-set-context@v1
      with:
          creds: '${{ secrets.AZURE_CREDENTIALS }}' # Azure credentials
          resource-group: ${{ env.RG_APP }} 
          cluster-name: ${{ env.AKS_CLUSTER_NAME }}
      id: setakscontext

    # kubectl config set-credentials cicd-runner --token=bearer_token
    # TODO for AAD Integration:
    # https://github.com/marketplace/actions/setup-kubelogin
    # https://github.com/Azure/kubelogin
    - uses: azure/use-kubelogin@v1
      env:
        GITHUB_TOKEN: ${{ secrets.GH_PAT }}
      with:
        kubelogin-version: 'latest'
      id: kubelogin

    - name: K8S Test with AAD
      run: |

            kubectl cluster-info
            kubectl config get-contexts
            kubectl get nodes -o wide

      shell: bash

    - name: Create Petclinic Namespace
      run: |

            kubectl create namespace ${{ env.PETCLINIC_NAMESPACE }} --dry-run=client -o yaml > ns-petclinic.yaml
            kubectl apply -f ns-petclinic.yaml

      shell: bash

    - name: Create Tanzu Namespace
      run: |

            kubectl create namespace ${{ env.TAP_NAMESPACE }} --dry-run=client -o yaml > ns-tanzu.yaml
            kubectl apply -f ns-tanzu.yaml

      shell: bash

    - name: Create Dev & Ops Namespace
      run: |

            kubectl create namespace development
            kubectl label namespace/development purpose=development

            kubectl create namespace staging
            kubectl label namespace/staging purpose=staging

            kubectl create namespace production
            kubectl label namespace/production purpose=production

            kubectl create namespace sre
            kubectl label namespace/sre purpose=sre

            kubectl get namespaces
            kubectl describe namespace production
            kubectl describe namespace sre

      shell: bash

    # https://learn.microsoft.com/en-us/azure/aks/workload-identity-deploy-cluster
    # https://blog.identitydigest.com/azuread-federate-mi/
    - name: Configure AAD WI
      run: |

            # "$(az identity show --resource-group "${RG_NAME}" --name "${{ env.RG_APP }}" --query 'clientId' -o tsv)"

            mkdir $CUSTOMERS_SERVICE_FOLDER/k8s/deploy
            mkdir $VETS_SERVICE_FOLDER/k8s/deploy
            mkdir $VISITS_SERVICE_FOLDER/k8s/deploy
            mkdir $CONFIG_SERVER_FOLDER/k8s/deploy

            echo "Cheking folder " $CUSTOMERS_SERVICE_FOLDER
            ls -al $CUSTOMERS_SERVICE_FOLDER/k8s

            echo "Cheking folder " $VETS_SERVICE_FOLDER
            ls -al $VETS_SERVICE_FOLDER/k8s

            echo "Cheking folder " $VISITS_SERVICE_FOLDER
            ls -al $VISITS_SERVICE_FOLDER/k8s

            echo "Cheking folder " $CONFIG_SERVER_FOLDER
            ls -al $CONFIG_SERVER_FOLDER/k8s

            export AKS_OIDC_ISSUER="$(az aks show -n ${{ env.AKS_CLUSTER_NAME }} -g ${{ env.RG_APP }} --query "oidcIssuerProfile.issuerUrl" -o tsv)"
            export PETCLINIC_NS=${{ env.PETCLINIC_NAMESPACE }}
            export AUDIENCES="api://AzureADTokenExchange"

            echo ""
            echo "AKS_OIDC_ISSUER:" $AKS_OIDC_ISSUER
            echo ""

            export USER_ASSIGNED_CLIENT_ID=$customersServiceClientId
            export SERVICE_ACCOUNT_NAME="sa-aad-wi-customers"
            ls -al $CUSTOMERS_SERVICE_FOLDER/k8s
            envsubst < $CUSTOMERS_SERVICE_FOLDER/k8s/sa-petclinic.yaml > $CUSTOMERS_SERVICE_FOLDER/k8s/deploy/sa-customers.yaml
            echo "Cheking folder " $CUSTOMERS_SERVICE_FOLDER
            ls -al $CUSTOMERS_SERVICE_FOLDER/k8s/deploy
            cat $CUSTOMERS_SERVICE_FOLDER/k8s/deploy/sa-customers.yaml
            kubectl apply -f $CUSTOMERS_SERVICE_FOLDER/k8s/deploy/sa-customers.yaml -n ${{ env.PETCLINIC_NAMESPACE }}
            az identity federated-credential create --name customersServiceFedIdentity --identity-name "${{ env.CUSTOMERS_SVC_APP_ID_NAME }}" --issuer "${AKS_OIDC_ISSUER}" --subject system:serviceaccount:"${{ env.PETCLINIC_NAMESPACE }}":"${SERVICE_ACCOUNT_NAME}" -g "${{ env.RG_APP }}"
            
            export USER_ASSIGNED_CLIENT_ID=$vetsServiceClientId
            export SERVICE_ACCOUNT_NAME="sa-aad-wi-vets"
            ls -al $VETS_SERVICE_FOLDER/k8s
            envsubst < $VETS_SERVICE_FOLDER/k8s/sa-petclinic.yaml > $VETS_SERVICE_FOLDER/k8s/deploy/sa-vets.yaml
            echo "Cheking folder " $VETS_SERVICE_FOLDER
            ls -al $VETS_SERVICE_FOLDER/k8s/deploy
            cat $VETS_SERVICE_FOLDER/k8s/deploy/sa-vets.yaml
            kubectl apply -f $VETS_SERVICE_FOLDER/k8s/deploy/sa-vets.yaml -n ${{ env.PETCLINIC_NAMESPACE }}
            az identity federated-credential create --name vetsServiceFedIdentity --identity-name "${{ env.VETS_SVC_APP_ID_NAME }}" --issuer "${AKS_OIDC_ISSUER}" --subject system:serviceaccount:"${{ env.PETCLINIC_NAMESPACE }}":"${SERVICE_ACCOUNT_NAME}" -g "${{ env.RG_APP }}"

            export USER_ASSIGNED_CLIENT_ID=$visitsServiceClientId
            export SERVICE_ACCOUNT_NAME="sa-aad-wi-visits"
            ls -al $VISITS_SERVICE_FOLDER/k8s
            envsubst < $VISITS_SERVICE_FOLDER/k8s/sa-petclinic.yaml > $VISITS_SERVICE_FOLDER/k8s/deploy/sa-visits.yaml
            echo "Cheking folder " $VISITS_SERVICE_FOLDER
            ls -al $VISITS_SERVICE_FOLDER/k8s/deploy
            cat $VISITS_SERVICE_FOLDER/k8s/deploy/sa-visits.yaml
            kubectl apply -f $VISITS_SERVICE_FOLDER/k8s/deploy/sa-visits.yaml -n ${{ env.PETCLINIC_NAMESPACE }}
            az identity federated-credential create --name visitsServiceFedIdentity --identity-name "${{ env.VISITS_SVC_APP_ID_NAME }}" --issuer "${AKS_OIDC_ISSUER}" --subject system:serviceaccount:"${{ env.PETCLINIC_NAMESPACE }}":"${SERVICE_ACCOUNT_NAME}" -g "${{ env.RG_APP }}"

            export USER_ASSIGNED_CLIENT_ID=$configServerClientId
            export SERVICE_ACCOUNT_NAME="sa-aad-wi-cfg"
            ls -al $CONFIG_SERVER_FOLDER/k8s
            envsubst < $CONFIG_SERVER_FOLDER/k8s/sa-petclinic.yaml > $CONFIG_SERVER_FOLDER/k8s/deploy/sa-cfg.yaml
            echo "Cheking folder " $CONFIG_SERVER_FOLDER
            ls -al $CONFIG_SERVER_FOLDER/k8s/deploy
            cat $CONFIG_SERVER_FOLDER/k8s/deploy/sa-cfg.yaml
            kubectl apply -f $CONFIG_SERVER_FOLDER/k8s/deploy/sa-cfg.yaml -n ${{ env.PETCLINIC_NAMESPACE }}
            az identity federated-credential create --name configServerFedIdentity --identity-name "${{ env.CONFIG_SERVER_APP_ID_NAME }}" --issuer "${AKS_OIDC_ISSUER}" --subject system:serviceaccount:"${{ env.PETCLINIC_NAMESPACE }}":"${SERVICE_ACCOUNT_NAME}" -g "${{ env.RG_APP }}"

      shell: bash

    - name: setup Petclinic K8S Config
      run: |

            cat <<EOF >> cm-petclinic.yaml
            apiVersion: v1
            kind: ConfigMap
            metadata:
              name: springcloudazure
            data:
              SPRING_CLOUD_AZURE_TENANT_ID: ${{ env.SPRING_CLOUD_AZURE_TENANT_ID }}
              SPRING_CLOUD_AZURE_KEY_VAULT_ENDPOINT: "${{ env.SPRING_CLOUD_AZURE_KEY_VAULT_ENDPOINT }}"
              CUSTOMERS_SVC_APP_IDENTITY_CLIENT_ID: $customersServiceClientId
              VETS_SVC_APP_IDENTITY_CLIENT_ID: $vetsServiceClientId
              VISITS_SVC_APP_IDENTITY_CLIENT_ID: $visitsServiceClientId
              CONFIG_SERVER_APP_IDENTITY_CLIENT_ID: $configServerClientId
            EOF

            cat cm-petclinic.yaml
            kubectl apply -f cm-petclinic.yaml -n ${{ env.PETCLINIC_NAMESPACE }}

            appins_cs=$(az deployment group show --name aks-petclinic-pre-req -g ${{ env.RG_APP }} \
              --query properties.outputs.appInsightsConnectionString.value)

            cat <<EOF >> cm-appins.yaml
            apiVersion: v1
            kind: ConfigMap
            metadata:
              name: appins
            data:
              APPLICATIONINSIGHTS_CONNECTION_STRING: $appins_cs
              APPLICATIONINSIGHTS_CONFIGURATION_FILE: "${{ env.APPLICATIONINSIGHTS_CONFIGURATION_FILE }}"
            EOF

            cat cm-appins.yaml
            kubectl apply -f cm-appins.yaml -n ${{ env.PETCLINIC_NAMESPACE }}

      shell: bash

    # security hardening for self-hosted agents: https://github.com/marketplace/actions/azure-login
    # https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#hardening-for-self-hosted-runners
    # if the runner is self-hosted which is not github provided it is recommended to manually logout at the end of the workflow as shown below.
    - name: Azure Logout security hardening
      run: |
          az logout
          az cache purge
          az account clear
      shell: bash

  call-dns-cfg-workflow:
    name: Configure DNS
    needs: deploy-iac
    uses: ./.github/workflows/dns-cfg.yml
    secrets: inherit

  call-db-init-workflow:
    name: Load Data to DB
    needs: deploy-iac
    uses: ./.github/workflows/sql-load.yml
    secrets: inherit

  call-maven-build-workflow:
    name: Trigger Maven for backend services
    needs: call-pre-req-workflow
    uses: ./.github/workflows/maven-build.yml # .github/workflows/maven-build.yml@main ==> references to workflows must be prefixed with format 'owner/repository/' or './' for local workflows
    secrets: inherit # pass all secrets
      # envPAT: ${{ secrets.envPAT }} # pass just this secret
    permissions: 
      contents: read
      packages: write        

  call-maven-build-ui-workflow:
    name: Trigger Maven for the UI
    needs: call-pre-req-workflow
    uses: ./.github/workflows/maven-build-ui.yml
    secrets: inherit
    permissions: 
      contents: read
      packages: write  
      
  #call-tap-setup-workflow:
  #  name: Trigger TAP installation
  #  needs: deploy-iac
  #  uses: ./.github/workflows/tap-setup.yml
  #  secrets: inherit