Release 6.0 - Hello Oracle EBS, YouTube, GA4! (#82)

And goodbye submodules. One repository to rule them all! 👑

1_click.sh

@@ -19,8 +19,5 @@
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 
 pushd "${SCRIPT_DIR}" 1> /dev/null
-echo -n "Please wait..."
-git submodule update --init --recursive &> /dev/null || true
-echo -e -n "\r                \r"
 src/utils/interactive/interactive.sh "${1}" "${2}"
 popd 1> /dev/null

cloudbuild.yaml

@@ -20,13 +20,21 @@ steps:
     script: |
       #!/usr/bin/env bash
       set -e
+
+      # Save absolute config file path to .env file
+      # that can be accessed by all Cloud Build steps since exported env
+      # variables do not persist between steps.
+
+      realpath "${_CONFIG_FILE}" > /workspace/config_file_full_path.env
+
       echo "Initial configuration ${_CONFIG_FILE}:"
-      cat ${_CONFIG_FILE}
+      cat "${_CONFIG_FILE}"
 
       python3 src/common/init_deployment_config.py --config-file ${_CONFIG_FILE} \
             --sub-validator "src/SAP/SAP_REPORTING" \
             --sub-validator "src/SFDC/src" \
-            --sub-validator "src/marketing/src"
+            --sub-validator "src/marketing/src" \
+            --sub-validator "src/OracleEBS/src"
 
       echo "Processed configuration:"
       cat ${_CONFIG_FILE}
@@ -38,10 +46,25 @@ steps:
     script: |
       #!/usr/bin/env bash
       export PYTHONPATH=${PYTHONPATH}:src/:.
+      declare -a _WORKER_POOL_OPTIONS
+
+      if [[ -n "${_WORKER_POOL_NAME}" ]]; then
+        _WORKER_POOL_OPTIONS+=(--worker-pool-name "${_WORKER_POOL_NAME}")
+      fi
+
+      if [[ -n "${_CLOUD_BUILD_REGION}" ]]; then
+        _WORKER_POOL_OPTIONS+=(--region "${_CLOUD_BUILD_REGION}")
+      fi
+
+      if [[ -n "${_BUILD_ACCOUNT}" ]]; then
+        _WORKER_POOL_OPTIONS+=(--build-account "${_BUILD_ACCOUNT}")
+      fi
+
       python3 src/k9/src/deploy_k9.py \
-              --config-file ${_CONFIG_FILE} \
-              --stage pre \
-              --logs-bucket ${_GCS_BUCKET}
+            --config-file "${_CONFIG_FILE}" \
+            --stage pre \
+            --logs-bucket "${_GCS_BUCKET}" \
+            "${_WORKER_POOL_OPTIONS[@]}"
 
   - name: gcr.io/kittycorn-public/deploy-kittycorn:v2.0
     id: 'sap-test-harness'
@@ -84,7 +107,8 @@ steps:
         cp -f ${_CONFIG_FILE} src/SAP/SAP_CDC/config/config.json
         gcloud builds submit ./src/SAP/SAP_CDC \
           --config=./src/SAP/SAP_CDC/cloudbuild.cdc.yaml \
-          --substitutions=_GCS_BUCKET=${_GCS_BUCKET}
+          --substitutions=_GCS_BUCKET="${_GCS_BUCKET}",_WORKER_POOL_NAME="${_WORKER_POOL_NAME}",_CLOUD_BUILD_REGION="${_CLOUD_BUILD_REGION}",_BUILD_ACCOUNT="${_BUILD_ACCOUNT}" \
+          --region="${_CLOUD_BUILD_REGION}"
       else
         echo "==Skipping CDC for SAP=="
       fi
@@ -95,11 +119,12 @@ steps:
     script: |
       #!/usr/bin/env bash
       _DEPLOY_SAP_=$(jq -r ."deploySAP" ${_CONFIG_FILE})
-      if [[ ${_DEPLOY_SAP_} == "true" ]]; then
-        cp -f ${_CONFIG_FILE} src/SAP/SAP_REPORTING/config/config.json
+      if [[ "${_DEPLOY_SAP_}" == "true" ]]; then
+        cp -f "${_CONFIG_FILE}" src/SAP/SAP_REPORTING/config/config.json
         gcloud builds submit ./src/SAP/SAP_REPORTING \
           --config=./src/SAP/SAP_REPORTING/cloudbuild.reporting.yaml \
-          --substitutions=_GCS_BUCKET=${_GCS_BUCKET}
+          --substitutions=_GCS_BUCKET="${_GCS_BUCKET}",_WORKER_POOL_NAME="${_WORKER_POOL_NAME}",_CLOUD_BUILD_REGION="${_CLOUD_BUILD_REGION}",_BUILD_ACCOUNT="${_BUILD_ACCOUNT}" \
+          --region="${_CLOUD_BUILD_REGION}"
       else
         echo "==Skipping Reporting for SAP=="
       fi
@@ -109,6 +134,7 @@ steps:
     waitFor: ['sap-reporting']
     script: |
       #!/usr/bin/env bash
+      _DEPLOY_SAP_ML_MODELS=$(jq -r ."deploySAPMLModels" ${_CONFIG_FILE})
       if [[ ${_DEPLOY_SAP_ML_MODELS} == "true" ]]
       then
         _DEPLOY_SAP_=$(jq -r ."deploySAP" ${_CONFIG_FILE})
@@ -122,10 +148,10 @@ steps:
         _DS_RAW_=$(jq -r ."SAP.datasets.raw" ${_CONFIG_FILE})
         _DS_CDC_=$(jq -r ."SAP.datasets.cdc" ${_CONFIG_FILE})
 
-        if [[ ${_DEPLOY_SAP_} == "true" && ${_SQL_FLAVOUR_} != "union" ]]; then
+        if [[ "${_DEPLOY_SAP_}" == "true" && "${_SQL_FLAVOUR_}" != "union" ]]; then
           gcloud builds submit ./src/SAP/SAP_ML_MODELS \
             --config=./src/SAP/SAP_ML_MODELS/cloudbuild.models.yaml \
-            --substitutions=_PJID_SRC=${_PJID_SRC_},_PJID_TGT=${_PJID_TGT_},_DS_RAW=${_DS_RAW_},_DS_CDC=${_DS_CDC_},_DS_REPORTING=${_DS_REPORTING_},_DS_MODELS=${_DS_MODELS_},_SQL_FLAVOUR=${_SQL_FLAVOUR_},_LOCATION=${_LOCATION_},_MANDT=${_MANDT_},_GCS_BUCKET=${_GCS_BUCKET}
+            --substitutions=_PJID_SRC="${_PJID_SRC_}",_PJID_TGT="${_PJID_TGT_}",_DS_RAW="${_DS_RAW_}",_DS_CDC="${_DS_CDC_}",_DS_REPORTING="${_DS_REPORTING_}",_DS_MODELS="${_DS_MODELS_}",_SQL_FLAVOUR="${_SQL_FLAVOUR_}",_LOCATION="${_LOCATION_}",_MANDT="${_MANDT_}",_GCS_BUCKET="${_GCS_BUCKET}",_WORKER_POOL_NAME="${_WORKER_POOL_NAME}",_CLOUD_BUILD_REGION="${_CLOUD_BUILD_REGION}",_BUILD_ACCOUNT="${_BUILD_ACCOUNT}" --region="${_CLOUD_BUILD_REGION}"
         else
           echo "==Skipping ML for SAP=="
         fi
@@ -138,12 +164,13 @@ steps:
     waitFor: ['sfdc-test-harness', 'k9-pre']
     script: |
       #!/usr/bin/env bash
-      _DEPLOY_SFDC_=$(jq -r ."deploySFDC" ${_CONFIG_FILE})
-      if [[ ${_DEPLOY_SFDC_} == "true" ]]; then
-        cp -f ${_CONFIG_FILE} src/SFDC/config/config.json
+      _DEPLOY_SFDC_="$(jq -r ."deploySFDC" "${_CONFIG_FILE}")"
+      if [[ "${_DEPLOY_SFDC_}" == "true" ]]; then
+        cp -f "${_CONFIG_FILE}" src/SFDC/config/config.json
         gcloud builds submit ./src/SFDC \
           --config=./src/SFDC/cloudbuild.sfdc.yaml \
-          --substitutions=_GCS_BUCKET=${_GCS_BUCKET}
+          --substitutions=_GCS_BUCKET="${_GCS_BUCKET}",_WORKER_POOL_NAME="${_WORKER_POOL_NAME}",_CLOUD_BUILD_REGION="${_CLOUD_BUILD_REGION}",_BUILD_ACCOUNT="${_BUILD_ACCOUNT}" \
+          --region="${_CLOUD_BUILD_REGION}"
       else
         echo "==Skipping SFDC=="
       fi
@@ -162,6 +189,8 @@ steps:
         _DEPLOY_LIVERAMP_=$(jq -r ."marketing.deployLiveRamp" ${_CONFIG_FILE})
         _DEPLOY_META_=$(jq -r ."marketing.deployMeta" ${_CONFIG_FILE})
         _DEPLOY_SFMC_=$(jq -r ."marketing.deploySFMC" ${_CONFIG_FILE})
+        _DEPLOY_DV360_=$(jq -r ."marketing.deployDV360" ${_CONFIG_FILE})
+        _DEPLOY_GA4_=$(jq -r ."marketing.deployGA4" ${_CONFIG_FILE})
         if [[ ${_DEPLOY_ADS_} == "true" ]]; then
           echo "Deploying Google Ads Test Harness."
           python3 src/common/create_test_harness.py --workload "marketing.GoogleAds" --dataset "raw"
@@ -186,6 +215,14 @@ steps:
           echo "Deploying SFMC Test Harness."
           python3 src/common/create_test_harness.py --workload "marketing.SFMC" --dataset "raw"
         fi
+        if [[ ${_DEPLOY_DV360_} == "true" ]]; then
+          echo "Deploying DV360 Test Harness."
+          python3 src/common/create_test_harness.py --workload "marketing.DV360" --dataset "raw"
+        fi
+        if [[ ${_DEPLOY_GA4_} == "true" ]]; then
+          echo "Deploying Google Analytics 4 Test Harness."
+          python3 src/common/create_test_harness.py --workload "marketing.GA4" --dataset "cdc"
+        fi
       else
         echo "==Skipping Test Harness for Marketing=="
       fi
@@ -195,55 +232,83 @@ steps:
     waitFor: ['k9-pre', 'marketing-test-harness']
     script: |
       #!/usr/bin/env bash
-      _DEPLOY_MARKETING_=$(jq -r ."deployMarketing" ${_CONFIG_FILE})
-      if [[ ${_DEPLOY_MARKETING_} == "true" ]]; then
-        cp -f ${_CONFIG_FILE} src/marketing/config/config.json
+      _DEPLOY_MARKETING_="$(jq -r ."deployMarketing" "${_CONFIG_FILE}")"
+      if [[ "${_DEPLOY_MARKETING_}" == "true" ]]; then
+        cp -f "${_CONFIG_FILE}" src/marketing/config/config.json
         gcloud builds submit ./src/marketing \
           --config=./src/marketing/cloudbuild.marketing.yaml \
-          --substitutions=_GCS_LOGS_BUCKET=${_GCS_BUCKET}
+          --substitutions=_GCS_LOGS_BUCKET="${_GCS_BUCKET}",_WORKER_POOL_NAME="${_WORKER_POOL_NAME}",_CLOUD_BUILD_REGION="${_CLOUD_BUILD_REGION}",_BUILD_ACCOUNT="${_BUILD_ACCOUNT}" \
+          --region="${_CLOUD_BUILD_REGION}"
       else
         echo "==Skipping Marketing=="
       fi
 
   - name: gcr.io/kittycorn-public/deploy-kittycorn:v2.0
-    id: 'k9-post'
-    waitFor: ['k9-pre', 'sap-reporting', 'sfdc-deploy', 'marketing-deploy']
+    id: 'oracle-ebs-test-harness'
+    waitFor: ['init_deploy_config']
     script: |
       #!/usr/bin/env bash
-      export PYTHONPATH=${PYTHONPATH}:src/:.
-      python3 src/k9/src/deploy_k9.py \
-              --config-file ${_CONFIG_FILE} \
-              --stage post \
-              --logs-bucket ${_GCS_BUCKET}
+      _DEPLOY_EBS_=$(jq -r ."deployOracleEBS" ${_CONFIG_FILE})
+      _TEST_DATA_=$(jq -r ."testData" ${_CONFIG_FILE})
 
-  - name: gcr.io/kittycorn-public/credly:latest
-    id: 'credly-badge'
-    waitFor: ['k9-pre', 'sap-reporting', 'sfdc-deploy']
+      if [[ ${_NO_TEST_DATA} != "true" && ${_TEST_DATA_} == "true" && ${_DEPLOY_EBS_} == "true" ]]; then
+        echo "Deploying Oracle EBS Test Harness."
+        python3 src/common/create_test_harness.py --workload "OracleEBS" --dataset "cdc"
+      else
+        echo "==Skipping Test Harness for Oracle=="
+      fi
+
+  - name: gcr.io/kittycorn-public/deploy-kittycorn:v2.0
+    id: 'oracle-ebs-deploy'
+    waitFor: ['k9-pre', 'oracle-ebs-test-harness']
     script: |
       #!/usr/bin/env bash
-      _FLAG_=$(jq -r ."shareWithCredly" ${_CONFIG_FILE})
-      if [[ ${_FLAG_} != "true" ]]; then
-        echo "===Skipping assigning a badge==="
+      _DEPLOY_EBS_=$(jq -r ."deployOracleEBS" ${_CONFIG_FILE})
+      if [[ ${_DEPLOY_EBS_} == "true" ]]; then
+        cp -f ${_CONFIG_FILE} src/OracleEBS/config/config.json
+        gcloud builds submit ./src/OracleEBS \
+          --config=./src/OracleEBS/cloudbuild.oracle_ebs.yaml \
+          --substitutions=_GCS_LOGS_BUCKET="${_GCS_BUCKET}",_WORKER_POOL_NAME="${_WORKER_POOL_NAME}",_CLOUD_BUILD_REGION="${_CLOUD_BUILD_REGION}",_BUILD_ACCOUNT="${_BUILD_ACCOUNT}" \
+          --region="${_CLOUD_BUILD_REGION}"
       else
-        _EMAIL_=$(jq -r ."userInfo.email" ${_CONFIG_FILE})
-        _FN_=$(jq -r ."userInfo.firstName" ${_CONFIG_FILE})
-        _LN_=$(jq -r ."userInfo.lastName" ${_CONFIG_FILE})
-        cd /usr/src/app
-        if [ -z ${_EMAIL_} ] || [ -z ${_FN_} ] || [ -z ${_LN_} ] || ! [ -f credly.py ]; then
-          echo "===Email, first name and last name are required in config.json==="
-          echo "===Skipping assigning a badge==="
-        else
-          python3 credly.py --fn ${_FN_} --ln ${_LN_} --email ${_EMAIL_} || exit 0
-        fi
+        echo "==Skipping Oracle EBS=="
+      fi
+
+  - name: gcr.io/kittycorn-public/deploy-kittycorn:v2.0
+    id: 'k9-post'
+    waitFor: ['k9-pre', 'sap-reporting', 'sfdc-deploy', 'marketing-deploy', 'oracle-ebs-deploy']
+    script: |
+      #!/usr/bin/env bash
+      export PYTHONPATH=${PYTHONPATH}:src/:.
+      declare -a _WORKER_POOL_OPTIONS
+
+      if [[ -n "${_WORKER_POOL_NAME}" ]]; then
+        _WORKER_POOL_OPTIONS+=(--worker-pool-name "${_WORKER_POOL_NAME}")
+      fi
+
+      if [[ -n "${_CLOUD_BUILD_REGION}" ]]; then
+        _WORKER_POOL_OPTIONS+=(--region "${_CLOUD_BUILD_REGION}")
+      fi
+
+      if [[ -n "${_BUILD_ACCOUNT}" ]]; then
+        _WORKER_POOL_OPTIONS+=(--build-account "${_BUILD_ACCOUNT}")
       fi
 
+      python3 src/k9/src/deploy_k9.py \
+            --config-file "${_CONFIG_FILE}" \
+            --stage post \
+            --logs-bucket "${_GCS_BUCKET}" \
+            "${_WORKER_POOL_OPTIONS[@]}"
+
 logsBucket: "gs://${_GCS_BUCKET}"
+serviceAccount: "${_BUILD_ACCOUNT}"
 timeout: 32400s
 substitutions:
   _CONFIG_FILE: "config/config.json"
-  _DEPLOY_SAP_ML_MODELS: "false"
   _NO_TEST_DATA: "false"
 options:
   substitution_option: "ALLOW_LOOSE"
   automapSubstitutions: true
+  pool:
+    name: "${_WORKER_POOL_NAME}"
 tags: ["cortex"]

config/config.json

@@ -3,19 +3,14 @@
     "deploySAP": true,
     "deploySFDC": true,
     "deployMarketing": true,
+    "deployOracleEBS": true,
     "deployDataMesh": true,
+    "deploySAPMLModels": false,
     "turboMode": true,
     "projectIdSource": "",
     "projectIdTarget": "",
     "targetBucket": "",
     "location": "US",
-    "languages": [
-        "E",
-        "S"
-    ],
-    "currencies": [
-        "USD"
-    ],
     "testDataProject": "kittycorn-public",
     "k9": {
         "datasets": {
@@ -31,19 +26,29 @@
     },
     "SAP": {
         "deployCDC": true,
+        "SQLFlavor": "ecc",
+        "mandt": "100",
+        "languages": [
+            "E",
+            "S"
+        ],
+        "currencies": [
+            "USD"
+        ],
         "datasets": {
             "cdc": "",
             "raw": "",
             "ml": "ML_MODELS",
             "reporting": "REPORTING"
-        },
-        "SQLFlavor": "ecc",
-        "mandt": "100"
+        }
     },
     "SFDC": {
         "deployCDC": true,
         "createMappingViews": true,
         "createPlaceholders": true,
+        "currencies": [
+            "USD"
+        ],
         "datasets": {
             "cdc": "",
             "raw": "",
@@ -57,6 +62,8 @@
         "deployLiveRamp": true,
         "deployMeta": true,
         "deploySFMC": true,
+        "deployDV360": true,
+        "deployGA4": true,
         "dataflowRegion": "",
         "GoogleAds": {
             "deployCDC": true,
@@ -105,12 +112,32 @@
                 "raw": "",
                 "reporting": "REPORTING_SFMC"
             }
+        },
+        "DV360": {
+            "deployCDC": true,
+            "datasets": {
+                "raw": "",
+                "cdc": "",
+                "reporting": "REPORTING_DV360"
+            }
+        },
+        "GA4": {
+            "datasets": {
+                "cdc": [
+                    {"property_id": 0, "name": ""}
+                ],
+                "reporting": "REPORTING_GA4"
+            }
         }
     },
-    "shareWithCredly": false,
-    "userInfo": {
-        "email": "",
-        "firstName": "",
-        "lastName": ""
+    "OracleEBS": {
+        "itemCategorySetIds": [1100000425],
+        "currencyConversionType": "Corporate",
+        "currencyConversionTargets": ["USD"],
+        "languages": ["US"],
+        "datasets": {
+            "cdc": "",
+            "reporting": "REPORTING_OracleEBS"
+        }
     }
 }