Merge branch 'patch/update-dbt-node-selection' of https://github.com/…

…stellar/stellar-etl-airflow into patch/update-dbt-node-selection
stellar · Nov 14, 2024 · 044e2bf · 044e2bf
2 parents c6c9b5d + 8b15c6c
commit 044e2bf
Show file tree

Hide file tree

Showing 4 changed files with 43 additions and 36 deletions.
diff --git a/.github/workflows/update_dbt_marts_schema_changelog.yml b/.github/workflows/update_dbt_marts_schema_changelog.yml
@@ -45,28 +45,28 @@ jobs:
           output=$(. scripts/update_dbt_marts_schema_changelog.sh)
           echo "$output" > changelog/dbt_marts.md
 
-      - name: Commit changes
-        id: commit_changes
-        run: |
-          git add changelog/dbt_marts.md
-          if git commit -m "Update changelog for DBT marts"; then
-            echo "Changes committed."
-            echo "changes_committed=true" >> $GITHUB_OUTPUT
-          else
-            echo "No changes to commit."
-            echo "changes_committed=false" >> $GITHUB_OUTPUT
-          fi
+      # - name: Commit changes
+      #   id: commit_changes
+      #   run: |
+      #     git add changelog/dbt_marts.md
+      #     if git commit -m "Update changelog for DBT marts"; then
+      #       echo "Changes committed."
+      #       echo "changes_committed=true" >> $GITHUB_OUTPUT
+      #     else
+      #       echo "No changes to commit."
+      #       echo "changes_committed=false" >> $GITHUB_OUTPUT
+      #     fi
 
-      - name: Push branch
-        if: steps.commit_changes.outputs.changes_committed == 'true'
-        run: |
-          git push origin ${{ steps.create_branch.outputs.branch }}
+      # - name: Push branch
+      #   if: steps.commit_changes.outputs.changes_committed == 'true'
+      #   run: |
+      #     git push origin ${{ steps.create_branch.outputs.branch }}
 
-      - name: Create Pull Request
-        if: steps.commit_changes.outputs.changes_committed == 'true'
-        run: |
-          gh pr create -B master -H ${{ steps.create_branch.outputs.branch }} \
-          --title 'Update schema changelog for DBT marts' \
-          --body 'This is an autogenerated PR to update schema changelog for marts.'
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      # - name: Create Pull Request
+      #   if: steps.commit_changes.outputs.changes_committed == 'true'
+      #   run: |
+      #     gh pr create -B master -H ${{ steps.create_branch.outputs.branch }} \
+      #     --title 'Update schema changelog for DBT marts' \
+      #     --body 'This is an autogenerated PR to update schema changelog for marts.'
+      #   env:
+      #     GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/update_source_data_schema_changelog.yml b/.github/workflows/update_source_data_schema_changelog.yml
@@ -35,14 +35,14 @@ jobs:
           rm -rf $OLD_SCHEMAS_DIR
           rm -rf repo_master_copy
 
-      - name: Commit and Push Changes
-        run: |
-          git config --local user.email "action@github.com"
-          git config --local user.name "GitHub Action"
-          git add changelog/source_data.md
-          if git commit -m "Update changelog for Source data"; then
-            echo "Changes committed."
-            git push
-          else
-            echo "No changes to commit."
-          fi
+      # - name: Commit and Push Changes
+      #   run: |
+      #     git config --local user.email "action@github.com"
+      #     git config --local user.name "GitHub Action"
+      #     git add changelog/source_data.md
+      #     if git commit -m "Update changelog for Source data"; then
+      #       echo "Changes committed."
+      #       git push
+      #     else
+      #       echo "No changes to commit."
+      #     fi
diff --git a/airflow_variables_prod.json b/airflow_variables_prod.json
@@ -125,7 +125,7 @@
     "partnership_assets__asset_activity_fact": false,
     "trade_agg": false
   },
-  "dbt_image_name": "stellar/stellar-dbt:420c216df",
+  "dbt_image_name": "stellar/stellar-dbt:96cd862b1",
   "dbt_internal_source_db": "hubble-261722",
   "dbt_internal_source_schema": "crypto_stellar_internal_2",
   "dbt_job_execution_timeout_seconds": 2400,

diff --git a/dags/dbt_data_quality_alerts_dag.py b/dags/dbt_data_quality_alerts_dag.py
@@ -24,7 +24,14 @@
 
     # Trigger elementary
     elementary_alerts = elementary_task(
-        dag, "dbt_data_quality", "monitor", resource_cfg="dbt"
+        dag,
+        "dbt_data_quality",
+        "monitor",
+        resource_cfg="dbt",
+        cmd_args=[
+            "--profiles-dir",
+            ".",
+        ],
     )
 
     elementary_alerts