diff --git a/.github/workflows/update_source_data_schema_changelog.yml b/.github/workflows/update_source_data_schema_changelog.yml index ad795991..ae8d4cae 100644 --- a/.github/workflows/update_source_data_schema_changelog.yml +++ b/.github/workflows/update_source_data_schema_changelog.yml @@ -27,9 +27,10 @@ jobs: cd $GITHUB_WORKSPACE cp -r schemas/ new_schemas/ git clone --branch master https://github.com/stellar/stellar-etl-airflow.git repo_master_copy - cp -r repo_master_copy/schemas/ old_schemas/ + cp -r repo_master_copy/schemas/ original_schemas/ output=$(python3 scripts/get_source_data_schema_changelog.py) - echo "$output" > changelog/source_data.md + if [ -n "$output" ]; then + echo "hello" - name: Commit changes id: commit_changes diff --git a/changelog/source_data.md b/changelog/source_data.md index e69de29b..d7b218ef 100644 --- a/changelog/source_data.md +++ b/changelog/source_data.md @@ -0,0 +1 @@ +## Junk data to check if append works diff --git a/schemas/dimAccounts_schema.json b/schemas/dimAccounts_schema.json deleted file mode 100644 index 67c848ff..00000000 --- a/schemas/dimAccounts_schema.json +++ /dev/null @@ -1,12 +0,0 @@ -[ - { - "mode": "NULLABLE", - "name": "address", - "type": "STRING" - }, - { - "mode": "NULLABLE", - "name": "account_id", - "type": "FLOAT" - } -] diff --git a/schemas/dimOffers_schema.json b/schemas/dimOffers_schema.json index 63b4e757..eb6b4f23 100644 --- a/schemas/dimOffers_schema.json +++ b/schemas/dimOffers_schema.json @@ -38,5 +38,10 @@ "mode": "NULLABLE", "name": "horizon_offer_id", "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "new_id", + "type": "INTEGER" } ] diff --git a/schemas/history_assets_schema.json b/schemas/history_assets_schema.json index c6576a96..a92dc485 100644 --- a/schemas/history_assets_schema.json +++ b/schemas/history_assets_schema.json @@ -7,7 +7,7 @@ { "mode": "NULLABLE", "name": "asset_code", - "type": "STRING" + "type": "INTEGER" }, { "mode": "NULLABLE", diff --git a/schemas/history_trades_schema.json b/schemas/history_trades_schema.json index 494d1893..e61d456a 100644 --- a/schemas/history_trades_schema.json +++ b/schemas/history_trades_schema.json @@ -29,11 +29,6 @@ "name": "selling_asset_issuer", "type": "STRING" }, - { - "mode": "NULLABLE", - "name": "selling_asset_type", - "type": "STRING" - }, { "mode": "NULLABLE", "name": "selling_asset_id", diff --git a/schemas/new_table_schema.json b/schemas/new_table_schema.json new file mode 100644 index 00000000..a0f621bf --- /dev/null +++ b/schemas/new_table_schema.json @@ -0,0 +1,118 @@ +[ + { + "mode": "NULLABLE", + "name": "liquidity_pool_id", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "type", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "fee", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "trustline_count", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "pool_share_count", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "asset_a_type", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "asset_a_code", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "asset_a_issuer", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "asset_a_id", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "asset_a_amount", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "asset_b_type", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "asset_b_code", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "asset_b_issuer", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "asset_b_id", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "asset_b_amount", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "last_modified_ledger", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "ledger_entry_change", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "deleted", + "type": "BOOLEAN" + }, + { + "mode": "NULLABLE", + "name": "batch_id", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "batch_run_date", + "type": "DATETIME" + }, + { + "mode": "NULLABLE", + "name": "batch_insert_ts", + "type": "TIMESTAMP" + }, + { + "mode": "NULLABLE", + "name": "closed_at", + "type": "TIMESTAMP" + }, + { + "mode": "NULLABLE", + "name": "ledger_sequence", + "type": "INTEGER" + } + ] + \ No newline at end of file diff --git a/schemas/offers_schema.json b/schemas/offers_schema.json index e5104ca0..0a8a11e4 100644 --- a/schemas/offers_schema.json +++ b/schemas/offers_schema.json @@ -47,7 +47,7 @@ { "mode": "NULLABLE", "name": "buying_asset_id", - "type": "INTEGER" + "type": "STRING" }, { "mode": "NULLABLE", diff --git a/schemas/trust_lines_schema.json b/schemas/trust_lines_schema.json index 1502d701..c6370df8 100644 --- a/schemas/trust_lines_schema.json +++ b/schemas/trust_lines_schema.json @@ -9,11 +9,6 @@ "name": "account_id", "type": "STRING" }, - { - "mode": "NULLABLE", - "name": "asset_type", - "type": "STRING" - }, { "mode": "NULLABLE", "name": "asset_issuer",