diff --git a/analysis/test_file.sql b/analysis/test_file.sql new file mode 100644 index 0000000..e69de29 diff --git a/dbt_project.yml b/dbt_project.yml index 0801a25..45fab5f 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -31,8 +31,8 @@ vars: incremental_lookback_period: 'hour' future_proof_date: '9999-12-31' -on-run-end: - - "{% if target.name == 'prod' %}{{ dbt_artifacts.upload_results(results) }}{% endif %}" +# on-run-end: +# - "{% if target.name == 'prod' %}{{ dbt_artifacts.upload_results(results) }}{% endif %}" # Configuring models @@ -42,12 +42,23 @@ models: marts: +materialized: table +tags: ['tag'] + dims: + +tags: ["myorders"] + facts: + +tags: ["myorders"] finance: +group: finance marketing: +group: marketing operations: +group: operations + accounting: + +group: accounting + +access: public + cfo: + +group: cfo + +access: public + intermediate: +materialized: view staging: @@ -65,7 +76,4 @@ models: +materialized: ephemeral tests: - rapid_onboarding_exemplar: - _samples: - staging: - +store_failures: "{{ true if target.name == 'prod' else false }}" + +store_failures: false diff --git a/macros/test_macro.sql b/macros/test_macro.sql new file mode 100644 index 0000000..0db9069 --- /dev/null +++ b/macros/test_macro.sql @@ -0,0 +1,5 @@ +{% macro test_macro() %} + + {{ log("This is a message by log macro", info=True) }} + +{% endmacro %} \ No newline at end of file diff --git a/models/_groups.yml b/models/_groups.yml new file mode 100644 index 0000000..5dbec80 --- /dev/null +++ b/models/_groups.yml @@ -0,0 +1,10 @@ +groups: + - name: cfo + owner: + # 'name' or 'email' is required; additional properties allowed + email: cfo@mycompany.com + + - name: accounting + owner: + email: finance@mycompany.com + \ No newline at end of file diff --git a/models/_samples/incremental/example_incremental_model.sql b/models/_samples/incremental/example_incremental_model.sql index cf8cb7b..032a1b4 100644 --- a/models/_samples/incremental/example_incremental_model.sql +++ b/models/_samples/incremental/example_incremental_model.sql @@ -7,6 +7,7 @@ with source as ( select * from {{ ref('example_source_for_incremental') }} + {% if is_incremental() %} -- this filter will only be applied on an incremental run where _etl_loaded_at > (select max(_etl_loaded_at) from {{ this }}) diff --git a/models/_samples/snapshot/example_join_snapshots.sql b/models/_samples/snapshot/example_join_snapshots.md similarity index 98% rename from models/_samples/snapshot/example_join_snapshots.sql rename to models/_samples/snapshot/example_join_snapshots.md index 201d1c7..9ea20d4 100644 --- a/models/_samples/snapshot/example_join_snapshots.sql +++ b/models/_samples/snapshot/example_join_snapshots.md @@ -1,4 +1,4 @@ -with order_snapshot as ( +{# with order_snapshot as ( select * exclude dbt_valid_to, coalesce(dbt_valid_to, cast('{{ var("future_proof_date") }}' as timestamp)) as dbt_valid_to @@ -47,3 +47,4 @@ final as ( ) select * from final +#} \ No newline at end of file diff --git a/models/intermediate/finance/int_daily_orders_fact.sql b/models/intermediate/finance/int_daily_orders_fact.sql new file mode 100644 index 0000000..afe02ff --- /dev/null +++ b/models/intermediate/finance/int_daily_orders_fact.sql @@ -0,0 +1,36 @@ +with + +dates as ( + + select * from {{ ref('dim_dates') }} + +), + +orders as ( + + select * from {{ ref('dim_orders') }} + +), + +countries as ( + + select * from {{ ref('dim_countries') }} + +), + +join_sources as ( + + select + ordr.order_sk, + country.country_sk as order_country_origin_sk, + date.date_sk as order_date_sk, + ordr.order_quantity + from orders ordr + left join dates date + on ordr.order_date = date.calendar_date + left join countries country + on ordr.order_country_origin = country.country_name + +) + +select * from join_sources diff --git a/models/marts/accounting/int_revenue_fct.sql b/models/marts/accounting/int_revenue_fct.sql new file mode 100644 index 0000000..22ebbec --- /dev/null +++ b/models/marts/accounting/int_revenue_fct.sql @@ -0,0 +1,67 @@ +{{ config( + materialized='table', + tags=["cfo"] +) }} + +with + +depreciations as ( + + select * from {{ ref('dim_depreciation_accounts') }} + where account_type = 'Major' + +), + +interests as ( + + select * from {{ ref('dim_interests_accounts') }} + where account_type = 'Major' + +), + +net_income as ( + + select * from {{ ref('dim_net_income') }} + where account_type = 'Major' + +), + + +taxes as ( + + select * from {{ ref('dim_tax_accounts') }} + where account_type = 'Major' + +), + +join_sources as ( + + select + net_inc.account_id, + net_inc.net_income_usd, + deprec.depreciation_cost_usd, + ints.interests_amount_usd, + tax.tax_usd, + net_inc.account_type + from net_income net_inc + left join depreciations deprec + on net_inc.account_id = deprec.account_id + left join interests ints + on net_inc.account_id = ints.account_id + left join taxes tax + on net_inc.account_id = tax.account_id + +), + +final as ( + + select + account_id, + account_type, + (net_income_usd + interests_amount_usd + tax_usd + depreciation_cost_usd)::numeric(10, 3) as ebitda + from join_sources + +) + +select * from final + diff --git a/models/marts/cfo/fct_revenue.sql b/models/marts/cfo/fct_revenue.sql new file mode 100644 index 0000000..3e7fe56 --- /dev/null +++ b/models/marts/cfo/fct_revenue.sql @@ -0,0 +1,14 @@ +{{ config( + materialized='table', + tags=["cfo"] +) }} + +with + +sources as ( + + select * from {{ ref('int_revenue_fct') }} + +) + +select * from sources diff --git a/models/marts/dims/_dims__unit_tests.yml b/models/marts/dims/_dims__unit_tests.yml new file mode 100644 index 0000000..70ea8a8 --- /dev/null +++ b/models/marts/dims/_dims__unit_tests.yml @@ -0,0 +1,119 @@ +version: 2 + +unit_tests: + - name: test_if_full_name_format_is_correct + description: "Checks if the format for customer full name is correct" + model: dim_customer_info + config: + tags: | + {%- if target.name == 'prod' -%} unit_test + {%- else -%} do_not_run + {% endif %} + given: + - input: ref('dim_orders') + rows: + - {"cust_fname": "Johnny", "cust_lname": "Bravo"} + + expect: + rows: + - {cust_fullname: "Bravo, Johnny"} + + - name: test_accuracy_of_ebitda_calculation + description: "Test if the EBITDA calculation is correct" + model: int_revenue_fct + config: + tags: | + {%- if target.name != 'prod' -%} unit_test + {%- else -%} do_not_run + {% endif %} + + given: + - input: ref('dim_depreciation_accounts') + rows: + - {"account_id": "1", "account_type": "Major", "depreciation_cost_usd": "-10"} + + - input: ref('dim_interests_accounts') + rows: + - {"account_id": "1", "account_type": "Major", "interests_amount_usd": "10"} + + - input: ref('dim_net_income') + rows: + - {"account_id": "1", "account_type": "Major", "net_income_usd": "10"} + + - input: ref('dim_tax_accounts') + rows: + - {"account_id": "1", "account_type": "Major", "tax_usd": "-10"} + + expect: + rows: + - {"account_id": "1", "account_type": "Major", "ebitda": "-1"} + + - name: test_accuracy_of_ebitda_calculation__sql_format + description: "Test if the EBITDA calculation is correct using SQL format" + model: int_revenue_fct + config: + tags: | + {%- if target.name != 'prod' -%} unit_test + {%- else -%} do_not_run + {% endif %} + + given: + - input: ref('dim_depreciation_accounts') + format: sql + rows: | + with + main as ( + select + '1'::numeric(10, 3) as account_id, + 'Major'::varchar as account_type, + '-10'::numeric(10, 3) as depreciation_cost_usd + ) + + select * from main + + - input: ref('dim_interests_accounts') + format: sql + rows: | + with + main as ( + select + '1'::numeric(10, 3) as account_id, + 'Major'::varchar as account_type, + '10'::numeric(10, 3) as interests_amount_usd + ) + + select * from main + + - input: ref('dim_net_income') + format: sql + rows: | + with + main as ( + select + '1'::numeric(10, 3) as account_id, + 'Major'::varchar as account_type, + '10'::numeric(10, 3) as net_income_usd + ) + + select * from main + + - input: ref('dim_tax_accounts') + format: sql + rows: | + with + main as ( + select + '1'::numeric(10, 3) as account_id, + 'Major'::varchar as account_type, + '-10'::numeric(10, 3) as tax_usd + ) + + select * from main + + expect: + format: sql + rows: | + select + 1::numeric(10, 3) as account_id, + 'Major' as account_type, + 0::numeric(10, 3) as ebitda \ No newline at end of file diff --git a/models/marts/dims/_properties.yml b/models/marts/dims/_properties.yml new file mode 100644 index 0000000..adf796a --- /dev/null +++ b/models/marts/dims/_properties.yml @@ -0,0 +1,65 @@ +version: 2 + + + +models: + - name: dim_orders + latest_version: 2 + + versions: + - v: 1 + config: + columns: + - name: order_sk + data_type: decimal(10,2) + data_tests: + - unique + - not_null + - name: cust_fname + data_type: string + - name: cust_lname + data_type: string + - name: order_country_origin + data_type: string + - name: order_date + data_type: date + - name: order_id + data_type: string + - name: order_quantity + data_type: decimal(15,2) + + - v: 2 + config: + alias: dim_orders + #group: accounting + columns: + - name: order_sk + data_type: decimal(10,2) + data_tests: + - unique + - not_null + - name: credit_card_number + data_type: varchar(100) + - name: cust_fname + data_type: string + - name: cust_lname + data_type: string + - name: order_country_origin + data_type: string + - name: order_date + data_type: date + - name: + data_type: string + + - name: dim_countries + config: + group: accounting + columns: + - name: country_name + data_tests: + - accepted_values: + values: + - "United States" + - "France" + - "Italy" + - "Philippines" diff --git a/models/marts/dims/cfo/dim_depreciation_accounts.sql b/models/marts/dims/cfo/dim_depreciation_accounts.sql new file mode 100644 index 0000000..80bf71c --- /dev/null +++ b/models/marts/dims/cfo/dim_depreciation_accounts.sql @@ -0,0 +1,16 @@ +{{ config( + materialized='table', + tags=["cfo"] +) }} + + +with + +source as ( + + select '1'::numeric(10, 3) as account_id, '-1000.00'::numeric(10, 3) depreciation_cost_usd,'Major'::varchar as account_type, '2024-01-01'::date as record_updated_ts union all + select '2'::numeric(10, 3) as account_id, '-1000.00'::numeric(10, 3) depreciation_cost_usd,'Minor'::varchar as account_type, '2024-01-01'::date as record_updated_ts + +) + +select * from source diff --git a/models/marts/dims/cfo/dim_interests_accounts.sql b/models/marts/dims/cfo/dim_interests_accounts.sql new file mode 100644 index 0000000..6fadd7f --- /dev/null +++ b/models/marts/dims/cfo/dim_interests_accounts.sql @@ -0,0 +1,16 @@ +{{ config( + materialized='table', + tags=["cfo"] +) }} + + +with + +source as ( + + select '1'::numeric(10, 3) as account_id, '10000.00'::numeric(10, 3) interests_amount_usd,'Major'::varchar as account_type, '2024-01-01'::date as record_updated_ts union all + select '2'::numeric(10, 3) as account_id, '10000.00'::numeric(10, 3) interests_amount_usd,'Minor'::varchar as account_type, '2024-01-01'::date as record_updated_ts + +) + +select * from source diff --git a/models/marts/dims/cfo/dim_net_income.sql b/models/marts/dims/cfo/dim_net_income.sql new file mode 100644 index 0000000..b434f50 --- /dev/null +++ b/models/marts/dims/cfo/dim_net_income.sql @@ -0,0 +1,16 @@ +{{ config( + materialized='table', + tags=["cfo"] +) }} + + +with + +source as ( + + select '1'::numeric(10, 3) as account_id, '10000.00'::numeric(10, 3) net_income_usd,'Major'::varchar as account_type, '2024-01-01'::date as record_updated_ts union all + select '2'::numeric(10, 3) as account_id, '10000.00'::numeric(10, 3) net_income_usd,'Minor'::varchar as account_type, '2024-01-01'::date as record_updated_ts + +) + +select * from source diff --git a/models/marts/dims/cfo/dim_tax_accounts.sql b/models/marts/dims/cfo/dim_tax_accounts.sql new file mode 100644 index 0000000..92087d9 --- /dev/null +++ b/models/marts/dims/cfo/dim_tax_accounts.sql @@ -0,0 +1,15 @@ +{{ config( + materialized='table', + tags=["cfo"] +) }} + +with + +source as ( + + select '1'::numeric(10, 3) as account_id, '-2000.00'::numeric(10, 3) tax_usd,'Major'::varchar as account_type, '2024-01-01'::date as record_updated_ts union all + select '2'::numeric(10, 3) as account_id, '-2000.00'::numeric(10, 3) tax_usd,'Minor'::varchar as account_type, '2024-01-01'::date as record_updated_ts + +) + +select * from source diff --git a/models/marts/dims/my_orders/dim_countries.sql b/models/marts/dims/my_orders/dim_countries.sql new file mode 100644 index 0000000..d70eda8 --- /dev/null +++ b/models/marts/dims/my_orders/dim_countries.sql @@ -0,0 +1,20 @@ +{{ + config( + materialized='table' + ) +}} + +with + +countries as ( + + select 1 as country_sk, 'United States'::varchar(100) as country_name union all + select 2 as country_sk, 'France'::varchar(100) as country_name union all + select 3 as country_sk, 'Italy'::varchar(100) as country_name union all + select 4 as country_sk, 'Philippines'::varchar(100) as country_name + -- union all + -- select 5 as country_sk, 'Italia'::varchar(100) as country_name + +) + +select * from countries \ No newline at end of file diff --git a/models/marts/dims/my_orders/dim_customer_info.sql b/models/marts/dims/my_orders/dim_customer_info.sql new file mode 100644 index 0000000..a81f840 --- /dev/null +++ b/models/marts/dims/my_orders/dim_customer_info.sql @@ -0,0 +1,16 @@ +with + +dim_orders as ( + + select * from {{ ref('dim_orders') }} + +), + +final as ( + + select + cust_lname || ', ' || cust_fname as cust_fullname, + from dim_orders +) + +select * from final diff --git a/models/marts/dims/my_orders/dim_dates.sql b/models/marts/dims/my_orders/dim_dates.sql new file mode 100644 index 0000000..af9975e --- /dev/null +++ b/models/marts/dims/my_orders/dim_dates.sql @@ -0,0 +1,19 @@ +{{ + config( + materialized='table' + ) +}} + +with + +dates as ( + + select '20240101'::decimal(18,2) as date_sk, '2024-01-01'::date as calendar_date union all + select '20240102'::decimal(18,2) as date_sk, '2024-01-02'::date as calendar_date union all + select '20240103'::decimal(18,2) as date_sk, '2024-01-03'::date as calendar_date union all + select '20240104'::decimal(18,2) as date_sk, '2024-01-04'::date as calendar_date union all + select '20240105'::decimal(18,2) as date_sk, '2024-01-05'::date as calendar_date + +) + +select * from dates \ No newline at end of file diff --git a/models/marts/dims/my_orders/dim_orders_v1.sql b/models/marts/dims/my_orders/dim_orders_v1.sql new file mode 100644 index 0000000..9d3f1e8 --- /dev/null +++ b/models/marts/dims/my_orders/dim_orders_v1.sql @@ -0,0 +1,28 @@ +{{ + config( + materialized='table' + ) +}} + + +with + +orders as ( + + select '1'::numeric(10,2) as order_sk, 'order-001' as order_id, '2024-01-01'::date as order_date, 'United States'::varchar(100) as order_country_origin,'John'::varchar(10) as cust_fname, 'Lennon'::varchar(10) as cust_lname, 1::decimal(17,3) as order_quantity union all + select '2'::numeric(10,2) as order_sk, 'order-002' as order_id, '2024-01-03'::date as order_date, 'France'::varchar(100) as order_country_origin,'Ringo'::varchar(10) as cust_fname, 'Starr'::varchar(10) as cust_lname, 2::decimal(17,2) as order_quantity union all + select '3'::numeric(10,2) as order_sk, 'order-003' as order_id, '2024-01-03'::date as order_date, 'Philippines'::varchar(100) as order_country_origin,'Paul'::varchar(10) as cust_fname, 'McCartney'::varchar(10) as cust_lname, 3::decimal(17,3) as order_quantity + +-- model contract example: missing column order_quantity + -- select '1'::numeric(10,2) as order_sk, 'order-001' as order_id, '2024-01-01'::date as order_date, 'United States'::varchar(100) as order_country_origin,'John'::varchar(10) as cust_fname, 'Lennon'::varchar(10) as cust_lname union all + -- select '2'::numeric(10,2) as order_sk, 'order-002' as order_id, '2024-01-03'::date as order_date, 'France'::varchar(100) as order_country_origin,'Ringo'::varchar(10) as cust_fname, 'Starr'::varchar(10) as cust_lname union all + -- select '3'::numeric(10,2) as order_sk, 'order-003' as order_id, '2024-01-03'::date as order_date, 'Philippines'::varchar(100) as order_country_origin,'Paul'::varchar(10) as cust_fname, 'McCartney'::varchar(10) as cust_lname + -- union all + -- select NULL::decimal(18,2) as order_sk, 'order-003' as order_id, '2024-01-03'::date as order_date, 'Philippines'::varchar(100) as order_country_origin,'Johnny'::varchar(10) as cust_fname, 'English'::varchar(10) as cust_lname, 3 as order_quantity union all + -- select '3'::decimal(18,2) as order_sk, 'order-003' as order_id, '2024-01-03'::date as order_date, 'Philippines'::varchar(100) as order_country_origin,'Mister'::varchar(10) as cust_fname, 'Bean'::varchar(10) as cust_lname, 3 as order_quantity + +) + +select * from orders + + diff --git a/models/marts/dims/my_orders/dim_orders_v2.sql b/models/marts/dims/my_orders/dim_orders_v2.sql new file mode 100644 index 0000000..068f5b3 --- /dev/null +++ b/models/marts/dims/my_orders/dim_orders_v2.sql @@ -0,0 +1,19 @@ +{{ + config( + materialized='table' + ) +}} + + +with + +orders as ( + + select '1'::numeric(10,2) as order_sk,'1234-567-00'::varchar(100) as credit_card_number ,'order-001' as order_id, '2024-01-01'::date as order_date, 'United States'::varchar(100) as order_country_origin,'John'::varchar(10) as cust_fname, 'Lennon'::varchar(10) as cust_lname union all + select '2'::numeric(10,2) as order_sk,'1234-567-00'::varchar(100) as credit_card_number ,'order-002' as order_id, '2024-01-03'::date as order_date, 'France'::varchar(100) as order_country_origin,'Ringo'::varchar(10) as cust_fname, 'Starr'::varchar(10) as cust_lname union all + select '3'::numeric(10,2) as order_sk,'1234-567-00'::varchar(100) as credit_card_number ,'order-003' as order_id, '2024-01-03'::date as order_date, 'Philippines'::varchar(100) as order_country_origin,'Paul'::varchar(10) as cust_fname, 'McCartney'::varchar(10) as cust_lname + +) +select * from orders + + diff --git a/models/marts/facts/_facts__properties.yml b/models/marts/facts/_facts__properties.yml new file mode 100644 index 0000000..553f0f7 --- /dev/null +++ b/models/marts/facts/_facts__properties.yml @@ -0,0 +1,10 @@ +version: 2 + +models: + - name: fct_daily_orders + columns: + - name: order_sk + tests: + - relationships: + to: ref('dim_orders') + field: order_sk \ No newline at end of file diff --git a/models/marts/facts/my_orders/fct_daily_orders.sql b/models/marts/facts/my_orders/fct_daily_orders.sql new file mode 100644 index 0000000..a3d5bd5 --- /dev/null +++ b/models/marts/facts/my_orders/fct_daily_orders.sql @@ -0,0 +1,25 @@ +with + +int_daily_orders_fact as ( + + select * from {{ ref('int_daily_orders_fact') }} + +), + +agg_order_quantity as ( + + select + order_sk, + order_date_sk, + order_country_origin_sk, + sum(order_quantity) as total_order_quantity + from int_daily_orders_fact + group by + order_sk, + order_date_sk, + order_country_origin_sk + +) + +select * from agg_order_quantity + diff --git a/models/python/dim_orders_ml.py b/models/python/dim_orders_ml.py new file mode 100644 index 0000000..95d5d8e --- /dev/null +++ b/models/python/dim_orders_ml.py @@ -0,0 +1,19 @@ +import numpy +import pandas as pd + +def model(dbt, session): + + dbt.config(materialized="table") + + upstream_model = dbt.ref("dim_orders") + sorted = upstream_model.sort_values(by="CUST_LNAME") + + print("This is a line in python") + + # Process data with the external package + + + # Return the DataFrame as the model output + return sorted + + \ No newline at end of file diff --git a/models/python/python_transformations.py b/models/python/python_transformations.py new file mode 100644 index 0000000..0c00fdd --- /dev/null +++ b/models/python/python_transformations.py @@ -0,0 +1,19 @@ +import string +import random +import pandas as pd + +def model(dbt, session): + dbt.config(materialized="table") + + upstream_model = dbt.ref("dim_orders") + + letters = string.ascii_lowercase + string.ascii_uppercase + string.digits + default_salt = (''.join(random.choice(letters) for i in range(30))) + + columns_without_cc_number = upstream_model[['order_sk', 'order_id', 'order_date', 'order_country_origin', 'cust_fname', 'cust_lname']] + + # Create DataFrame with the string as a single row in one column + hashed_value = pd.DataFrame([default_salt], columns=['hashed_id']) + + return columns_without_cc_number + \ No newline at end of file diff --git a/models/test_model.sql b/models/test_model.sql new file mode 100644 index 0000000..e36a24f --- /dev/null +++ b/models/test_model.sql @@ -0,0 +1,18 @@ +{{ + config( + materialized='table' + ) +}} + + +with + +src as ( + + select 1 as col_id + +) + +select * from src + +{{ log("This is a log inside the model", info=True) }} \ No newline at end of file diff --git a/models/test_query.sql b/models/test_query.sql new file mode 100644 index 0000000..e69de29 diff --git a/seeds/_seeds_properties.yml b/seeds/_seeds_properties.yml new file mode 100644 index 0000000..c4c5956 --- /dev/null +++ b/seeds/_seeds_properties.yml @@ -0,0 +1,7 @@ +version: 2 + +seeds: + - name: seeds_test + config: + database: analytics + schema: dbt_kfajardo \ No newline at end of file diff --git a/seeds/test_seed.csv b/seeds/test_seed.csv new file mode 100644 index 0000000..dbd888a --- /dev/null +++ b/seeds/test_seed.csv @@ -0,0 +1,2 @@ +col1,col2,col3 +1,2,3 \ No newline at end of file diff --git a/snapshots/_samples/example_generate_schema_snapshot.sql b/snapshots/_samples/example_generate_schema_snapshot.sql deleted file mode 100644 index 2174d7b..0000000 --- a/snapshots/_samples/example_generate_schema_snapshot.sql +++ /dev/null @@ -1,14 +0,0 @@ -{% snapshot example_generate_schema_snapshot %} - {{ - config( - target_database='analytics', - target_schema=generate_schema_name('snapshots'), - unique_key='id', - strategy='timestamp', - updated_at='order_updated_at', - invalidate_hard_deletes=True - ) - }} - - select * from {{ ref('example_orders_line_items_source_for_snapshot') }} - {% endsnapshot %} \ No newline at end of file diff --git a/snapshots/_samples/example_orders_line_items_snapshot.sql b/snapshots/_samples/example_orders_line_items_snapshot.sql deleted file mode 100644 index 63603f5..0000000 --- a/snapshots/_samples/example_orders_line_items_snapshot.sql +++ /dev/null @@ -1,14 +0,0 @@ -{% snapshot example_orders_line_items_snapshot %} - {{ - config( - target_database='analytics', - target_schema='snapshots', - unique_key='id', - strategy='timestamp', - updated_at='order_updated_at', - invalidate_hard_deletes=True - ) - }} - - select * from {{ ref('example_orders_line_items_source_for_snapshot') }} - {% endsnapshot %} \ No newline at end of file diff --git a/snapshots/_samples/example_orders_snapshot.sql b/snapshots/_samples/example_orders_snapshot.sql deleted file mode 100644 index e4e7ad6..0000000 --- a/snapshots/_samples/example_orders_snapshot.sql +++ /dev/null @@ -1,14 +0,0 @@ -{% snapshot example_orders_snapshot %} - {{ - config( - target_database='analytics', - target_schema='snapshots', - unique_key='order_id', - strategy='timestamp', - updated_at='order_updated_at', - invalidate_hard_deletes=True - ) - }} - - select * from {{ ref('example_orders_source_for_snapshot') }} - {% endsnapshot %} \ No newline at end of file diff --git a/snapshots/jaffle_snapshots.yml b/snapshots/jaffle_snapshots.yml new file mode 100644 index 0000000..98b9695 --- /dev/null +++ b/snapshots/jaffle_snapshots.yml @@ -0,0 +1,51 @@ +version: 2 + +snapshots: +# This is a pretty standard snapshot + - name: snapshot_stg_payments + relation: source('stripe', 'payment') + config: + enabled: true + strategy: timestamp + updated_at: _BATCHED_AT + unique_key: ID + persist_docs: + relation: true + columns: true +# Another standard snapshot but this time with a different table. + - name: snapshot_example_orders + relation: ref('example_orders_source_for_snapshot') + config: + enabled: true + strategy: timestamp + updated_at: order_updated_at + unique_key: order_id + persist_docs: + relation: true + columns: true + invalidate_hard_deletes: true + +# This snapshot has the check_cols example + - name: snapshot_int_line_items + relation: ref('int_line_items_amounts_calculated') + config: + enabled: true + strategy: check + check_cols: ['gross_item_sales_amount', 'net_item_sales_amount'] + unique_key: order_item_id + persist_docs: + relation: true + columns: true +# This snapshot has invalidate hard deletes as an example + - name: snapshot_orders_line_items + relation: ref('example_orders_line_items_source_for_snapshot') + config: + enabled: true + strategy: timestamp + unique_key: id + updated_at: order_updated_at + invalidate_hard_deletes: true + persist_docs: + relation: true + columns: true + diff --git a/snapshots/snapshot_int_line_items.sql b/snapshots/snapshot_int_line_items.sql deleted file mode 100644 index 90bc550..0000000 --- a/snapshots/snapshot_int_line_items.sql +++ /dev/null @@ -1,22 +0,0 @@ -{% snapshot snapshot_int_line_items %} - --- Example showing check columns strategy and using custom schemas (this is rare) - --- if you don't want to write to the same schema in development and prod, --- use environment-aware variable from dbt_project.yml - --- if you do not have a reliable updated at timestamp, checek columns for changes. - -{{ - config( - target_database='analytics', - target_schema=var('example_target_snapshot_schema'), - unique_key='order_item_id', - strategy='check', - check_cols=['gross_item_sales_amount', 'net_item_sales_amount'], - ) -}} - -select * from {{ ref('int_line_items_amounts_calculated') }} - -{% endsnapshot %} \ No newline at end of file diff --git a/snapshots/snapshot_stg_payments.sql b/snapshots/snapshot_stg_payments.sql deleted file mode 100644 index adc61b2..0000000 --- a/snapshots/snapshot_stg_payments.sql +++ /dev/null @@ -1,16 +0,0 @@ -{% snapshot snapshot_stg_payments %} - -{{ - config( - target_database='analytics', - target_schema='snapshots', - unique_key='ID', - - strategy='timestamp', - updated_at='_BATCHED_AT', - ) -}} - -select * from {{ source('stripe', 'payment') }} - -{% endsnapshot %} \ No newline at end of file diff --git a/tests/assert_duplicates__fct_daily_orders.sql b/tests/assert_duplicates__fct_daily_orders.sql new file mode 100644 index 0000000..517e7cd --- /dev/null +++ b/tests/assert_duplicates__fct_daily_orders.sql @@ -0,0 +1,21 @@ +with + + +source as ( + + select * from {{ ref('fct_daily_orders') }} + +), + +validations as ( + + select + order_sk, + count(*) as rec_cnt + from {{ ref('fct_daily_orders') }} + group by order_sk + having count(*) > 1 + +) + +select * from validations \ No newline at end of file diff --git a/tests/assert_null__payment.sql b/tests/assert_null__payment.sql new file mode 100644 index 0000000..3627f80 --- /dev/null +++ b/tests/assert_null__payment.sql @@ -0,0 +1,12 @@ +with + +source as ( + + select * from {{ source('stripe', 'payment') }} + +) + +select +id +from source +where id is null \ No newline at end of file diff --git a/tests/fixtures/1_fixture.sql b/tests/fixtures/1_fixture.sql new file mode 100644 index 0000000..e69de29 diff --git a/tests/fixtures/input/dim_depreciation_accounts__fixture.sql b/tests/fixtures/input/dim_depreciation_accounts__fixture.sql new file mode 100644 index 0000000..e69de29 diff --git a/tests/fixtures/input/dim_net_income__fixture.sql b/tests/fixtures/input/dim_net_income__fixture.sql new file mode 100644 index 0000000..e69de29