Skip to content

Commit f2b2da4

Browse files
bachng2017bachng2017
authored
modification for more common and pluggable syntax (#85)
1 parent 625258e commit f2b2da4

15 files changed

Lines changed: 83 additions & 38 deletions

‎macros/public/store/export_alerts.sql‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,8 @@
99
from {{ ref('re_data_alerts') }}
1010
where
1111
case
12-
when type = 'anomaly' then time_window_end between '{{ start_date }}' and '{{ end_date }}'
13-
else time_window_end >= '{{ start_date }}'
12+
when type = 'anomaly' then {{ in_date_window('time_window_end', start_date, end_date) }}
13+
else {{ in_date_window('time_window_end', start_date, none) }}
1414
end
1515
order by time_window_end desc
1616
{% endset %}
@@ -19,4 +19,4 @@
1919
{% set alerts_file_path = alerts_path or 'target/re_data/alerts.json' %}
2020
{% do query_result.to_json(alerts_file_path) %}
2121
{{ save_monitored(monitored_path) }}
22-
{% endmacro %}
22+
{% endmacro %}

‎macros/public/store/export_tests_history.sql‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,11 @@
1515
compiled_sql
1616
from
1717
{{ ref('re_data_test_history') }}
18-
where date(run_at) between '{{start_date}}' and '{{end_date}}'
18+
where {{ in_date_window('run_at', start_date, end_date) }}
1919
{% endset %}
2020

2121
{% set query_result = run_query(tests_history_query) %}
2222
{% set tests_history_file_path = tests_history_path or 'target/re_data/tests_history.json' %}
2323
{% do query_result.to_json(tests_history_file_path) %}
2424

25-
{% endmacro %}
25+
{% endmacro %}

‎macros/public/store/generate_overview.sql‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@
3939
{{ to_single_json(['metric', 'value', 'time_window_end', 'interval_length_sec']) }} as {{ re_data.quote_column('data') }}
4040
from
4141
{{ ref('re_data_base_metrics') }}
42-
where date(time_window_end) between '{{start_date}}' and '{{end_date}}'
42+
where {{ in_date_window('time_window_end', start_date, end_date) }}
4343
and interval_length_sec = {{interval_length_sec}}
4444
) union all
4545
(
@@ -48,7 +48,7 @@
4848
{{ to_single_json(['id', 'metric', 'z_score_value', 'last_value', 'last_avg', 'last_stddev', 'time_window_end', 'interval_length_sec']) }} as {{ re_data.quote_column('data') }}
4949
from
5050
{{ ref('re_data_anomalies') }}
51-
where date(time_window_end) between '{{start_date}}' and '{{end_date}}'
51+
where {{ in_date_window('time_window_end', start_date, end_date) }}
5252
and interval_length_sec = {{interval_length_sec}}
5353
) union all
5454
(
@@ -57,7 +57,7 @@
5757
{{ to_single_json(['id', 'operation', 'data_type', 'is_nullable', 'prev_column_name', 'prev_data_type', 'prev_is_nullable', 'detected_time']) }} as {{ re_data.quote_column('data') }}
5858
from
5959
schema_changes_casted
60-
where date(detected_time) >= '{{start_date}}'
60+
where {{ in_date_window('detected_time', start_date, none) }}
6161
) union all
6262
(
6363
select
@@ -78,8 +78,8 @@
7878
{{ ref('re_data_alerts') }}
7979
where
8080
case
81-
when type = 'anomaly' then time_window_end between '{{ start_date }}' and '{{ end_date }}'
82-
else time_window_end >= '{{ start_date }}'
81+
when type = 'anomaly' then {{ in_date_window('time_window_end', start_date, end_date) }}
82+
else {{ in_date_window('time_window_end', start_date, none) }}
8383
end
8484
)
8585
order by {{ re_data.quote_column('computed_on')}} desc
@@ -90,4 +90,4 @@
9090
{% do overview_result.to_json(overview_file_path) %}
9191
{{ save_monitored(monitored_path) }}
9292

93-
{% endmacro %}
93+
{% endmacro %}

‎macros/utils/generate_alert_message.sql‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,4 +58,4 @@
5858
else
5959
'Test ' || {{ test_name }} || ' failed for column ' || {{ column_name }} || '.'
6060
end
61-
{% endmacro %}
61+
{% endmacro %}

‎macros/utils/json_extract.sql‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
{% macro json_extract(string, string_path) -%}
2+
3+
{{ adapter.dispatch('json_extract', 're_data') (string, string_path) }}
4+
5+
{%- endmacro %}
6+
7+
-- use fivetran_utils by default
8+
{% macro default__json_extract(string, string_path) %}
9+
10+
fivetran_utils.json_extract(string, string_path)
11+
12+
{% endmacro %}

‎macros/utils/percentile.sql‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
{% macro percentile(percentile_field, partition_field, percent) -%}
2+
3+
{{ adapter.dispatch('percentile', 're_data') (percentile_field, partition_field, percent) }}
4+
5+
{%- endmacro %}
6+
7+
-- use fivetran_utils by default
8+
{% macro default__percentile(percentile_field, partition_field, percent) %}
9+
fivetran_utils.percentile(percentile_field, partition_field, percent)
10+
{% endmacro %}
11+

‎macros/utils/time_macros.sql‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,3 +77,24 @@
7777
{% macro bigquery__format_timestamp(column_name) %}
7878
FORMAT_TIMESTAMP('%Y-%m-%d %H:%I:%S', {{column_name}})
7979
{% endmacro %}
80+
81+
/*
82+
provide a common way to compare time vs a range: start_date <= target <= end_date
83+
if start_date is none: target <= end_date
84+
if end_date is none: target >= start_date
85+
think none as infinity
86+
*/
87+
{%- macro in_date_window(target, start_date, end_date) %}
88+
{{ adapter.dispatch('in_date_window','re_data')(target, start_date, end_date) }}
89+
{% endmacro -%}
90+
91+
{% macro default__in_date_window(target, start_date, end_date) %}
92+
{% if start_date is not none and end_date is not none %}
93+
{{target}} between {{start_date}} and {{end_date}}
94+
{% elif start_date is none %}
95+
{{target}} <= {{end_date}}
96+
{% elif end_date is none %}
97+
{{target}} >= {{start_date}}
98+
{% endif %}
99+
{% endmacro %}
100+

‎models/alerts/re_data_anomalies.sql‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,8 @@ select
1818
z.last_median_absolute_deviation,
1919
z.last_mean_absolute_deviation,
2020
z.last_iqr,
21-
z.last_first_quartile - (cast( {{ fivetran_utils.json_extract('m.anomaly_detector', 'whisker_boundary_multiplier') }} as {{numeric_type()}} ) * z.last_iqr) lower_bound,
22-
z.last_third_quartile + (cast( {{ fivetran_utils.json_extract('m.anomaly_detector', 'whisker_boundary_multiplier') }} as {{numeric_type()}} ) * z.last_iqr) upper_bound,
21+
z.last_first_quartile - (cast( {{ json_extract('m.anomaly_detector', 'whisker_boundary_multiplier') }} as {{numeric_type()}} ) * z.last_iqr) lower_bound,
22+
z.last_third_quartile + (cast( {{ json_extract('m.anomaly_detector', 'whisker_boundary_multiplier') }} as {{numeric_type()}} ) * z.last_iqr) upper_bound,
2323
z.last_first_quartile,
2424
z.last_third_quartile,
2525
z.time_window_end,
@@ -35,15 +35,15 @@ and {{ split_and_return_nth_value('table_name', '.', 2) }} = m.schema
3535
and {{ split_and_return_nth_value('table_name', '.', 3) }} = m.name
3636
where
3737
case
38-
when {{ fivetran_utils.json_extract('m.anomaly_detector', 'name') }} = 'z_score'
39-
then abs(z_score_value) > cast({{ fivetran_utils.json_extract('m.anomaly_detector', 'threshold') }} as {{ numeric_type() }})
40-
when {{ fivetran_utils.json_extract('m.anomaly_detector', 'name') }} = 'modified_z_score'
41-
then abs(modified_z_score_value) > cast( {{ fivetran_utils.json_extract('m.anomaly_detector', 'threshold') }} as {{numeric_type()}} )
42-
when {{ fivetran_utils.json_extract('m.anomaly_detector', 'name') }} = 'boxplot'
38+
when {{ json_extract('m.anomaly_detector', 'name') }} = 'z_score'
39+
then abs(z_score_value) > cast({{ json_extract('m.anomaly_detector', 'threshold') }} as {{ numeric_type() }})
40+
when {{ json_extract('m.anomaly_detector', 'name') }} = 'modified_z_score'
41+
then abs(modified_z_score_value) > cast( {{ json_extract('m.anomaly_detector', 'threshold') }} as {{numeric_type()}} )
42+
when {{ json_extract('m.anomaly_detector', 'name') }} = 'boxplot'
4343
then (
44-
z.last_value < z.last_first_quartile - (cast( {{ fivetran_utils.json_extract('m.anomaly_detector', 'whisker_boundary_multiplier') }} as {{numeric_type()}} ) * z.last_iqr)
44+
z.last_value < z.last_first_quartile - (cast( {{ json_extract('m.anomaly_detector', 'whisker_boundary_multiplier') }} as {{numeric_type()}} ) * z.last_iqr)
4545
or
46-
z.last_value > z.last_third_quartile + (cast( {{ fivetran_utils.json_extract('m.anomaly_detector', 'whisker_boundary_multiplier') }} as {{numeric_type()}} ) * z.last_iqr)
46+
z.last_value > z.last_third_quartile + (cast( {{ json_extract('m.anomaly_detector', 'whisker_boundary_multiplier') }} as {{numeric_type()}} ) * z.last_iqr)
4747
)
4848
else false
4949
end

‎models/alerts/re_data_schema_changes.sql‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@
5353

5454
with curr_monitored_schema as (
5555
select * from {{ ref('re_data_columns_over_time')}}
56-
where detected_time = '{{ most_recent_time }}'
56+
where detected_time = cast('{{ most_recent_time }}' as {{ timestamp_type() }})
5757
and table_name in (
5858
select {{ full_table_name('name', 'schema', 'database') }} from {{ ref('re_data_selected')}}
5959
)
@@ -62,7 +62,7 @@
6262

6363
prev_monitored_schema as (
6464
select * from {{ ref('re_data_columns_over_time')}}
65-
where detected_time = '{{ prev_most_recent}}'
65+
where detected_time = cast('{{ prev_most_recent}}' as {{ timestamp_type() }})
6666
and table_name in (
6767
select {{ full_table_name('name', 'schema', 'database') }} from {{ ref('re_data_selected')}}
6868
)
@@ -143,7 +143,7 @@
143143
all_changes.prev_column_name,
144144
all_changes.prev_data_type,
145145
all_changes.prev_is_nullable,
146-
{{dbt_utils.current_timestamp_in_utc()}} as detected_time
146+
cast({{dbt_utils.current_timestamp_in_utc()}} as {{ timestamp_type() }}) as detected_time
147147
from all_changes
148148
)
149149

@@ -164,4 +164,4 @@
164164
detected_time
165165
from all_with_time
166166

167-
{% endif %}
167+
{% endif %}

‎models/alerts/re_data_z_score.sql‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ with z_score_without_id as (
3030
stats.last_first_quartile,
3131
stats.last_third_quartile,
3232
{{ time_window_end() }} as time_window_end,
33-
{{dbt_utils.current_timestamp_in_utc()}} as computed_on
33+
cast( {{dbt_utils.current_timestamp_in_utc()}} as {{ timestamp_type() }} ) as computed_on
3434
from
3535
{{ ref('re_data_last_stats') }} as stats,
3636
{{ ref('re_data_last_metrics') }} as last_metric
@@ -73,4 +73,4 @@ select
7373
interval_length_sec,
7474
computed_on
7575

76-
from z_score_without_id
76+
from z_score_without_id

0 commit comments

Comments
 (0)