fix compute_billing_per_hour and add lookback_incremental_billing_win…

…dow_days
bqbooster · Nov 8, 2024 · 7c65827 · 7c65827
1 parent f7cc1de
commit 7c65827
Show file tree

Hide file tree

Showing 4 changed files with 13 additions and 3 deletions.
diff --git a/.changes/unreleased/Fixes-20241108-164224.yaml b/.changes/unreleased/Fixes-20241108-164224.yaml
@@ -0,0 +1,6 @@
+kind: Fixes
+body: Fix lookback for compute billing and make it customizable via `lookback_incremental_billing_window_days` var
+time: 2024-11-08T16:42:24.170779+01:00
+custom:
+    Author: Kayrnt
+    Issue: ""
diff --git a/dbt_project.yml b/dbt_project.yml
@@ -49,7 +49,11 @@ vars:
   bi_engine_gb_hourly_price: "{{ env_var('DBT_BQ_MONITORING_BI_ENGINE_GB_HOURLY_PRICE', 0.0416) }}"
 
   # Project input configuration
-  lookback_window_days: "{{ env_var('DBT_BQ_MONITORING_LOOKBACK_WINDOW_DAYS', 7) }}"
+  # The number of days to look back for regular tables, you might use up to 180 days usually.
+  # Expiration on intermediate tables is aligned so that it can store data as old your maximum lookback window as it's partitioned by time.
+  lookback_window_days: "{{ env_var('DBT_BQ_MONITORING_LOOKBACK_WINDOW_DAYS', 7) }}" 
+  # Billing data can be late, a safe window is to refresh data for the past 3 days but you can increase it for exceptional cases
+  lookback_incremental_billing_window_days: "{{ env_var('DBT_BQ_MONITORING_LOOKBACK_INCREMENTAL_BILLING_WINDOW_DAYS', 3) }}"
   # Project output configuration
   output_materialization: "{{ env_var('DBT_BQ_MONITORING_OUTPUT_MATERIALIZATION', 'table') }}"
   output_limit_size: "{{ env_var('DBT_BQ_MONITORING_OUTPUT_LIMIT_SIZE', 1000) }}"

diff --git a/models/compute/intermediate/billing/compute_billing_per_hour.sql b/models/compute/intermediate/billing/compute_billing_per_hour.sql
@@ -22,6 +22,6 @@ WHERE
 ((service.description LIKE 'BigQuery' AND LOWER(sku.description) LIKE '%analysis%')
 OR (service.description LIKE 'BigQuery Reservation API'))
 {% if is_incremental() %}
-AND TIMESTAMP_TRUNC(usage_start_time, HOUR) >= _dbt_max_partition
+AND TIMESTAMP_TRUNC(usage_start_time, HOUR) >= TIMESTAMP_SUB(_dbt_max_partition, INTERVAL {{ var('lookback_incremental_billing_window_days') }} DAY)
 {% endif %}
 GROUP BY ALL
diff --git a/models/storage/datamart/storage_billing_per_hour.sql b/models/storage/datamart/storage_billing_per_hour.sql
@@ -22,6 +22,6 @@ WHERE
 (service.description LIKE '%BigQuery%'
 AND LOWER(sku.description) LIKE '%storage%')
 {% if is_incremental() %}
-AND TIMESTAMP_TRUNC(usage_start_time, HOUR) >= TIMESTAMP_SUB(_dbt_max_partition, INTERVAL 3 DAY)
+AND TIMESTAMP_TRUNC(usage_start_time, HOUR) >= TIMESTAMP_SUB(_dbt_max_partition, INTERVAL {{ var('lookback_incremental_billing_window_days') }} DAY)
 {% endif %}
 GROUP BY ALL