diff --git a/dbt_project.yml b/dbt_project.yml index b2ecd57d..81487611 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -7,3 +7,7 @@ profile: 'default' on-run-end: - "{{ log_dbt_results(results) }}" + +vars: + start_timestamp: null + batch_size: null diff --git a/models/root/document_metadata.sql b/models/root/document_metadata.sql index d3718e3a..4c7f0b47 100644 --- a/models/root/document_metadata.sql +++ b/models/root/document_metadata.sql @@ -13,12 +13,30 @@ ) }} -SELECT - _id as uuid, - _deleted, - saved_timestamp, - doc->>'type' as doc_type -from {{ source('couchdb', env_var('POSTGRES_TABLE')) }} source_table -{% if is_incremental() %} -WHERE source_table.saved_timestamp >= {{ max_existing_timestamp('saved_timestamp') }} +WITH source_table_CTE AS ( + SELECT + _id as uuid, + _deleted, + saved_timestamp, + doc->>'type' as doc_type + FROM {{ source('couchdb', env_var('POSTGRES_TABLE')) }} +) + +{% if var("start_timestamp") is not none and var("batch_size") is not none %} + SELECT * + FROM source_table_CTE + WHERE saved_timestamp >= '{{ var("start_timestamp") }}' + ORDER BY saved_timestamp + LIMIT {{ var('batch_size') }} +{% else %} + + SELECT + _id as uuid, + _deleted, + saved_timestamp, + doc->>'type' as doc_type + FROM {{ source('couchdb', env_var('POSTGRES_TABLE')) }} source_table + {% if is_incremental() %} + WHERE source_table.saved_timestamp >= {{ max_existing_timestamp('saved_timestamp') }} + {% endif %} {% endif %}