fewer adapters will need to re-implemnt basic_load_csv_rows (dbt-labs…

…#3623) * fewer adapters will need to re-implemnt basic_load_csv_rows * hack version * reordering per convention * make redundant basic_load_csv_rows * for next version * Update core/dbt/include/global_project/macros/materializations/seed/seed.sql Co-authored-by: Jeremy Cohen <[email protected]> * Move up changelog entry Co-authored-by: Jeremy Cohen <[email protected]> Co-authored-by: Jeremy Cohen <[email protected]>
TeddyCr · Sep 9, 2021 · 6f495a8 · 6f495a8
1 parent 946fdd8
commit 6f495a8
Show file tree

Hide file tree

Showing 2 changed files with 50 additions and 38 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -10,10 +10,12 @@
 
 - Use GitHub Actions for CI ([#3688](https://github.com/dbt-labs/dbt/issues/3688), [#3669](https://github.com/dbt-labs/dbt/pull/3669))
 - Better dbt hub registry packages version logging that prompts the user for upgrades to relevant packages ([#3560](https://github.com/dbt-labs/dbt/issues/3560), [#3763](https://github.com/dbt-labs/dbt/issues/3763), [#3759](https://github.com/dbt-labs/dbt/pull/3759))
+- Allow the default seed macro's SQL parameter, `%s`, to be replaced by dispatching a new macro, `get_binding_char()`. This enables adapters with parameter marker characters such as `?` to not have to override `basic_load_csv_rows`.  ([#3622](https://github.com/fishtown-analytics/dbt/issues/3622), [#3623](https://github.com/fishtown-analytics/dbt/pull/3623))
 
 Contributors:
 - [@sungchun12](https://github.com/sungchun12) ([#3759](https://github.com/dbt-labs/dbt/pull/3759))
 - [@xemuliam](https://github.com/xemuliam) ([#3606](https://github.com/dbt-labs/dbt/pull/3606))
+- [@swanderz](https://github.com/swanderz) [#3623](https://github.com/fishtown-analytics/dbt/pull/3623)
 
 ## dbt 0.21.0b2 (August 19, 2021)
 

diff --git a/core/dbt/include/global_project/macros/materializations/seed/seed.sql b/core/dbt/include/global_project/macros/materializations/seed/seed.sql
@@ -3,14 +3,6 @@
   {{ adapter.dispatch('create_csv_table')(model, agate_table) }}
 {%- endmacro %}
 
-{% macro reset_csv_table(model, full_refresh, old_relation, agate_table) -%}
-  {{ adapter.dispatch('reset_csv_table')(model, full_refresh, old_relation, agate_table) }}
-{%- endmacro %}
-
-{% macro load_csv_rows(model, agate_table) -%}
-  {{ adapter.dispatch('load_csv_rows')(model, agate_table) }}
-{%- endmacro %}
-
 {% macro default__create_csv_table(model, agate_table) %}
   {%- set column_override = model['config'].get('column_types', {}) -%}
   {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}
@@ -33,6 +25,9 @@
   {{ return(sql) }}
 {% endmacro %}
 
+{% macro reset_csv_table(model, full_refresh, old_relation, agate_table) -%}
+  {{ adapter.dispatch('reset_csv_table')(model, full_refresh, old_relation, agate_table) }}
+{%- endmacro %}
 
 {% macro default__reset_csv_table(model, full_refresh, old_relation, agate_table) %}
     {% set sql = "" %}
@@ -47,6 +42,21 @@
     {{ return(sql) }}
 {% endmacro %}
 
+{% macro get_binding_char() -%}
+  {{ adapter.dispatch('get_binding_char')() }}
+{%- endmacro %}
+
+{% macro default__get_binding_char() %}
+  {{ return('%s') }}
+{% endmacro %}
+
+{% macro get_batch_size() -%}
+  {{ adapter.dispatch('get_batch_size')() }}
+{%- endmacro %}
+
+{% macro default__get_batch_size() %}
+  {{ return(10000) }}
+{% endmacro %}
 
 {% macro get_seed_column_quoted_csv(model, column_names) %}
   {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}
@@ -59,48 +69,48 @@
     {{ return(dest_cols_csv) }}
 {% endmacro %}
 
+{% macro load_csv_rows(model, agate_table) -%}
+  {{ adapter.dispatch('load_csv_rows')(model, agate_table) }}
+{%- endmacro %}
 
-{% macro basic_load_csv_rows(model, batch_size, agate_table) %}
-    {% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %}
-    {% set bindings = [] %}
+{% macro default__load_csv_rows(model, agate_table) %}
 
-    {% set statements = [] %}
+  {% set batch_size = get_batch_size() %}
 
-    {% for chunk in agate_table.rows | batch(batch_size) %}
-        {% set bindings = [] %}
+  {% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %}
+  {% set bindings = [] %}
 
-        {% for row in chunk %}
-            {% do bindings.extend(row) %}
-        {% endfor %}
+  {% set statements = [] %}
 
-        {% set sql %}
-            insert into {{ this.render() }} ({{ cols_sql }}) values
-            {% for row in chunk -%}
-                ({%- for column in agate_table.column_names -%}
-                    %s
-                    {%- if not loop.last%},{%- endif %}
-                {%- endfor -%})
-                {%- if not loop.last%},{%- endif %}
-            {%- endfor %}
-        {% endset %}
+  {% for chunk in agate_table.rows | batch(batch_size) %}
+      {% set bindings = [] %}
 
-        {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %}
+      {% for row in chunk %}
+          {% do bindings.extend(row) %}
+      {% endfor %}
 
-        {% if loop.index0 == 0 %}
-            {% do statements.append(sql) %}
-        {% endif %}
-    {% endfor %}
+      {% set sql %}
+          insert into {{ this.render() }} ({{ cols_sql }}) values
+          {% for row in chunk -%}
+              ({%- for column in agate_table.column_names -%}
+                  {{ get_binding_char() }}
+                  {%- if not loop.last%},{%- endif %}
+              {%- endfor -%})
+              {%- if not loop.last%},{%- endif %}
+          {%- endfor %}
+      {% endset %}
 
-    {# Return SQL so we can render it out into the compiled files #}
-    {{ return(statements[0]) }}
-{% endmacro %}
+      {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %}
 
+      {% if loop.index0 == 0 %}
+          {% do statements.append(sql) %}
+      {% endif %}
+  {% endfor %}
 
-{% macro default__load_csv_rows(model, agate_table) %}
-  {{ return(basic_load_csv_rows(model, 10000, agate_table) )}}
+  {# Return SQL so we can render it out into the compiled files #}
+  {{ return(statements[0]) }}
 {% endmacro %}
 
-
 {% materialization seed, default %}
 
   {%- set identifier = model['alias'] -%}