airbytehq · marcosmarxm · Aug 20, 2021 · Aug 17, 2021 · Aug 17, 2021 · Aug 17, 2021
diff --git a/...s/bases/base-normalization/dbt-project-template/macros/cross_db_utils/json_operations.sql b/...s/bases/base-normalization/dbt-project-template/macros/cross_db_utils/json_operations.sql
@@ -68,15 +68,27 @@
 {%- endmacro %}
 
 {% macro postgres__json_extract(from_table, json_column, json_path_list, normalized_json_path) -%}
-    jsonb_extract_path({{ from_table }}.{{ json_column }}, {{ format_json_path(json_path_list) }})
+    {%- if json_column|string() == "_airbyte_nested_data" %}
+        jsonb_extract_path({{ json_column }}, {{ format_json_path(json_path_list) }})
+    {% else %}
+        jsonb_extract_path({{ from_table }}.{{ json_column }}, {{ format_json_path(json_path_list) }})
+    {% endif -%}
 {%- endmacro %}
 
 {% macro mysql__json_extract(from_table, json_column, json_path_list, normalized_json_path) -%}
-    json_extract({{ from_table }}.{{ json_column }}, {{ format_json_path(normalized_json_path) }})
+    {%- if json_column|string() == "_airbyte_nested_data" %}
+        json_extract({{ json_column }}, {{ format_json_path(normalized_json_path) }})
+    {% else %}
+        json_extract({{ from_table }}.{{ json_column }}, {{ format_json_path(normalized_json_path) }})
+    {% endif -%}
 {%- endmacro %}
 
 {% macro redshift__json_extract(from_table, json_column, json_path_list, normalized_json_path) -%}
-    case when json_extract_path_text({{ from_table }}.{{ json_column }}, {{ format_json_path(json_path_list) }}, true) != '' then json_extract_path_text({{ from_table }}.{{ json_column }}, {{ format_json_path(json_path_list) }}, true) end
+    {%- if json_column|string() == "_airbyte_nested_data" %}
+        case when json_extract_path_text({{ json_column }}, {{ format_json_path(json_path_list) }}, true) != '' then json_extract_path_text({{ json_column }}, {{ format_json_path(json_path_list) }}, true) end
+    {% else %}
+        case when json_extract_path_text({{ from_table }}.{{ json_column }}, {{ format_json_path(json_path_list) }}, true) != '' then json_extract_path_text({{ from_table }}.{{ json_column }}, {{ format_json_path(json_path_list) }}, true) end
+    {% endif -%}
 {%- endmacro %}
 
 {% macro snowflake__json_extract(from_table, json_column, json_path_list, normalized_json_path) -%}

diff --git a/...t_primary_key_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql b/...t_primary_key_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql
@@ -6,7 +6,7 @@
 -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema
 select
     json_extract_scalar(_airbyte_data, "$['id']") as id,
-    json_extract(table_alias._airbyte_data, "$['conflict_stream_array']") as conflict_stream_array,
+    json_extract_array(_airbyte_data, "$['conflict_stream_array']") as conflict_stream_array,
     _airbyte_emitted_at
 from `dataline-integration-testing`.test_normalization._airbyte_raw_conflict_stream_array as table_alias
 -- conflict_stream_array;

diff --git a/...t_primary_key_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql b/...t_primary_key_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql
@@ -8,9 +8,7 @@ select
     cast(id as 
     string
 ) as id,
-    cast(conflict_stream_array as 
-    string
-) as conflict_stream_array,
+    conflict_stream_array,
     _airbyte_emitted_at
 from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_array_ab1`
 -- conflict_stream_array;

diff --git a/...t_primary_key_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql b/...t_primary_key_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql
@@ -8,7 +8,7 @@ select
     *,
     to_hex(md5(cast(concat(coalesce(cast(id as 
     string
-), ''), '-', coalesce(cast(conflict_stream_array as 
+), ''), '-', coalesce(cast(array_to_string(conflict_stream_array, "|", "") as 
     string
 ), '')) as 
     string

diff --git a/...final/airbyte_ctes/test_normalization/conflict_stream_array_conflict_stream_array_ab1.sql b/...final/airbyte_ctes/test_normalization/conflict_stream_array_conflict_stream_array_ab1.sql
diff --git a/...final/airbyte_ctes/test_normalization/conflict_stream_array_conflict_stream_array_ab2.sql b/...final/airbyte_ctes/test_normalization/conflict_stream_array_conflict_stream_array_ab2.sql
diff --git a/...final/airbyte_ctes/test_normalization/conflict_stream_array_conflict_stream_array_ab3.sql b/...final/airbyte_ctes/test_normalization/conflict_stream_array_conflict_stream_array_ab3.sql
diff --git a/...st_normalization/conflict_stream_array_conflict_stream_array_conflict_stream_name_ab1.sql b/...st_normalization/conflict_stream_array_conflict_stream_array_conflict_stream_name_ab1.sql
diff --git a/...st_normalization/conflict_stream_array_conflict_stream_array_conflict_stream_name_ab2.sql b/...st_normalization/conflict_stream_array_conflict_stream_array_conflict_stream_name_ab2.sql
diff --git a/...st_normalization/conflict_stream_array_conflict_stream_array_conflict_stream_name_ab3.sql b/...st_normalization/conflict_stream_array_conflict_stream_array_conflict_stream_name_ab3.sql
diff --git a/...s/final/airbyte_tables/test_normalization/conflict_stream_array_conflict_stream_array.sql b/...s/final/airbyte_tables/test_normalization/conflict_stream_array_conflict_stream_array.sql
diff --git a/...s/test_normalization/conflict_stream_array_conflict_stream_array_conflict_stream_name.sql b/...s/test_normalization/conflict_stream_array_conflict_stream_array_conflict_stream_name.sql
diff --git a/...ey_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql b/...ey_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql
@@ -2,7 +2,7 @@
 -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema
 select
     {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id,
-    {{ json_extract('table_alias', '_airbyte_data', ['conflict_stream_array'], ['conflict_stream_array']) }} as conflict_stream_array,
+    {{ json_extract_array('_airbyte_data', ['conflict_stream_array'], ['conflict_stream_array']) }} as conflict_stream_array,
     _airbyte_emitted_at
 from {{ source('test_normalization', '_airbyte_raw_conflict_stream_array') }} as table_alias
 -- conflict_stream_array

diff --git a/...ey_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql b/...ey_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql
@@ -2,7 +2,7 @@
 -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type
 select
     cast(id as {{ dbt_utils.type_string() }}) as id,
-    cast(conflict_stream_array as {{ type_json() }}) as conflict_stream_array,
+    conflict_stream_array,
     _airbyte_emitted_at
 from {{ ref('conflict_stream_array_ab1') }}
 -- conflict_stream_array

diff --git a/...ey_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql b/...ey_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql
@@ -4,7 +4,7 @@ select
     *,
     {{ dbt_utils.surrogate_key([
         'id',
-        'conflict_stream_array',
+        array_to_string('conflict_stream_array'),
     ]) }} as _airbyte_conflict_stream_array_hashid
 from {{ ref('conflict_stream_array_ab2') }}
 -- conflict_stream_array

diff --git a/...rated/airbyte_ctes/test_normalization/conflict_stream_array_conflict_stream_array_ab1.sql b/...rated/airbyte_ctes/test_normalization/conflict_stream_array_conflict_stream_array_ab1.sql
diff --git a/...rated/airbyte_ctes/test_normalization/conflict_stream_array_conflict_stream_array_ab2.sql b/...rated/airbyte_ctes/test_normalization/conflict_stream_array_conflict_stream_array_ab2.sql
diff --git a/...rated/airbyte_ctes/test_normalization/conflict_stream_array_conflict_stream_array_ab3.sql b/...rated/airbyte_ctes/test_normalization/conflict_stream_array_conflict_stream_array_ab3.sql
diff --git a/...st_normalization/conflict_stream_array_conflict_stream_array_conflict_stream_name_ab1.sql b/...st_normalization/conflict_stream_array_conflict_stream_array_conflict_stream_name_ab1.sql
diff --git a/...st_normalization/conflict_stream_array_conflict_stream_array_conflict_stream_name_ab2.sql b/...st_normalization/conflict_stream_array_conflict_stream_array_conflict_stream_name_ab2.sql
diff --git a/...st_normalization/conflict_stream_array_conflict_stream_array_conflict_stream_name_ab3.sql b/...st_normalization/conflict_stream_array_conflict_stream_array_conflict_stream_name_ab3.sql
diff --git a/...nerated/airbyte_tables/test_normalization/conflict_stream_array_conflict_stream_array.sql b/...nerated/airbyte_tables/test_normalization/conflict_stream_array_conflict_stream_array.sql
diff --git a/...s/test_normalization/conflict_stream_array_conflict_stream_array_conflict_stream_name.sql b/...s/test_normalization/conflict_stream_array_conflict_stream_array_conflict_stream_name.sql
diff --git a/...ms/final/airbyte_ctes/test_normalization/conflict_stream_array_2lict_stream_array_ab1.sql b/...ms/final/airbyte_ctes/test_normalization/conflict_stream_array_2lict_stream_array_ab1.sql
diff --git a/...ms/final/airbyte_ctes/test_normalization/conflict_stream_array_2lict_stream_array_ab2.sql b/...ms/final/airbyte_ctes/test_normalization/conflict_stream_array_2lict_stream_array_ab2.sql
diff --git a/...ms/final/airbyte_ctes/test_normalization/conflict_stream_array_2lict_stream_array_ab3.sql b/...ms/final/airbyte_ctes/test_normalization/conflict_stream_array_2lict_stream_array_ab3.sql