add TestLargeEphemeralCompilation (#8376) (#8717)

.changes/unreleased/Under the Hood-20230912-230619.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: add a test for ephemeral cte injection
+time: 2023-09-12T23:06:19.938207+01:00
+custom:
+  Author: michelleark
+  Issue: "8376"

tests/functional/materializations/fixtures.py

@@ -0,0 +1,245 @@
+fct_eph_first_sql = """
+-- fct_eph_first.sql
+{{ config(materialized='ephemeral') }}
+
+with int_eph_first as(
+    select * from {{ ref('int_eph_first') }}
+)
+
+select * from int_eph_first
+"""
+
+int_eph_first_sql = """
+-- int_eph_first.sql
+{{ config(materialized='ephemeral') }}
+
+select
+    1 as first_column,
+    2 as second_column
+"""
+
+schema_yml = """
+version: 2
+
+models:
+  - name: int_eph_first
+    columns:
+      - name: first_column
+        tests:
+          - not_null
+      - name: second_column
+        tests:
+          - not_null
+
+  - name: fct_eph_first
+    columns:
+      - name: first_column
+        tests:
+          - not_null
+      - name: second_column
+        tests:
+          - not_null
+
+"""
+
+bar_sql = """
+{{ config(materialized = 'table') }}
+
+WITH foo AS (
+
+    SELECT * FROM {{ ref('foo') }}
+
+), foo_1 AS (
+
+    SELECT * FROM {{ ref('foo_1') }}
+
+), foo_2 AS (
+
+   SELECT * FROM {{ ref('foo_2') }}
+
+)
+
+SELECT * FROM foo
+UNION ALL
+SELECT * FROM foo_1
+UNION ALL
+SELECT * FROM foo_2
+"""
+
+bar1_sql = """
+{{ config(materialized = 'table') }}
+
+WITH foo AS (
+
+    SELECT * FROM {{ ref('foo') }}
+
+), foo_1 AS (
+
+    SELECT * FROM {{ ref('foo_1') }}
+
+), foo_2 AS (
+
+   SELECT * FROM {{ ref('foo_2') }}
+
+)
+
+SELECT * FROM foo
+UNION ALL
+SELECT * FROM foo_1
+UNION ALL
+SELECT * FROM foo_2
+"""
+
+bar2_sql = """
+{{ config(materialized = 'table') }}
+
+WITH foo AS (
+
+    SELECT * FROM {{ ref('foo') }}
+
+), foo_1 AS (
+
+    SELECT * FROM {{ ref('foo_1') }}
+
+), foo_2 AS (
+
+   SELECT * FROM {{ ref('foo_2') }}
+
+)
+
+SELECT * FROM foo
+UNION ALL
+SELECT * FROM foo_1
+UNION ALL
+SELECT * FROM foo_2
+"""
+
+bar3_sql = """
+{{ config(materialized = 'table') }}
+
+WITH foo AS (
+
+    SELECT * FROM {{ ref('foo') }}
+
+), foo_1 AS (
+
+    SELECT * FROM {{ ref('foo_1') }}
+
+), foo_2 AS (
+
+   SELECT * FROM {{ ref('foo_2') }}
+
+)
+
+SELECT * FROM foo
+UNION ALL
+SELECT * FROM foo_1
+UNION ALL
+SELECT * FROM foo_2
+"""
+
+bar4_sql = """
+{{ config(materialized = 'table') }}
+
+WITH foo AS (
+
+    SELECT * FROM {{ ref('foo') }}
+
+), foo_1 AS (
+
+    SELECT * FROM {{ ref('foo_1') }}
+
+), foo_2 AS (
+
+   SELECT * FROM {{ ref('foo_2') }}
+
+)
+
+SELECT * FROM foo
+UNION ALL
+SELECT * FROM foo_1
+UNION ALL
+SELECT * FROM foo_2
+"""
+
+bar5_sql = """
+{{ config(materialized = 'table') }}
+
+WITH foo AS (
+
+    SELECT * FROM {{ ref('foo') }}
+
+), foo_1 AS (
+
+    SELECT * FROM {{ ref('foo_1') }}
+
+), foo_2 AS (
+
+   SELECT * FROM {{ ref('foo_2') }}
+
+)
+
+SELECT * FROM foo
+UNION ALL
+SELECT * FROM foo_1
+UNION ALL
+SELECT * FROM foo_2
+"""
+
+baz_sql = """
+{{ config(materialized = 'table') }}
+SELECT * FROM {{ ref('bar') }}
+"""
+
+baz1_sql = """
+{{ config(materialized = 'table') }}
+SELECT * FROM {{ ref('bar_1') }}
+"""
+
+foo_sql = """
+{{ config(materialized = 'ephemeral') }}
+
+with source as (
+
+    select 1 as id
+
+), renamed as (
+
+    select id as uid from source
+
+)
+
+select * from renamed
+"""
+
+foo1_sql = """
+{{ config(materialized = 'ephemeral') }}
+
+WITH source AS (
+
+    SELECT 1 AS id
+
+), RENAMED as (
+
+    SELECT id as UID FROM source
+
+)
+
+SELECT * FROM renamed
+"""
+
+foo2_sql = """
+{{ config(materialized = 'ephemeral') }}
+
+WITH source AS (
+
+    SELECT 1 AS id
+
+), RENAMED as (
+
+    SELECT id as UID FROM source
+
+)
+
+SELECT * FROM renamed
+"""

tests/functional/materializations/test_ephemeral_compilation.py

@@ -9,51 +9,23 @@
 # fails fairly regularly if that is broken, but does occasionally work (depending
 # on the order in which things are compiled). It requires multi-threading to fail.
 
-
-fct_eph_first_sql = """
--- fct_eph_first.sql
-{{ config(materialized='ephemeral') }}
-
-with int_eph_first as(
-    select * from {{ ref('int_eph_first') }}
+from tests.functional.materializations.fixtures import (
+    fct_eph_first_sql,
+    int_eph_first_sql,
+    schema_yml,
+    bar_sql,
+    bar1_sql,
+    bar2_sql,
+    bar3_sql,
+    bar4_sql,
+    bar5_sql,
+    baz_sql,
+    baz1_sql,
+    foo_sql,
+    foo1_sql,
+    foo2_sql,
 )
 
-select * from int_eph_first
-"""
-
-int_eph_first_sql = """
--- int_eph_first.sql
-{{ config(materialized='ephemeral') }}
-
-select
-    1 as first_column,
-    2 as second_column
-"""
-
-schema_yml = """
-version: 2
-
-models:
-  - name: int_eph_first
-    columns:
-      - name: first_column
-        tests:
-          - not_null
-      - name: second_column
-        tests:
-          - not_null
-
-  - name: fct_eph_first
-    columns:
-      - name: first_column
-        tests:
-          - not_null
-      - name: second_column
-        tests:
-          - not_null
-
-"""
-
 
 SUPPRESSED_CTE_EXPECTED_OUTPUT = """-- fct_eph_first.sql
 
@@ -89,3 +61,28 @@ def test__suppress_injected_ctes(self, project):
         node = node_result.node
         assert isinstance(node, ModelNode)
         assert node.compiled_code == SUPPRESSED_CTE_EXPECTED_OUTPUT
+
+
+# From: https://github.com/jeremyyeo/ephemeral-invalid-sql-repro/tree/main/models
+class TestLargeEphemeralCompilation:
+    @pytest.fixture(scope="class")
+    def models(self):
+
+        return {
+            "bar.sql": bar_sql,
+            "bar_1.sql": bar1_sql,
+            "bar_2.sql": bar2_sql,
+            "bar_3.sql": bar3_sql,
+            "bar_4.sql": bar4_sql,
+            "bar_5.sql": bar5_sql,
+            "baz.sql": baz_sql,
+            "baz_1.sql": baz1_sql,
+            "foo.sql": foo_sql,
+            "foo_1.sql": foo1_sql,
+            "foo_2.sql": foo2_sql,
+        }
+
+    def test_ephemeral_compilation(self, project):
+        # 8/11 table models are built as expected. no compilation errors
+        results = run_dbt(["build"])
+        assert len(results) == 8