add more staging columns to dynamo and fix bugs (#656)

meltano · Jun 27, 2023 · 9721dda · 9721dda
1 parent db66c1e
commit 9721dda
Show file tree

Hide file tree

Showing 5 changed files with 12 additions and 16 deletions.
diff --git a/data/transform/models/marts/telemetry/base/cloud_executions_base.sql b/data/transform/models/marts/telemetry/base/cloud_executions_base.sql
@@ -8,8 +8,8 @@ WITH open_source_agg AS (
         MIN(
             CASE WHEN cli_command = 'schedule' THEN started_ts END
         ) AS oss_run_started_ts,
-        MIN(
-            CASE WHEN cli_command = 'schedule' THEN finished_ts END
+        MAX(
+            finished_ts
         ) AS oss_run_finished_ts
     FROM {{ ref('fact_cli_executions') }}
     WHERE cloud_execution_id IS NOT NULL
@@ -34,6 +34,7 @@ SELECT
     stg_dynamodb__workload_metadata_table.command_text_hash,
     stg_dynamodb__workload_metadata_table.cloud_job_name_hash,
     stg_dynamodb__workload_metadata_table.cloud_schedule_name_hash,
+    stg_dynamodb__workload_metadata_table.cloud_deployment_name_hash,
     cloud_schedule_frequency.schedule_freq_day,
     cloud_schedule_frequency.schedule_freq_rolling_avg,
     COALESCE(
@@ -102,12 +103,6 @@ INNER JOIN {{ ref('stg_dynamodb__projects_table') }}
     ON
         stg_dynamodb__workload_metadata_table.cloud_project_id
         = stg_dynamodb__projects_table.cloud_project_id
-LEFT JOIN {{ ref('stg_dynamodb__project_deployments') }}
-    ON
-        stg_dynamodb__workload_metadata_table.cloud_project_id
-        = stg_dynamodb__project_deployments.cloud_project_id
-        AND stg_dynamodb__workload_metadata_table.cloud_environment_name_hash
-        = stg_dynamodb__project_deployments.cloud_environment_name_hash
 LEFT JOIN {{ ref('stg_dynamodb__project_schedules_table') }}
     ON
         stg_dynamodb__workload_metadata_table.cloud_project_id
@@ -116,7 +111,7 @@ LEFT JOIN {{ ref('stg_dynamodb__project_schedules_table') }}
         = stg_dynamodb__project_schedules_table.tenant_resource_key
         AND stg_dynamodb__workload_metadata_table.cloud_schedule_name_hash
         = stg_dynamodb__project_schedules_table.cloud_schedule_name_hash
-        AND stg_dynamodb__project_deployments.cloud_deployment_name_hash
+        AND stg_dynamodb__workload_metadata_table.cloud_deployment_name_hash
         = stg_dynamodb__project_schedules_table.cloud_deployment_name_hash
 LEFT JOIN open_source_agg
     ON

diff --git a/data/transform/models/marts/telemetry/base/cloud_schedule_frequency.sql b/data/transform/models/marts/telemetry/base/cloud_schedule_frequency.sql
@@ -4,10 +4,6 @@ WITH joined AS (
         stg_dynamodb__workload_metadata_table.started_ts::DATE AS date_day,
         stg_dynamodb__workload_metadata_table.cloud_execution_id
     FROM {{ ref('stg_dynamodb__workload_metadata_table') }}
-    LEFT JOIN {{ ref('stg_dynamodb__project_deployments') }}
-        ON
-            stg_dynamodb__workload_metadata_table.cloud_environment_name_hash
-            = stg_dynamodb__project_deployments.cloud_environment_name_hash
     LEFT JOIN {{ ref('stg_dynamodb__project_schedules_table') }}
         ON
             stg_dynamodb__workload_metadata_table.cloud_schedule_name_hash
@@ -17,7 +13,7 @@ WITH joined AS (
             AND stg_dynamodb__workload_metadata_table.cloud_project_id
             = stg_dynamodb__project_schedules_table.cloud_project_id
             AND
-            stg_dynamodb__project_deployments.cloud_deployment_name_hash
+            stg_dynamodb__workload_metadata_table.cloud_deployment_name_hash
             = stg_dynamodb__project_schedules_table.cloud_deployment_name_hash
     WHERE
         stg_dynamodb__project_schedules_table.cloud_schedule_name_hash

diff --git a/data/transform/models/staging/dynamodb/stg_dynamodb__project_schedules_table.sql b/data/transform/models/staging/dynamodb/stg_dynamodb__project_schedules_table.sql
@@ -42,6 +42,7 @@ renamed AS (
         enabled AS is_enabled,
         tenant_resource_key,
         cloud_project_id,
+        eventbridge_name,
         SHA2_HEX(cloud_deployment_name) AS cloud_deployment_name_hash,
         SHA2_HEX(cloud_schedule_name) AS cloud_schedule_name_hash
     FROM clean_source

diff --git a/data/transform/models/staging/dynamodb/stg_dynamodb__projects_table.sql b/data/transform/models/staging/dynamodb/stg_dynamodb__projects_table.sql
@@ -18,7 +18,9 @@ renamed AS (
         project_surrogate_key,
         project_id AS cloud_project_id,
         project_name,
-        tenant_resource_key
+        tenant_resource_key,
+        SHA2_HEX(git_repository) AS git_repository_hash,
+        SHA2_HEX(project_root_path) AS project_root_path_hash
     FROM source
     WHERE row_num = 1
 

diff --git a/data/transform/models/staging/dynamodb/stg_dynamodb__workload_metadata_table.sql b/data/transform/models/staging/dynamodb/stg_dynamodb__workload_metadata_table.sql
@@ -18,6 +18,7 @@ renamed AS (
         NULLIF(exit_code, 'N/A')::INT AS cloud_exit_code,
         NULLIF(start_time, 'N/A')::TIMESTAMP_NTZ AS started_ts,
         ttl::INT AS cloud_run_ttl,
+        stopped_reason,
         SHA2_HEX(command_text) AS command_text_hash,
         SHA2_HEX(environment_name) AS cloud_environment_name_hash,
         SHA2_HEX(job_name) AS cloud_job_name_hash,
@@ -27,7 +28,8 @@ renamed AS (
         ) AS tenant_resource_key,
         SPLIT_PART(
             "TENANT_RESOURCE_KEY::PROJECT_ID", '::', 2 -- noqa: RF05
-        ) AS cloud_project_id
+        ) AS cloud_project_id,
+        SHA2_HEX(deployment_name) AS cloud_deployment_name_hash
     FROM source
     WHERE
         row_num = 1