Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add more staging columns to dynamo and fix bugs #656

Merged
merged 4 commits into from
Jun 27, 2023
Merged

add more staging columns to dynamo and fix bugs #656

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
0ceac3b
add more staging columns to dynamo and fix bugs
pnadolny13 Jun 27, 2023
a2194bd
Merge branch 'main' into cloud_usage_bug_fixes
pnadolny13 Jun 27, 2023
cbf3f49
fix sqlfluff errors
pnadolny13 Jun 27, 2023
9156223
refactor deployments join thats no longer needed
pnadolny13 Jun 27, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 4 additions & 9 deletions data/transform/models/marts/telemetry/base/cloud_executions_base.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@ WITH open_source_agg AS (
MIN(
CASE WHEN cli_command = 'schedule' THEN started_ts END
) AS oss_run_started_ts,
MIN(
CASE WHEN cli_command = 'schedule' THEN finished_ts END
MAX(
finished_ts
) AS oss_run_finished_ts
FROM {{ ref('fact_cli_executions') }}
WHERE cloud_execution_id IS NOT NULL
Expand All @@ -34,6 +34,7 @@ SELECT
stg_dynamodb__workload_metadata_table.command_text_hash,
stg_dynamodb__workload_metadata_table.cloud_job_name_hash,
stg_dynamodb__workload_metadata_table.cloud_schedule_name_hash,
stg_dynamodb__workload_metadata_table.cloud_deployment_name_hash,
cloud_schedule_frequency.schedule_freq_day,
cloud_schedule_frequency.schedule_freq_rolling_avg,
COALESCE(
Expand Down Expand Up @@ -102,12 +103,6 @@ INNER JOIN {{ ref('stg_dynamodb__projects_table') }}
ON
stg_dynamodb__workload_metadata_table.cloud_project_id
= stg_dynamodb__projects_table.cloud_project_id
LEFT JOIN {{ ref('stg_dynamodb__project_deployments') }}
ON
stg_dynamodb__workload_metadata_table.cloud_project_id
= stg_dynamodb__project_deployments.cloud_project_id
AND stg_dynamodb__workload_metadata_table.cloud_environment_name_hash
= stg_dynamodb__project_deployments.cloud_environment_name_hash
LEFT JOIN {{ ref('stg_dynamodb__project_schedules_table') }}
ON
stg_dynamodb__workload_metadata_table.cloud_project_id
Expand All @@ -116,7 +111,7 @@ LEFT JOIN {{ ref('stg_dynamodb__project_schedules_table') }}
= stg_dynamodb__project_schedules_table.tenant_resource_key
AND stg_dynamodb__workload_metadata_table.cloud_schedule_name_hash
= stg_dynamodb__project_schedules_table.cloud_schedule_name_hash
AND stg_dynamodb__project_deployments.cloud_deployment_name_hash
AND stg_dynamodb__workload_metadata_table.cloud_deployment_name_hash
= stg_dynamodb__project_schedules_table.cloud_deployment_name_hash
LEFT JOIN open_source_agg
ON
Expand Down
6 changes: 1 addition & 5 deletions data/transform/models/marts/telemetry/base/cloud_schedule_frequency.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,6 @@ WITH joined AS (
stg_dynamodb__workload_metadata_table.started_ts::DATE AS date_day,
stg_dynamodb__workload_metadata_table.cloud_execution_id
FROM {{ ref('stg_dynamodb__workload_metadata_table') }}
LEFT JOIN {{ ref('stg_dynamodb__project_deployments') }}
ON
stg_dynamodb__workload_metadata_table.cloud_environment_name_hash
= stg_dynamodb__project_deployments.cloud_environment_name_hash
LEFT JOIN {{ ref('stg_dynamodb__project_schedules_table') }}
ON
stg_dynamodb__workload_metadata_table.cloud_schedule_name_hash
Expand All @@ -17,7 +13,7 @@ WITH joined AS (
AND stg_dynamodb__workload_metadata_table.cloud_project_id
= stg_dynamodb__project_schedules_table.cloud_project_id
AND
stg_dynamodb__project_deployments.cloud_deployment_name_hash
stg_dynamodb__workload_metadata_table.cloud_deployment_name_hash
= stg_dynamodb__project_schedules_table.cloud_deployment_name_hash
WHERE
stg_dynamodb__project_schedules_table.cloud_schedule_name_hash
Expand Down
1 change: 1 addition & 0 deletions data/transform/models/staging/dynamodb/stg_dynamodb__project_schedules_table.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ renamed AS (
enabled AS is_enabled,
tenant_resource_key,
cloud_project_id,
eventbridge_name,
SHA2_HEX(cloud_deployment_name) AS cloud_deployment_name_hash,
SHA2_HEX(cloud_schedule_name) AS cloud_schedule_name_hash
FROM clean_source
Expand Down
4 changes: 3 additions & 1 deletion data/transform/models/staging/dynamodb/stg_dynamodb__projects_table.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,9 @@ renamed AS (
project_surrogate_key,
project_id AS cloud_project_id,
project_name,
tenant_resource_key
tenant_resource_key,
SHA2_HEX(git_repository) AS git_repository_hash,
SHA2_HEX(project_root_path) AS project_root_path_hash
FROM source
WHERE row_num = 1

Expand Down
4 changes: 3 additions & 1 deletion data/transform/models/staging/dynamodb/stg_dynamodb__workload_metadata_table.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ renamed AS (
NULLIF(exit_code, 'N/A')::INT AS cloud_exit_code,
NULLIF(start_time, 'N/A')::TIMESTAMP_NTZ AS started_ts,
ttl::INT AS cloud_run_ttl,
stopped_reason,
SHA2_HEX(command_text) AS command_text_hash,
SHA2_HEX(environment_name) AS cloud_environment_name_hash,
SHA2_HEX(job_name) AS cloud_job_name_hash,
Expand All @@ -27,7 +28,8 @@ renamed AS (
) AS tenant_resource_key,
SPLIT_PART(
"TENANT_RESOURCE_KEY::PROJECT_ID", '::', 2 -- noqa: RF05
) AS cloud_project_id
) AS cloud_project_id,
SHA2_HEX(deployment_name) AS cloud_deployment_name_hash
FROM source
WHERE
row_num = 1
Expand Down