Skip to content
This repository has been archived by the owner on Nov 25, 2024. It is now read-only.

Remove sql package usage #3

Merged
merged 4 commits into from
Jan 4, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 0 additions & 9 deletions environments.json

This file was deleted.

8 changes: 4 additions & 4 deletions includes/connector_daily_api_calls.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
const sql = require("@dataform/sql")();
const sql = require("./sql")

module.exports = (params) => {

Expand All @@ -8,7 +8,7 @@ module.exports = (params) => {
with daily_api_calls as (
select
connector_name,
${sql.timestamps.truncate(sql.asTimestamp("created_at"), "day")} as date,
${sql.timestampTruncate(sql.asTimestamp("created_at"), "day")} as date,
count(*) as number_of_api_calls
from
${ctx.ref(params.defaultConfig.schema, params.stagingTablePrefix + "fivetran_log_log")}
Expand All @@ -32,6 +32,6 @@ from
${ctx.ref(params.defaultConfig.schema, "fivetran_log_connector_status")} as connector_status
left join daily_api_calls on daily_api_calls.connector_name = connector_status.connector_name
where
${sql.asTimestamp("daily_api_calls.date")} <= ${sql.timestamps.currentUTC()}
${sql.asTimestamp("daily_api_calls.date")} <= ${sql.currentUTC()}
`)
}
}
4 changes: 2 additions & 2 deletions includes/connector_status.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
const sql = require("@dataform/sql")();
const sql = require("./sql")

module.exports = (params) => {

Expand All @@ -24,7 +24,7 @@ select
from
${ctx.ref(params.defaultConfig.schema, params.stagingTablePrefix + "fivetran_log_log")}
where
${sql.timestamps.diff("day", sql.timestamps.currentUTC(), "created_at")} <= 30
${sql.timestampDiff("day", sql.currentUTC(), "created_at")} <= 30
and event_subtype in (
'create_table',
'alter_table',
Expand Down
4 changes: 2 additions & 2 deletions includes/mar_table_history.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
const sql = require("@dataform/sql")();
const sql = require("./sql")

module.exports = (params) => {

Expand All @@ -9,7 +9,7 @@ with
active_volume as (
select
*,
${sql.timestamps.truncate(sql.asTimestamp("measured_at"), "month")} as measured_month
${sql.timestampTruncate(sql.asTimestamp("measured_at"), "month")} as measured_month
from
${ctx.ref(params.defaultConfig.schema, params.stagingTablePrefix + "fivetran_log_active_volume")}
where
Expand Down
63 changes: 63 additions & 0 deletions includes/sql.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
const getDialect = () => {
const dataformWarehouse = global.dataform.projectConfig.warehouse;
if (!dataformWarehouse) {
return "standard";
}
return {
bigquery: "standard",
redshift: "redshift",
postgres: "postgres",
snowflake: "snowflake",
sqldatawarehouse: "mssql",
}[dataformWarehouse];
};

const asTimestamp = (castableToTimestamp) => {
return `cast(${castableToTimestamp} as timestamp)`;
};

const timestampDiff = (datePart, start, end) => {
const dialect = getDialect();
if (dialect === "snowflake" || dialect === "redshift") {
return `datediff(${datePart}, ${start}, ${end})`;
}
return `timestamp_diff(${end}, ${start}, ${datePart})`;
};

const timestampTruncate = (timestamp, timestampUnit) => {
const dialect = getDialect();
if (dialect === "snowflake") {
return `date_trunc(${timestampUnit}, ${timestamp})`;
}
if (dialect === "redshift") {
return `date_trunc('${timestampUnit}', ${timestamp})`;
}
return `timestamp_trunc(${timestamp}, ${timestampUnit})`;
};

const currentUTC = () => {
const dialect = getDialect();
if (dialect === "redshift") {
return "current_timestamp::timestamp";
}
if (dialect === "snowflake") {
return "convert_timezone('UTC', current_timestamp())::timestamp";
}
return "current_timestamp()";
};

const stringAgg = (field, delimiter = ",") => {
const dialect = getDialect();
if (dialect === "snowflake" || dialect === "redshift") {
return `listagg(${field}, '${delimiter}')`;
}
return `string_agg(${field}, '${delimiter}')`;
};

module.exports = {
asTimestamp,
timestampDiff,
timestampTruncate,
currentUTC,
stringAgg,
};
139 changes: 133 additions & 6 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 1 addition & 2 deletions package.json
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
{
"name": "dataform-fivetran-log",
"dependencies": {
"@dataform/core": "1.15.5",
"@dataform/sql": "0.3.0"
"@dataform/core": "1.15.5"
}
}