From d8e1985599af9d7fa2cdeff85d52a60057360aed Mon Sep 17 00:00:00 2001 From: Elias Kassell Date: Tue, 3 Jan 2023 16:50:59 +0000 Subject: [PATCH 1/4] Remove dependence on the @dataform/sql package --- includes/connector_daily_api_calls.js | 8 +- includes/connector_status.js | 4 +- includes/mar_table_history.js | 4 +- includes/sql.js | 60 +++++++++++ package-lock.json | 139 ++++++++++++++++++++++++-- package.json | 3 +- 6 files changed, 202 insertions(+), 16 deletions(-) create mode 100644 includes/sql.js diff --git a/includes/connector_daily_api_calls.js b/includes/connector_daily_api_calls.js index ac97d15..663922a 100644 --- a/includes/connector_daily_api_calls.js +++ b/includes/connector_daily_api_calls.js @@ -1,14 +1,14 @@ -const sql = require("@dataform/sql")(); +const sql = require("./sql") module.exports = (params) => { - + return publish("fivetran_log_connector_daily_api_calls", { ...params.defaultConfig }).query(ctx => ` with daily_api_calls as ( select connector_name, - ${sql.timestamps.truncate(sql.asTimestamp("created_at"), "day")} as date, + ${sql.timestampTruncate(sql.asTimestamp("created_at"), "day")} as date, count(*) as number_of_api_calls from ${ctx.ref(params.defaultConfig.schema, params.stagingTablePrefix + "fivetran_log_log")} @@ -32,6 +32,6 @@ from ${ctx.ref(params.defaultConfig.schema, "fivetran_log_connector_status")} as connector_status left join daily_api_calls on daily_api_calls.connector_name = connector_status.connector_name where - ${sql.asTimestamp("daily_api_calls.date")} <= ${sql.timestamps.currentUTC()} + ${sql.asTimestamp("daily_api_calls.date")} <= ${sql.currentUTC()} `) } \ No newline at end of file diff --git a/includes/connector_status.js b/includes/connector_status.js index d4a4af4..e6b87bb 100644 --- a/includes/connector_status.js +++ b/includes/connector_status.js @@ -1,4 +1,4 @@ -const sql = require("@dataform/sql")(); +const sql = require("./sql") module.exports = (params) => { @@ -24,7 +24,7 @@ select from ${ctx.ref(params.defaultConfig.schema, params.stagingTablePrefix + "fivetran_log_log")} where - ${sql.timestamps.diff("day", sql.timestamps.currentUTC(), "created_at")} <= 30 + ${sql.timestampDiff("day", sql.currentUTC(), "created_at")} <= 30 and event_subtype in ( 'create_table', 'alter_table', diff --git a/includes/mar_table_history.js b/includes/mar_table_history.js index 54310d7..0c2d084 100644 --- a/includes/mar_table_history.js +++ b/includes/mar_table_history.js @@ -1,4 +1,4 @@ -const sql = require("@dataform/sql")(); +const sql = require("./sql") module.exports = (params) => { @@ -9,7 +9,7 @@ with active_volume as ( select *, - ${sql.timestamps.truncate(sql.asTimestamp("measured_at"), "month")} as measured_month + ${sql.timestampTruncate(sql.asTimestamp("measured_at"), "month")} as measured_month from ${ctx.ref(params.defaultConfig.schema, params.stagingTablePrefix + "fivetran_log_active_volume")} where diff --git a/includes/sql.js b/includes/sql.js new file mode 100644 index 0000000..4947337 --- /dev/null +++ b/includes/sql.js @@ -0,0 +1,60 @@ +const getDialect = () => { + const dataformWarehouse = global.dataform.projectConfig.warehouse; + if (!dataformWarehouse) { + return "standard"; + } + return { + bigquery: "standard", + redshift: "redshift", + postgres: "postgres", + snowflake: "snowflake", + sqldatawarehouse: "mssql", + }; +}; + +const asTimestamp = (castableToTimestamp) => { + return `cast(${castableToTimestamp} as timestamp)`; +}; + +const timestampDiff = (datePart, start, end) => { + const dialect = getDialect(); + if (dialect === "snowflake" || dialect === "redshift") { + return `datediff(${datePart}, ${start}, ${end})`; + } + return `timestamp_diff(${end}, ${start}, ${datePart})`; +}; + +const timestampTruncate = (timestamp, timestampUnit) => { + if (this.dialect === "snowflake") { + return `date_trunc(${timestampUnit}, ${timestamp})`; + } + if (this.dialect === "redshift") { + return `date_trunc('${timestampUnit}', ${timestamp})`; + } + return `timestamp_trunc(${timestamp}, ${timestampUnit})`; +}; + +const currentUTC = () => { + if (this.dialect === "redshift") { + return "current_timestamp::timestamp"; + } + if (this.dialect === "snowflake") { + return "convert_timezone('UTC', current_timestamp())::timestamp"; + } + return "current_timestamp()"; +}; + +function stringAgg(field, delimiter = ",") { + if (this.dialect === "snowflake" || this.dialect === "redshift") { + return `listagg(${field}, '${delimiter}')`; + } + return `string_agg(${field}, '${delimiter}')`; +} + +module.exports = { + asTimestamp, + timestampDiff, + timestampTruncate, + currentUTC, + stringAgg, +}; diff --git a/package-lock.json b/package-lock.json index 35d30b2..85ec16c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,138 @@ { + "name": "dataform-fivetran-log", + "lockfileVersion": 2, "requires": true, - "lockfileVersion": 1, + "packages": { + "": { + "name": "dataform-fivetran-log", + "dependencies": { + "@dataform/core": "1.15.5" + } + }, + "node_modules/@dataform/core": { + "version": "1.15.5", + "resolved": "https://registry.npmjs.org/@dataform/core/-/core-1.15.5.tgz", + "integrity": "sha512-Tn9v1Voxojgr9lEb738+0+BFfY3mx7XDC09nRgMmxYrErtmdrJcTKzsVPDZad3r7axyaZafSrd7rRo9aGsNnow==", + "dependencies": { + "moo": "^0.5.0", + "protobufjs": "6.8.8", + "semver": "^5.6.0", + "tarjan-graph": "^2.0.0" + } + }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha1-m4sMxmPWaafY9vXQiToU00jzD78=" + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==" + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", + "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==" + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", + "integrity": "sha1-NVy8mLr61ZePntCV85diHx0Ga3A=" + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", + "integrity": "sha1-upn7WYYUr2VwDBYZ/wbUVLDYTEU=", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1", + "@protobufjs/inquire": "^1.1.0" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha1-Xp4avctz/Ap8uLKR33jIy9l7h9E=" + }, + "node_modules/@protobufjs/inquire": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", + "integrity": "sha1-/yAOPnzyQp4tyvwRQIKOjMY48Ik=" + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha1-bMKyDFya1q0NzP0hynZz2Nf79o0=" + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha1-Cf0V8tbTq/qbZbw2ZQbWrXhG/1Q=" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", + "integrity": "sha1-p3c2C1s5oaLlEG+OhY8v0tBgxXA=" + }, + "node_modules/@types/long": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.1.tgz", + "integrity": "sha512-5tXH6Bx/kNGd3MgffdmP4dy2Z+G4eaXw0SE81Tq3BNadtnMR5/ySMzX4SLEzHJzSmPNn4HIdpQsBvXMUykr58w==" + }, + "node_modules/@types/node": { + "version": "10.17.44", + "resolved": "https://registry.npmjs.org/@types/node/-/node-10.17.44.tgz", + "integrity": "sha512-vHPAyBX1ffLcy4fQHmDyIUMUb42gHZjPHU66nhvbMzAWJqHnySGZ6STwN3rwrnSd1FHB0DI/RWgGELgKSYRDmw==" + }, + "node_modules/long": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz", + "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==" + }, + "node_modules/moo": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/moo/-/moo-0.5.1.tgz", + "integrity": "sha512-I1mnb5xn4fO80BH9BLcF0yLypy2UKl+Cb01Fu0hJRkJjlCRtxZMWkTdAtDd5ZqCOxtCkhmRwyI57vWT+1iZ67w==" + }, + "node_modules/protobufjs": { + "version": "6.8.8", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.8.8.tgz", + "integrity": "sha512-AAmHtD5pXgZfi7GMpllpO3q1Xw1OYldr+dMUlAnffGTAhqkg72WdmSY71uKBF/JuyiKs8psYbtKrhi0ASCD8qw==", + "hasInstallScript": true, + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.4", + "@protobufjs/eventemitter": "^1.1.0", + "@protobufjs/fetch": "^1.1.0", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.0", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.0", + "@types/long": "^4.0.0", + "@types/node": "^10.1.0", + "long": "^4.0.0" + }, + "bin": { + "pbjs": "bin/pbjs", + "pbts": "bin/pbts" + } + }, + "node_modules/semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "bin": { + "semver": "bin/semver" + } + }, + "node_modules/tarjan-graph": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/tarjan-graph/-/tarjan-graph-2.0.0.tgz", + "integrity": "sha512-fDe57nO2Ukw2A/jHwVeiEgERGrGHukf3aHmR/YZ9BrveOtHVlFs289AnVeb1wD2aj9g01ZZ6f7VyMJ2QxI2NBQ==" + } + }, "dependencies": { "@dataform/core": { "version": "1.15.5", @@ -13,11 +145,6 @@ "tarjan-graph": "^2.0.0" } }, - "@dataform/sql": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/@dataform/sql/-/sql-0.3.0.tgz", - "integrity": "sha512-iei7NBNI6h4pxCSRWMzEAfWLPyjcOrrNO81USkk9wXJUIThd/z+2y/wvQtm0EFmDHeVdORzJJ15hmoYyTCgKFA==" - }, "@protobufjs/aspromise": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", diff --git a/package.json b/package.json index a21ba78..67b9dd8 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,6 @@ { "name": "dataform-fivetran-log", "dependencies": { - "@dataform/core": "1.15.5", - "@dataform/sql": "0.3.0" + "@dataform/core": "1.15.5" } } From b9d227c70bd610b671fc5689e381b2b331c841d8 Mon Sep 17 00:00:00 2001 From: Elias Kassell Date: Tue, 3 Jan 2023 18:01:19 +0000 Subject: [PATCH 2/4] Formatting --- includes/connector_daily_api_calls.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/includes/connector_daily_api_calls.js b/includes/connector_daily_api_calls.js index 663922a..84b256f 100644 --- a/includes/connector_daily_api_calls.js +++ b/includes/connector_daily_api_calls.js @@ -1,7 +1,7 @@ const sql = require("./sql") module.exports = (params) => { - + return publish("fivetran_log_connector_daily_api_calls", { ...params.defaultConfig }).query(ctx => ` @@ -34,4 +34,4 @@ from where ${sql.asTimestamp("daily_api_calls.date")} <= ${sql.currentUTC()} `) -} \ No newline at end of file +} From 04f83fdec1c996a3ae251c7b4fc6abb16ba620a1 Mon Sep 17 00:00:00 2001 From: Elias Kassell Date: Tue, 3 Jan 2023 18:09:35 +0000 Subject: [PATCH 3/4] Fix dialect --- includes/sql.js | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/includes/sql.js b/includes/sql.js index 4947337..6dee729 100644 --- a/includes/sql.js +++ b/includes/sql.js @@ -9,7 +9,7 @@ const getDialect = () => { postgres: "postgres", snowflake: "snowflake", sqldatawarehouse: "mssql", - }; + }[dataformWarehouse]; }; const asTimestamp = (castableToTimestamp) => { @@ -25,27 +25,30 @@ const timestampDiff = (datePart, start, end) => { }; const timestampTruncate = (timestamp, timestampUnit) => { - if (this.dialect === "snowflake") { + const dialect = getDialect(); + if (dialect === "snowflake") { return `date_trunc(${timestampUnit}, ${timestamp})`; } - if (this.dialect === "redshift") { + if (dialect === "redshift") { return `date_trunc('${timestampUnit}', ${timestamp})`; } return `timestamp_trunc(${timestamp}, ${timestampUnit})`; }; const currentUTC = () => { - if (this.dialect === "redshift") { + const dialect = getDialect(); + if (dialect === "redshift") { return "current_timestamp::timestamp"; } - if (this.dialect === "snowflake") { + if (dialect === "snowflake") { return "convert_timezone('UTC', current_timestamp())::timestamp"; } return "current_timestamp()"; }; function stringAgg(field, delimiter = ",") { - if (this.dialect === "snowflake" || this.dialect === "redshift") { + const dialect = getDialect(); + if (dialect === "snowflake" || dialect === "redshift") { return `listagg(${field}, '${delimiter}')`; } return `string_agg(${field}, '${delimiter}')`; From d5ae46167ea88aebabd8ecae82c0ece7867b8f0f Mon Sep 17 00:00:00 2001 From: Elias Kassell Date: Wed, 4 Jan 2023 08:56:09 +0000 Subject: [PATCH 4/4] Remove environments.json --- environments.json | 9 --------- includes/sql.js | 4 ++-- 2 files changed, 2 insertions(+), 11 deletions(-) delete mode 100644 environments.json diff --git a/environments.json b/environments.json deleted file mode 100644 index b825e2f..0000000 --- a/environments.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "environments": [ - { - "name": "production", - "configOverride": {}, - "gitRef": "master" - } - ] -} diff --git a/includes/sql.js b/includes/sql.js index 6dee729..0997535 100644 --- a/includes/sql.js +++ b/includes/sql.js @@ -46,13 +46,13 @@ const currentUTC = () => { return "current_timestamp()"; }; -function stringAgg(field, delimiter = ",") { +const stringAgg = (field, delimiter = ",") => { const dialect = getDialect(); if (dialect === "snowflake" || dialect === "redshift") { return `listagg(${field}, '${delimiter}')`; } return `string_agg(${field}, '${delimiter}')`; -} +}; module.exports = { asTimestamp,