From 3a158cb215b8bc1aaaa1808d4a8d2fa9a3c1d008 Mon Sep 17 00:00:00 2001 From: Elias Kassell Date: Tue, 3 Jan 2023 14:23:09 +0000 Subject: [PATCH 1/5] Remove @dataform/sql package and common_v1 example --- examples/common_v1/BUILD | 18 - examples/common_v1/dataform.json | 5 - .../definitions/example_backticks.sql | 3 - .../common_v1/definitions/example_deferred.js | 1 - .../definitions/example_incremental.sql | 4 - .../common_v1/definitions/example_inline.sql | 2 - .../definitions/example_js_blocks.sql | 5 - .../common_v1/definitions/example_table.sql | 2 - .../definitions/example_table_dependency.sql | 3 - .../definitions/example_using_inline.sql | 3 - .../common_v1/definitions/example_view.sql | 2 - .../common_v1/definitions/sample_data.sql | 3 - examples/common_v1/definitions/test.js | 10 - examples/common_v1/includes/macros.js | 4 - examples/common_v1/package.json | 6 - packages/@dataform/sql/BUILD | 53 -- packages/@dataform/sql/index.ts | 20 - sql/BUILD | 37 -- sql/README.md | 61 -- sql/builders/aggregate.ts | 83 --- sql/builders/from.ts | 66 -- sql/builders/index.ts | 4 - sql/builders/join.ts | 42 -- sql/builders/json.ts | 21 - sql/builders/select.ts | 39 -- sql/builders/union.ts | 25 - sql/builders/with.ts | 25 - sql/index.ts | 263 -------- sql/integration.spec.ts | 572 ------------------ sql/timestamps.ts | 98 --- tests/api/BUILD | 2 - tests/api/examples.spec.ts | 364 ----------- 32 files changed, 1846 deletions(-) delete mode 100644 examples/common_v1/BUILD delete mode 100644 examples/common_v1/dataform.json delete mode 100644 examples/common_v1/definitions/example_backticks.sql delete mode 100644 examples/common_v1/definitions/example_deferred.js delete mode 100644 examples/common_v1/definitions/example_incremental.sql delete mode 100644 examples/common_v1/definitions/example_inline.sql delete mode 100644 examples/common_v1/definitions/example_js_blocks.sql delete mode 100644 examples/common_v1/definitions/example_table.sql delete mode 100644 examples/common_v1/definitions/example_table_dependency.sql delete mode 100644 examples/common_v1/definitions/example_using_inline.sql delete mode 100644 examples/common_v1/definitions/example_view.sql delete mode 100644 examples/common_v1/definitions/sample_data.sql delete mode 100644 examples/common_v1/definitions/test.js delete mode 100644 examples/common_v1/includes/macros.js delete mode 100644 examples/common_v1/package.json delete mode 100644 packages/@dataform/sql/BUILD delete mode 100644 packages/@dataform/sql/index.ts delete mode 100644 sql/BUILD delete mode 100644 sql/README.md delete mode 100644 sql/builders/aggregate.ts delete mode 100644 sql/builders/from.ts delete mode 100644 sql/builders/index.ts delete mode 100644 sql/builders/join.ts delete mode 100644 sql/builders/json.ts delete mode 100644 sql/builders/select.ts delete mode 100644 sql/builders/union.ts delete mode 100644 sql/builders/with.ts delete mode 100644 sql/index.ts delete mode 100644 sql/integration.spec.ts delete mode 100644 sql/timestamps.ts diff --git a/examples/common_v1/BUILD b/examples/common_v1/BUILD deleted file mode 100644 index 3672e2d24..000000000 --- a/examples/common_v1/BUILD +++ /dev/null @@ -1,18 +0,0 @@ -package(default_visibility = ["//tests:__subpackages__"]) - -load("//tools:node_modules.bzl", "node_modules") - -filegroup( - name = "files", - srcs = glob([ - "**/*.*", - ]), -) - -node_modules( - name = "node_modules", - deps = [ - "//packages/@dataform/core:package_tar", - "//packages/@dataform/sql:package_tar", - ], -) diff --git a/examples/common_v1/dataform.json b/examples/common_v1/dataform.json deleted file mode 100644 index 5a4532042..000000000 --- a/examples/common_v1/dataform.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "defaultSchema": "df_integration_test", - "assertionSchema": "df_integration_test_assertions", - "defaultLocation": "US" -} diff --git a/examples/common_v1/definitions/example_backticks.sql b/examples/common_v1/definitions/example_backticks.sql deleted file mode 100644 index a11fff3f0..000000000 --- a/examples/common_v1/definitions/example_backticks.sql +++ /dev/null @@ -1,3 +0,0 @@ ---js dependencies("sample_data"); -select * from -`tada-analytics.df_integration_test.sample_data\` diff --git a/examples/common_v1/definitions/example_deferred.js b/examples/common_v1/definitions/example_deferred.js deleted file mode 100644 index 1a8ef1e3d..000000000 --- a/examples/common_v1/definitions/example_deferred.js +++ /dev/null @@ -1 +0,0 @@ -macros.deferredPublish("example_deferred", "select 1 as test"); diff --git a/examples/common_v1/definitions/example_incremental.sql b/examples/common_v1/definitions/example_incremental.sql deleted file mode 100644 index b90344d0b..000000000 --- a/examples/common_v1/definitions/example_incremental.sql +++ /dev/null @@ -1,4 +0,0 @@ ---js type("incremental"); ---js where(ctx => `ts > (select max(ts) from ${self()}) or (select max(ts) from ${self()}) is null`) ---js const sql = require("@dataform/sql"); -select ${sql().timestamps.currentUTC()} as ts diff --git a/examples/common_v1/definitions/example_inline.sql b/examples/common_v1/definitions/example_inline.sql deleted file mode 100644 index fba34a82e..000000000 --- a/examples/common_v1/definitions/example_inline.sql +++ /dev/null @@ -1,2 +0,0 @@ -${type("inline")} -select * from ${ref("sample_data")} diff --git a/examples/common_v1/definitions/example_js_blocks.sql b/examples/common_v1/definitions/example_js_blocks.sql deleted file mode 100644 index bd922651f..000000000 --- a/examples/common_v1/definitions/example_js_blocks.sql +++ /dev/null @@ -1,5 +0,0 @@ -/*js -type("table"); -*/ ---js var foo = () => "foo"; -select 1 as ${foo()} diff --git a/examples/common_v1/definitions/example_table.sql b/examples/common_v1/definitions/example_table.sql deleted file mode 100644 index d0cd09dcf..000000000 --- a/examples/common_v1/definitions/example_table.sql +++ /dev/null @@ -1,2 +0,0 @@ -${type("table")} -select * from ${ref("sample_data")} diff --git a/examples/common_v1/definitions/example_table_dependency.sql b/examples/common_v1/definitions/example_table_dependency.sql deleted file mode 100644 index 9183bc39b..000000000 --- a/examples/common_v1/definitions/example_table_dependency.sql +++ /dev/null @@ -1,3 +0,0 @@ -${type("view")} --- Make sure it's possible to drop tables which have dependent views. -select * from ${ref("example_table")} diff --git a/examples/common_v1/definitions/example_using_inline.sql b/examples/common_v1/definitions/example_using_inline.sql deleted file mode 100644 index bb9750753..000000000 --- a/examples/common_v1/definitions/example_using_inline.sql +++ /dev/null @@ -1,3 +0,0 @@ -${type("table")} -select * from ${ref("example_inline")} -where true diff --git a/examples/common_v1/definitions/example_view.sql b/examples/common_v1/definitions/example_view.sql deleted file mode 100644 index e65ebe886..000000000 --- a/examples/common_v1/definitions/example_view.sql +++ /dev/null @@ -1,2 +0,0 @@ -${type("view")} -select * from ${ref("sample_data")} diff --git a/examples/common_v1/definitions/sample_data.sql b/examples/common_v1/definitions/sample_data.sql deleted file mode 100644 index 2008e4c43..000000000 --- a/examples/common_v1/definitions/sample_data.sql +++ /dev/null @@ -1,3 +0,0 @@ -select 1 as sample union all -select 2 as sample union all -select 3 as sample diff --git a/examples/common_v1/definitions/test.js b/examples/common_v1/definitions/test.js deleted file mode 100644 index 3aaed1a65..000000000 --- a/examples/common_v1/definitions/test.js +++ /dev/null @@ -1,10 +0,0 @@ -assert("sample_data_assertion") - .query(ctx => `select * from ${ctx.ref("sample_data")} where sample > 3`) - .dependencies([ "example_backticks", - "example_deferred", - "example_incremental", - "example_inline", - "example_js_blocks", - "example_table", - "example_using_inline", - "example_view"]); diff --git a/examples/common_v1/includes/macros.js b/examples/common_v1/includes/macros.js deleted file mode 100644 index c3dddb8c6..000000000 --- a/examples/common_v1/includes/macros.js +++ /dev/null @@ -1,4 +0,0 @@ -module.exports = { - foo: value => `foo_${value}`, - deferredPublish: (name, arg) => publish(name, arg) -}; diff --git a/examples/common_v1/package.json b/examples/common_v1/package.json deleted file mode 100644 index 7bf7a5673..000000000 --- a/examples/common_v1/package.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "name": "@dataform/examples-bigquery", - "version": "0.1.0", - "description": "Example Dataform project (BigQuery).", - "private": true -} diff --git a/packages/@dataform/sql/BUILD b/packages/@dataform/sql/BUILD deleted file mode 100644 index b82aa386f..000000000 --- a/packages/@dataform/sql/BUILD +++ /dev/null @@ -1,53 +0,0 @@ -load("//tools:ts_library.bzl", "ts_library") -load("//packages:index.bzl", "pkg_bundle", "pkg_bundle_dts", "pkg_json", "pkg_npm_tar") - -package(default_visibility = ["//visibility:public"]) - -ts_library( - name = "sql", - srcs = glob(["**/*.ts"]), - deps = [ - "//sql", - "@npm//@types/node", - ], -) - -pkg_json( - name = "json", - package_name = "@dataform/sql", - description = "General purpose SQL helper functions for building queries in JS across dialects.", - layers = [ - "//:package.json", - "//packages/@dataform:package.layer.json", - ], - main = "bundle.js", - types = "bundle.d.ts", - version = "0.4.0", -) - -pkg_bundle( - name = "bundle", - entry_point = "index.ts", - externals = [], - deps = [ - ":sql", - ], -) - -pkg_bundle_dts( - name = "bundle.d", - entry_point = "index.ts", - externals = [], - deps = [ - ":sql", - ], -) - -pkg_npm_tar( - name = "package", - deps = [ - ":bundle", - ":bundle.d", - ":package.json", - ], -) diff --git a/packages/@dataform/sql/index.ts b/packages/@dataform/sql/index.ts deleted file mode 100644 index c7721c5bd..000000000 --- a/packages/@dataform/sql/index.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { ISqlDialect, Sql } from "df/sql"; - -/** - * If we are in a dataform context, try to infer the current sql dialect, otherwise use "standard" as a default. - */ -const getDefaultDialect = () => { - const dataformWarehouse = (global as any).dataform.projectConfig.warehouse; - if (!dataformWarehouse) { - return "standard"; - } - return ({ - bigquery: "standard", - redshift: "redshift", - postgres: "postgres", - snowflake: "snowflake", - sqldatawarehouse: "mssql" - } as { [key: string]: ISqlDialect })[dataformWarehouse]; -}; - -module.exports = (dialect: ISqlDialect = getDefaultDialect()) => new Sql(dialect); diff --git a/sql/BUILD b/sql/BUILD deleted file mode 100644 index 85e33a3bd..000000000 --- a/sql/BUILD +++ /dev/null @@ -1,37 +0,0 @@ -load("//tools:ts_library.bzl", "ts_library") -load("//testing:index.bzl", "ts_test_suite") - -package(default_visibility = ["//visibility:public"]) - -ts_library( - name = "sql", - srcs = glob( - [ - "**/*.ts", - ], - exclude = ["**/*.spec.ts"], - ), - deps = [ - "//:modules-fix", - "@npm//@types/long", - ], -) - -ts_test_suite( - name = "tests", - srcs = glob(["**/*.spec.ts"]), - data = [ - "//test_credentials:bigquery.json", - "//test_credentials:snowflake.json", - "//test_credentials:redshift.json", - "@npm//source-map-support", - ], - templated_args = ["--node_options=--require=source-map-support/register"], - deps = [ - ":sql", - "//api", - "//testing", - "@npm//@types/chai", - "@npm//chai", - ], -) diff --git a/sql/README.md b/sql/README.md deleted file mode 100644 index 99199b449..000000000 --- a/sql/README.md +++ /dev/null @@ -1,61 +0,0 @@ -Useful functions for constructing SQL statements in Dataform packages. - -## Supported warehouses - -- BigQuery -- Redshift -- Snowflake - -_If you would like us to add support for another warehouse, please get in touch via [email](mailto:team@dataform.co) or [Slack](https://dataform.co/slack)_ - -## Installation - -(TODO) - -## Functions - -### Common SQL functions - -#### surrogateKey - -Creates a unique hash from a list of fields. Useful for generating a surrogate key for a table. - -`${sql.surrogateKey(["field_one", "field_two"])}` - -#### windowFunction - -Creates a window function with the given configuration. - -`${sql.windowFunction({ name: "window_function_name", value: "target_column", ignoreNulls: true, windowSpecification: { partitionFields: ["field_to_partition_by_one", "field_to_partition_by_two"], orderFields: ["field_to_order_by_one", "field_to_order_by_two"], frameClause: "rows between 0 preceding and unbounded following" } })}` - -#### asTimestamp - -Casts the field to timestamp type - -`${sql.asTimestamp("field")}` - -#### asString - -Casts the field to timestamp type - -`${sql.asString("field")}` - -#### stringAgg - -Groups the values in a field into a concatenated string, with an optional delimiter value that defaults to "," - -`${sql.stringAgg("field", "delimiter")}` - -### Timestamp functions - -#### diff - -Calculate the time difference between two timestamps. - -`${sql.timestamps.diff("date_part", "start_timestamp", "end_timestamp")}` - -#### add - -Add a period of time to a timestamp. - -`${sql.timestamps.add("timestamp", 1, "hour")` diff --git a/sql/builders/aggregate.ts b/sql/builders/aggregate.ts deleted file mode 100644 index 204f8cb91..000000000 --- a/sql/builders/aggregate.ts +++ /dev/null @@ -1,83 +0,0 @@ -import { IOrdering } from "df/sql/builders"; -import { - build, - indent, - ISelectBuilder, - ISelectOrBuilder, - ISelectSchema, - Select -} from "df/sql/builders/select"; - -export class AggregateBuilder implements ISelectBuilder { - private selectedDimensions: ISelectSchema = {}; - private selectedMetrics: ISelectSchema = {}; - private whereClauses: string[] = []; - private selectedOrdering: IOrdering; - private selectedLimit: number; - - constructor(private readonly from: ISelectOrBuilder) {} - - public metrics(select: MS): AggregateBuilder { - this.selectedMetrics = { ...this.selectedMetrics, ...select }; - return this; - } - - public dimensions(select: DS): AggregateBuilder { - this.selectedDimensions = { ...this.selectedDimensions, ...select }; - return this; - } - - public ordering(ordering: IOrdering) { - this.selectedOrdering = ordering; - return this; - } - - public where(...wheres: string[]) { - wheres.forEach(where => this.whereClauses.push(where)); - return this; - } - - public limit(limit: number) { - this.selectedLimit = limit; - return this; - } - - public build() { - const hasDimensions = Object.keys(this.selectedDimensions).length > 0; - const hasMetrics = Object.keys(this.selectedMetrics).length > 0; - const whereExpression = - this.whereClauses.length > 0 ? `\nwhere\n${indent(this.whereClauses.join(" and\n"))}` : ""; - const orderingExpression = this.selectedOrdering - ? `\norder by\n ${this.selectedOrdering.expression} ${ - this.selectedOrdering.descending ? "desc" : "asc" - }` - : ""; - const limitExpression = this.selectedLimit ? `\nlimit ${this.selectedLimit}` : ""; - return Select.create( - `(\n${indent( - `select\n` + - (!hasDimensions && !hasMetrics ? indent("*") : "") + - `${Object.keys(this.selectedDimensions) - .map(alias => indent(`${this.selectedDimensions[alias]} as ${alias}`)) - .join(",\n")}${hasDimensions && hasMetrics ? ",\n" : ""}` + - `${Object.keys(this.selectedMetrics) - .map(alias => indent(`${this.selectedMetrics[alias]} as ${alias}`)) - .join(",\n")}\n` + - `from\n` + - `${indent(build(this.from))}` + - `${whereExpression}` + - `${ - hasDimensions - ? `\ngroup by ${Array.from( - new Array(Object.keys(this.selectedDimensions).length).keys() - ) - .map(i => i + 1) - .join(", ")}` - : "" - }` + - `${orderingExpression}` + - `${limitExpression}` - )}\n)` - ); - } -} diff --git a/sql/builders/from.ts b/sql/builders/from.ts deleted file mode 100644 index e09ebf79d..000000000 --- a/sql/builders/from.ts +++ /dev/null @@ -1,66 +0,0 @@ -import { IOrdering } from "df/sql/builders"; -import { - build, - indent, - ISelectBuilder, - ISelectOrBuilder, - ISelectSchema, - Select -} from "df/sql/builders/select"; - -export class FromBuilder implements ISelectBuilder { - private columns: ISelectSchema = {}; - private whereClauses: string[] = []; - private selectedOrdering: IOrdering; - private selectedLimit: number; - - constructor(private readonly from: ISelectOrBuilder) {} - - public select(select: MS): FromBuilder { - if (select instanceof Array) { - select.forEach(value => (this.columns[value] = value)); - } else { - this.columns = { ...this.columns, ...select }; - } - return this; - } - - public ordering(ordering: IOrdering) { - this.selectedOrdering = ordering; - return this; - } - - public where(...wheres: string[]) { - wheres.forEach(where => this.whereClauses.push(where)); - return this; - } - - public limit(limit: number) { - this.selectedLimit = limit; - return this; - } - - public build() { - const hasColumns = Object.keys(this.columns).length > 0; - const whereExpression = - this.whereClauses.length > 0 ? `\nwhere\n${indent(this.whereClauses.join(" and\n "))}` : ""; - const orderingExpression = this.selectedOrdering - ? `\norder by\n ${this.selectedOrdering.expression} ${ - this.selectedOrdering.descending ? "desc" : "asc" - }` - : ""; - const limitExpression = this.selectedLimit ? `\nlimit ${this.selectedLimit}` : ""; - return Select.create(`(\n${indent(`select\n` + - (hasColumns - ? `${Object.keys(this.columns) - .map(alias => ` ${this.columns[alias]} as ${alias}`) - .join(",\n")}` - : indent("*")) + - `\nfrom\n` + - `${indent(build(this.from))}` + - `${whereExpression}` + - `${orderingExpression}` + - `${limitExpression}`)}\n)` - ); - } -} diff --git a/sql/builders/index.ts b/sql/builders/index.ts deleted file mode 100644 index e39b3132c..000000000 --- a/sql/builders/index.ts +++ /dev/null @@ -1,4 +0,0 @@ -export interface IOrdering { - expression: string; - descending?: boolean; -} diff --git a/sql/builders/join.ts b/sql/builders/join.ts deleted file mode 100644 index d12e1aac6..000000000 --- a/sql/builders/join.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { - build, - indent, - ISelectBuilder, - ISelectOrBuilder, - ISelectSchema, - Select -} from "df/sql/builders/select"; - -export interface IJoin { - select: ISelectOrBuilder; - on?: [string, string]; - type?: "base" | "left" | "right" | "inner" | "outer" | "cross" | "full outer"; -} - -export interface IJoins { - [alias: string]: IJoin; -} - -export class JoinBuilder - implements ISelectBuilder<{ [K in keyof J]: J[K] extends IJoin ? JS : {} }> { - constructor(private readonly joins: J) {} - - public build() { - const baseAlias = Object.keys(this.joins).find(key => !!key || key === "base"); - const baseSelect = this.joins[baseAlias]; - const joinAliases = Object.keys(this.joins).filter(alias => alias !== baseAlias); - return Select.create<{ [K in keyof J]: J[K] extends IJoin ? JS : {} }>(`${indent( - build(baseSelect.select) - )} ${baseAlias} -${joinAliases - .map( - alias => `${this.joins[alias].type} join -${build(this.joins[alias].select)} ${alias} ${ - !!this.joins[alias].on - ? `on (${baseAlias}.${this.joins[alias].on[0]} = ${alias}.${this.joins[alias].on[1]})` - : "" - }` - ) - .join(`\n`)}`); - } -} diff --git a/sql/builders/json.ts b/sql/builders/json.ts deleted file mode 100644 index ff137524f..000000000 --- a/sql/builders/json.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { Sql } from "df/sql"; -import { indent, ISelectBuilder, ISelectSchema, Select } from "df/sql/builders/select"; - -export class JSONBuilder implements ISelectBuilder { - constructor(private sql: Sql, private data: S[]) {} - - public build() { - return Select.create(`( -${indent( - this.data - .map( - row => - `select ${Object.keys(row) - .map(alias => `${this.sql.literal(row[alias] as string | number)} as ${alias}`) - .join(", ")}` - ) - .join(`\nunion all\n`) -)} -)`); - } -} diff --git a/sql/builders/select.ts b/sql/builders/select.ts deleted file mode 100644 index af1983697..000000000 --- a/sql/builders/select.ts +++ /dev/null @@ -1,39 +0,0 @@ -export interface ISelectSchema { - [alias: string]: string | number | ISelectSchema; -} - -export class Select { - public static create(query: string) { - return new Select(query); - } - constructor(public readonly query: string) {} - - public toString() { - return this.query; - } -} - -export interface ISelectBuilder { - build(): Select; -} - -export type ISelectOrBuilder = ISelectBuilder | Select | string; - -export function build(selectOrBuilder: ISelectOrBuilder) { - if (typeof selectOrBuilder === "string") { - return Select.create(selectOrBuilder); - } - - if (selectOrBuilder instanceof Select) { - return selectOrBuilder; - } - - return selectOrBuilder.build(); -} - -export function indent(value: string | Select) { - return String(value) - .split("\n") - .map(row => ` ${row}`) - .join("\n"); -} diff --git a/sql/builders/union.ts b/sql/builders/union.ts deleted file mode 100644 index 360bee820..000000000 --- a/sql/builders/union.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { - build, - indent, - ISelectBuilder, - ISelectOrBuilder, - ISelectSchema, - Select -} from "df/sql/builders/select"; - -export class UnionBuilder implements ISelectBuilder { - constructor(private selects: Array>) {} - - public build() { - // TODO: Check all columns have the same values for a union. - return Select.create( - `( -${indent( - this.selects.map(select => build(select).query).join(` -union all -`) -)} -)` - ); - } -} diff --git a/sql/builders/with.ts b/sql/builders/with.ts deleted file mode 100644 index ed1523212..000000000 --- a/sql/builders/with.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { build, indent, ISelectOrBuilder, ISelectSchema, Select } from "df/sql/builders/select"; - -export interface IWiths { - [alias: string]: ISelectOrBuilder; -} - -export class WithBuilder { - constructor(private readonly withs: IWiths) {} - - public select(select: ISelectOrBuilder) { - return Select.create( - (Object.keys(this.withs).length > 0 - ? `with -${Object.keys(this.withs) - .map( - key => `${key} as ( -${indent(build(this.withs[key]))} -)` - ) - .join(",\n")} -` - : "") + build(select).query - ); - } -} diff --git a/sql/index.ts b/sql/index.ts deleted file mode 100644 index 52f2f5698..000000000 --- a/sql/index.ts +++ /dev/null @@ -1,263 +0,0 @@ -import { AggregateBuilder } from "df/sql/builders/aggregate"; -import { FromBuilder } from "df/sql/builders/from"; -import { IJoins, JoinBuilder } from "df/sql/builders/join"; -import { JSONBuilder } from "df/sql/builders/json"; -import { ISelectOrBuilder, ISelectSchema } from "df/sql/builders/select"; -import { UnionBuilder } from "df/sql/builders/union"; -import { IWiths, WithBuilder } from "df/sql/builders/with"; -import { Timestamps } from "df/sql/timestamps"; - -export type ISqlDialect = "standard" | "snowflake" | "postgres" | "mssql" | "redshift"; - -export class Sql { - public readonly timestamps: Timestamps; - - constructor(private readonly dialect: ISqlDialect = "standard") { - this.timestamps = new Timestamps(dialect); - } - - public literal(value: string | number | null) { - if (value === null) { - return "null"; - } - if (typeof value === "string") { - return `'${value}'`; - } - return String(value); - } - - public not(expression: string): string { - return `not (${expression})`; - } - - public countDistinct(expression: string) { - return `count(distinct ${expression})`; - } - - public conditional(condition: string, then: string, otherwise: string = "null") { - if (this.dialect === "snowflake") { - return `iff(${condition}, ${then}, ${otherwise || "null"})`; - } - if (this.dialect === "postgres" || this.dialect === "redshift") { - return `case when ${condition} then ${then} else ${otherwise} end`; - } - return `if(${condition}, ${then}, ${otherwise || "null"})`; - } - - public equals(expression: string, expected: string) { - if (expected.trim().toLowerCase() === "null") { - return `${expression} is null`; - } - return `${expression} = ${expected}`; - } - - public gt(left: string, right: string) { - return `${left} > ${right}`; - } - - public gteq(left: string, right: string) { - return `${left} >= ${right}`; - } - - public lt(left: string, right: string) { - return `${left} < ${right}`; - } - - public lteq(left: string, right: string) { - return `${left} <= ${right}`; - } - - public in(expression: string, values: string[]) { - if (values.length === 0) { - // If there are no values, then this must be false. - return "false"; - } - // You can't check for nulls using "in". Instead, create an or expression using is null. - const containsNull = values.find(value => value.toLowerCase() === "null"); - const nonNullValues = values.filter(value => value.toLowerCase() !== "null"); - const ors: string[] = []; - if (containsNull) { - ors.push(this.equals(expression, "null")); - } - if (nonNullValues.length > 0) { - ors.push(`${expression} in (${values.join(", ")})`); - } - return this.or(ors); - } - - public sum(expression: string | number) { - return `sum(${expression})`; - } - - public min(expression: string | number) { - return `min(${expression})`; - } - - public max(expression: string | number) { - return `max(${expression})`; - } - - public avg(expression: string | number) { - return `avg(${expression})`; - } - - public count() { - return `sum(1)`; - } - - public coalesce(...expressions: string[]) { - return `coalesce(${expressions.join(", ")})`; - } - - public or(expressions: string[]) { - return `(${expressions.join(" or ")})`; - } - - public and(expressions: string[]) { - return expressions.join(" and "); - } - - public withWrappingBrackets(expression: string) { - return `(${expression})`; - } - - public safeDivide(numerator: string, denominator: string) { - return `${numerator} / nullif(${denominator}, 0)`; - } - - // Casting functions. - - public asTimestamp(castableToTimestamp: string) { - return `cast(${castableToTimestamp} as timestamp)`; - } - - public asString(castableToString: string) { - if (this.dialect === "postgres" || this.dialect === "redshift") { - return `cast(${castableToString} as varchar)`; - } - return `cast(${castableToString} as string)`; - } - - // Surrogate keys. - - public surrogateKey(columnNames: string[]) { - const columnsAsStrings = columnNames.map(id => this.asString(id)).join(`,`); - if (this.dialect === "standard") { - return this.asString(`farm_fingerprint(concat(${columnsAsStrings}))`); - } - if (this.dialect === "mssql") { - return this.asString(`hashbytes("md5", (concat(${columnsAsStrings})))`); - } - return this.asString(`md5(concat(${columnsAsStrings}))`); - } - - // window function - - public windowFunction( - name: string, - value: string, - ignoreNulls: boolean = false, - windowSpecification?: { - partitionFields?: string[]; - orderFields?: string[]; - frameClause?: string; - } - ) { - const partitionFieldsAsString = windowSpecification.partitionFields - ? [...windowSpecification.partitionFields].join(`, `) - : ""; - const orderFieldsAsString = windowSpecification.orderFields - ? [...windowSpecification.orderFields].join(`, `) - : ""; - - if (this.dialect === "standard" || this.dialect === "mssql" || this.dialect === "snowflake") { - return `${name}(${value} ${ignoreNulls ? `ignore nulls` : ``}) over (${ - windowSpecification.partitionFields ? `partition by ${partitionFieldsAsString}` : `` - } ${windowSpecification.orderFields ? `order by ${orderFieldsAsString}` : ``} ${ - windowSpecification.frameClause ? windowSpecification.frameClause : `` - })`; - } - - // For some window functions in Redshift, a frame clause is always required - const requiresFrame = [ - "avg", - "count", - "first_value", - "last_value", - "max", - "min", - "nth_value", - "stddev_samp", - "stddev_pop", - "stddev", - "sum", - "variance", - "var_samp", - "var_pop" - ].includes(name.toLowerCase()); - - if (this.dialect === "redshift") { - return `${name}(${value} ${ignoreNulls ? `ignore nulls` : ``}) over (${ - windowSpecification.partitionFields ? `partition by ${partitionFieldsAsString}` : `` - } ${windowSpecification.orderFields ? `order by ${orderFieldsAsString}` : ``} ${ - windowSpecification.orderFields - ? windowSpecification.frameClause - ? windowSpecification.frameClause - : requiresFrame - ? `rows between unbounded preceding and unbounded following` - : `` - : `` - })`; - } - - if (this.dialect === "postgres") { - return `${name}(${value}) over (${ - windowSpecification.partitionFields ? `partition by ${partitionFieldsAsString}` : `` - } ${windowSpecification.orderFields || ignoreNulls ? `order by` : ``} ${ - ignoreNulls ? `case when ${value} is not null then 0 else 1 end asc` : `` - } ${orderFieldsAsString && ignoreNulls ? `,` : ``} ${orderFieldsAsString} ${ - windowSpecification.orderFields - ? windowSpecification.frameClause - ? windowSpecification.frameClause - : requiresFrame - ? `rows between unbounded preceding and unbounded following` - : `` - : `` - })`; - } - } - - // String aggregation - - public stringAgg(field: string, delimiter = ",") { - if (this.dialect === "snowflake" || this.dialect === "redshift") { - return `listagg(${field}, '${delimiter}')`; - } - return `string_agg(${field}, '${delimiter}')`; - } - - // Convenience methods for builders. - public json(data: S[]) { - return new JSONBuilder(this, data); - } - - public join(joins: J) { - return new JoinBuilder(joins); - } - - public with(withs: IWiths) { - return new WithBuilder(withs); - } - - public union(...selects: Array>) { - return new UnionBuilder(selects); - } - - public aggregate(from: ISelectOrBuilder) { - return new AggregateBuilder(from); - } - - public from(from: ISelectOrBuilder) { - return new FromBuilder(from); - } -} diff --git a/sql/integration.spec.ts b/sql/integration.spec.ts deleted file mode 100644 index 120103c62..000000000 --- a/sql/integration.spec.ts +++ /dev/null @@ -1,572 +0,0 @@ -import { expect } from "chai"; - -import * as dfapi from "df/api"; -import * as dbadapters from "df/api/dbadapters"; -import { Sql } from "df/sql"; -import { build, ISelectOrBuilder, ISelectSchema } from "df/sql/builders/select"; -import { suite, test } from "df/testing"; - -suite("builders", { parallel: true }, ({ before, after }) => { - let bigquery: dbadapters.IDbAdapter; - let snowflake: dbadapters.IDbAdapter; - let redshift: dbadapters.IDbAdapter; - - before("create bigquery", async () => { - bigquery = await dbadapters.create( - { ...dfapi.credentials.read("bigquery", "test_credentials/bigquery.json"), location: "EU" }, - "bigquery" - ); - }); - - after("close bigquery", () => bigquery.close()); - - before("create snowflake", async () => { - snowflake = await dbadapters.create( - dfapi.credentials.read("snowflake", "test_credentials/snowflake.json"), - "snowflake" - ); - }); - - after("close snowflake", () => snowflake.close()); - - before("create redshift", async () => { - redshift = await dbadapters.create( - dfapi.credentials.read("redshift", "test_credentials/redshift.json"), - "redshift" - ); - }); - - after("close redshift", () => redshift.close()); - - for (const { name, dbadapter, sql } of [ - { name: "bigquery", dbadapter: () => bigquery, sql: new Sql("standard") }, - { name: "snowflake", dbadapter: () => snowflake, sql: new Sql("snowflake") } - ]) { - suite(name, { parallel: true }, () => { - const execute = async (select: ISelectOrBuilder) => { - const query = build(select).query; - try { - const rows = (await dbadapter().execute(query)).rows as S[]; - // Snowflake upper cases column names. Turn them back to lowercase for easier testing. - return rows.map(row => - Object.keys(row).reduce( - (acc, key) => ({ ...acc, [String(key).toLowerCase()]: row[key] }), - {} - ) - ); - } catch (e) { - throw new Error(`Error during query: ${e}\n${query}`); - } - }; - - test("timestamps", async () => { - const rows = [ - { - millis: 1591786375000, - truncated_millis: 1591747200000 - } - ]; - const query = sql.from(sql.json(rows)).select({ - millis: sql.timestamps.toMillis(sql.asTimestamp(sql.timestamps.fromMillis("millis"))), - truncated_millis: sql.timestamps.toMillis( - sql.timestamps.truncate(sql.asTimestamp(sql.timestamps.fromMillis("millis")), "day") - ) - }); - - const result = await execute(query); - expect(result).deep.equals(rows); - }); - - test("conditionals", async () => { - const rows = [ - { - v: "a" - }, - { - v: "b" - } - ]; - const query = sql.from(sql.json(rows)).select({ - ifac: sql.conditional(sql.in("v", [sql.literal("a")]), sql.literal("c"), "v"), - ifbd: sql.conditional(sql.equals("v", sql.literal("b")), sql.literal("d"), "v") - }); - - const result = await execute(query); - expect(result).deep.equals([ - { - ifac: "c", - ifbd: "a" - }, - { - ifac: "b", - ifbd: "d" - } - ]); - }); - - test("safe divide", async () => { - const rows = [ - { - a: 4, - b: 2 - }, - { - a: 4, - b: 0 - } - ]; - const query = sql.from(sql.json(rows)).select({ - v: sql.safeDivide("a", "b") - }); - - const result = await execute(query); - expect(result).deep.equals([ - { - v: 2 - }, - { - v: null - } - ]); - }); - - test("safe divide", async () => { - const rows = [ - { - a: 4, - b: 2 - }, - { - a: 4, - b: 0 - } - ]; - const query = sql.from(sql.json(rows)).select({ - v: sql.safeDivide("a", "b") - }); - - const result = await execute(query); - expect(result).deep.equals([ - { - v: 2 - }, - { - v: null - } - ]); - }); - - test("as string", async () => { - const rows = [ - { - a: 1, - b: "b" - } - ]; - const query = sql.from(sql.json(rows)).select({ - a: sql.asString("a"), - b: sql.asString("b") - }); - - const result = await execute(query); - expect(result).deep.equals([ - { - a: "1", - b: "b" - } - ]); - }); - - test("surrogate key", async () => { - const rows = [ - { - a: 1, - b: "b" - }, - { - a: 2, - b: "c" - } - ]; - const query = sql.from(sql.json(rows)).select({ - key: sql.surrogateKey(["a", "b"]) - }); - - const result: any = await execute(query); - expect(result.length).equals(2); - expect(result[0].key).not.equals(result[1].key); - }); - - test("window function", async () => { - const rows = [ - { - key: 1, - sort: 1, - value: 1, - partition_field: "a" - }, - { - key: 2, - sort: 2, - value: 1, - partition_field: "a" - }, - { - key: 3, - sort: 3, - value: null, - partition_field: "b" - }, - { - key: 4, - sort: 4, - value: 1, - partition_field: "b" - } - ]; - const query = sql.from(sql.json(rows)).select({ - lag: sql.windowFunction("lag", "value", false, { orderFields: ["key"] }), - max: sql.windowFunction("max", "value", false, { partitionFields: ["partition_field"] }), - first_value: sql.windowFunction("first_value", "value", true, { - partitionFields: ["partition_field"], - orderFields: ["sort"] - }) - }); - - const result: any = await execute(query); - expect(result.length).equals(4); - expect(result[1].lag).equals(1); - expect(result[0].max).equals(1); - expect(result[0].first_value).equals(1); - }); - // skipping this test for redshift as the function is only supported on user-defined-tables - // i.e. not when just querying a select statement with no table - if (name !== "redshift") { - test("string agg", async () => { - const rows = [ - { - a: "foo" - }, - { - a: "bar" - } - ]; - const query = sql.from(sql.json(rows)).select({ - agg: sql.stringAgg("a"), - agg_hyphen: sql.stringAgg("a", "-") - }); - - const result: any = await execute(query); - expect(result.length).equals(1); - expect(result[0]).deep.equals({ - agg: "foo,bar", - agg_hyphen: "foo-bar" - }); - }); - } - - test("timestamp diff", async () => { - const rows = [ - { - millis: 1604575623426, - previous_day: 1604489223426, - previous_hour: 1604572023426, - previous_minute: 1604575563426, - previous_second: 1604575622426, - previous_millisecond: 1604575623425 - } - ]; - const query = sql.from(sql.json(rows)).select({ - day: sql.timestamps.diff( - "day", - sql.timestamps.fromMillis("previous_day"), - sql.timestamps.fromMillis("millis") - ), - hour: sql.timestamps.diff( - "hour", - sql.timestamps.fromMillis("previous_hour"), - sql.timestamps.fromMillis("millis") - ), - minute: sql.timestamps.diff( - "minute", - sql.timestamps.fromMillis("previous_minute"), - sql.timestamps.fromMillis("millis") - ), - second: sql.timestamps.diff( - "second", - sql.timestamps.fromMillis("previous_second"), - sql.timestamps.fromMillis("millis") - ), - millisecond: sql.timestamps.diff( - "millisecond", - sql.timestamps.fromMillis("previous_millisecond"), - sql.timestamps.fromMillis("millis") - ) - }); - - const result: any = await execute(query); - - expect(result[0]).deep.equals({ - day: 1, - hour: 1, - minute: 1, - second: 1, - millisecond: 1 - }); - }); - - test("timestamp add", async () => { - const rows = [ - { - previous_day: 1604489223000, - previous_hour: 1604572023000, - previous_minute: 1604575563000, - previous_second: 1604575622000 - } - ]; - const query = sql.from(sql.json(rows)).select({ - day: sql.timestamps.toMillis( - sql.timestamps.add(sql.timestamps.fromMillis("previous_day"), 1, "day") - ), - hour: sql.timestamps.toMillis( - sql.timestamps.add(sql.timestamps.fromMillis("previous_hour"), 1, "hour") - ), - minute: sql.timestamps.toMillis( - sql.timestamps.add(sql.timestamps.fromMillis("previous_minute"), 1, "minute") - ), - second: sql.timestamps.toMillis( - sql.timestamps.add(sql.timestamps.fromMillis("previous_second"), 1, "second") - ) - }); - - const result: any = await execute(query); - - expect(result[0]).deep.equals({ - day: 1604575623000, - hour: 1604575623000, - minute: 1604575623000, - second: 1604575623000 - }); - }); - - test("json", async () => { - const rows = [ - { - d: "1", - m: 1 - }, - { - d: "2", - m: 2 - } - ]; - const selectJson = sql.json(rows); - const result = await execute(selectJson); - expect(result).deep.equals(rows); - }); - - test("aggregate", async () => { - const source = sql.json([ - { - d: "1", - m: 1 - }, - { - d: "2", - m: 2 - }, - { - d: "2", - m: 2 - } - ]); - - const select = sql - .aggregate(source) - .dimensions({ - d: "d" - }) - .metrics({ - m: sql.sum("m") - }) - .ordering({ - expression: "m", - descending: true - }); - const result = await execute(select); - expect(result).deep.equals([ - { - d: "2", - m: 4 - }, - { - d: "1", - m: 1 - } - ]); - }); - - test("union", async () => { - const union = sql.union( - sql.json([ - { - v: "a" - } - ]), - sql.json([ - { - v: "b" - } - ]) - ); - const result = await execute(union); - expect(result).deep.equals([ - { - v: "a" - }, - { - v: "b" - } - ]); - }); - - test("join", async () => { - const sourceA = sql.json([ - { - v1: "a" - } - ]); - - const sourceB = sql.json([ - { - v2: "a" - } - ]); - - const join = sql.from( - sql.join({ - a: { - select: sourceA, - type: "base" - }, - b: { - select: sourceB, - type: "left", - on: ["v1", "v2"] - } - }) - ); - - const result = await execute(join); - expect(result).deep.equals([ - { - v1: "a", - v2: "a" - } - ]); - }); - - test("with", async () => { - const source = sql.json([ - { - v: "a" - } - ]); - - const select = sql - .with({ - renamed: source - }) - .select(sql.from("renamed").select({ v: "v" })); - const result = await execute(select); - expect(result).deep.equals([ - { - v: "a" - } - ]); - }); - - test("combination", async () => { - const source = sql.json([ - { - d1: "a", - d2: "c", - m1: 1, - m2: 2 - }, - { - d1: "b", - d2: "c", - m1: 2, - m2: 2 - } - ]); - - const aggregate = sql - .with({ - filtered: sql.from(source).where(sql.equals("d2", sql.literal("c"))), - top_values: sql - .aggregate("filtered") - .dimensions({ - d1t: "d1" - }) - .metrics({ - m1t: sql.sum("m1") - }) - .ordering({ - expression: "m1t", - descending: true - }) - .limit(1) - }) - .select( - sql.union( - sql - .aggregate( - sql.join({ - top_values: { - select: "top_values", - type: "base" - }, - vals: { - select: "filtered", - type: "right", - on: ["d1t", "d1"] - } - }) - ) - .dimensions({ - d1: "d1t" - }) - .metrics({ - m1: sql.sum("m1") - }), - sql - .aggregate("filtered") - .dimensions({ - d1: sql.literal("d") - }) - .metrics({ - m1: sql.sum("m1") - }) - ) - ); - const result = await execute(aggregate); - expect(result).deep.members([ - { - d1: "b", - m1: 2 - }, - { - d1: null, - m1: 1 - }, - { - d1: "d", - m1: 3 - } - ]); - }); - }); - } -}); diff --git a/sql/timestamps.ts b/sql/timestamps.ts deleted file mode 100644 index 7e6a86caa..000000000 --- a/sql/timestamps.ts +++ /dev/null @@ -1,98 +0,0 @@ -import { ISqlDialect } from "df/sql"; - -export type DurationUnit = - | "millisecond" - | "second" - | "minute" - | "hour" - | "day" - | "week" - | "month" - | "quarter" - | "year"; - -export class Timestamps { - constructor(private readonly dialect: ISqlDialect) {} - - public fromMillis(timestampMillis: string) { - if (this.dialect === "snowflake") { - return `to_timestamp(${timestampMillis}, 3)`; - } - if (this.dialect === "postgres" || this.dialect === "redshift") { - return `timestamp 'epoch' + (${timestampMillis}) * interval '0.001 second'`; - } - return `timestamp_millis(${timestampMillis.toString()})`; - } - - public truncate(timestamp: string, timestampUnit: DurationUnit) { - if (this.dialect === "snowflake") { - return `date_trunc(${timestampUnit}, ${timestamp})`; - } - if (this.dialect === "postgres" || this.dialect === "redshift") { - return `date_trunc('${timestampUnit}', ${timestamp})`; - } - return `timestamp_trunc(${timestamp}, ${timestampUnit})`; - } - - public toMillis(timestamp: string) { - if (this.dialect === "snowflake") { - return `date_part(epoch_milliseconds, ${timestamp})`; - } - if (this.dialect === "postgres" || this.dialect === "redshift") { - return `extract('epoch' from ${timestamp})::bigint * 1000`; - } - return `unix_millis(${timestamp})`; - } - - public currentUTC() { - if (this.dialect === "postgres" || this.dialect === "redshift") { - return "current_timestamp::timestamp"; - } - if (this.dialect === "snowflake") { - return "convert_timezone('UTC', current_timestamp())::timestamp"; - } - if (this.dialect === "mssql") { - return "CURRENT_TIMESTAMP"; - } - return "current_timestamp()"; - } - - public diff(datePart: DurationUnit, start: string, end: string) { - if (this.dialect === "standard") { - return `timestamp_diff(${end}, ${start}, ${datePart})`; - } - if (this.dialect === "snowflake" || this.dialect === "mssql" || this.dialect === "redshift") { - return `datediff(${datePart}, ${start}, ${end})`; - } - if (this.dialect === "postgres") { - if (datePart.toLowerCase() === "day") { - return `date_part('day', ${end} - ${start})`; - } - if (datePart.toLowerCase() === "hour") { - return `24 * date_part('day', ${end} - ${start}) + date_part('hour', ${end} - ${start})`; - } - if (datePart.toLowerCase() === "minute") { - return `24 * date_part('day', ${end} - ${start}) + 60 * date_part('hour', ${end} - ${start}) + date_part('minute', ${end} - ${start})`; - } - if (datePart.toLowerCase() === "second") { - return `24 * date_part('day', ${end} - ${start}) + 60 * date_part('hour', ${end} - ${start}) + 60 * date_part('minute', ${end} - ${start}) + date_part('second', ${end} - ${start})`; - } - if (datePart.toLowerCase() === "millisecond") { - return `24 * date_part('day', ${end} - ${start}) + 60 * date_part('hour', ${end} - ${start}) + 60 * date_part('minute', ${end} - ${start}) + 1000 * date_part('second', ${end} - ${start}) + date_part('millisecond', ${end} - ${start})`; - } - } - } - - public add(timestamp: string, units: number, datePart: DurationUnit) { - if (this.dialect === "standard") { - return `timestamp_add(${timestamp}, interval ${units} ${datePart})`; - } - if (this.dialect === "postgres") { - return `${timestamp} + interval '1 ${datePart}' * ${units}`; - } - if (this.dialect === "snowflake") { - return `timestampadd(${datePart}, ${units}, ${timestamp})`; - } - return `dateadd(${datePart}, ${units}, ${timestamp})`; - } -} diff --git a/tests/api/BUILD b/tests/api/BUILD index 43933900e..3a9fb703e 100644 --- a/tests/api/BUILD +++ b/tests/api/BUILD @@ -6,8 +6,6 @@ ts_test_suite( data = [ "//examples/backwards_compatibility:files", "//examples/backwards_compatibility:node_modules", - "//examples/common_v1:files", - "//examples/common_v1:node_modules", "//examples/common_v2:files", "//examples/common_v2:node_modules", "//examples/formatter:files", diff --git a/tests/api/examples.spec.ts b/tests/api/examples.spec.ts index 322a24dcf..1a6578814 100644 --- a/tests/api/examples.spec.ts +++ b/tests/api/examples.spec.ts @@ -753,370 +753,6 @@ suite("examples", () => { } }); - suite("common_v1", async () => { - test("bigquery compiles", async () => { - const graph = await compile({ - projectDir: path.resolve("examples/common_v1"), - projectConfigOverride: { warehouse: "bigquery", defaultDatabase: "tada-analytics" } - }); - const tableNames = graph.tables.map((t: dataform.ITable) => targetAsReadableString(t.target)); - - expect(graph.graphErrors).to.eql(dataform.GraphErrors.create()); - - // Check JS blocks get processed. - expect(tableNames).includes("tada-analytics.df_integration_test.example_js_blocks"); - const exampleJsBlocks = graph.tables.filter( - (t: dataform.ITable) => - targetAsReadableString(t.target) === - "tada-analytics.df_integration_test.example_js_blocks" - )[0]; - expect(exampleJsBlocks.type).equals("table"); - expect(exampleJsBlocks.query).equals("select 1 as foo"); - - // Check we can import and use an external package. - expect(tableNames).includes("tada-analytics.df_integration_test.example_incremental"); - const exampleIncremental = graph.tables.filter( - (t: dataform.ITable) => - targetAsReadableString(t.target) === - "tada-analytics.df_integration_test.example_incremental" - )[0]; - expect(exampleIncremental.query).equals("select current_timestamp() as ts"); - expect(exampleIncremental.where.trim()).equals( - "ts > (select max(ts) from `tada-analytics.df_integration_test.example_incremental`) or (select max(ts) from `tada-analytics.df_integration_test.example_incremental`) is null" - ); - - // Check tables defined in includes are not included. - expect(tableNames).not.includes("example_ignore"); - - // Check SQL files with raw back-ticks get escaped. - expect(tableNames).includes("tada-analytics.df_integration_test.example_backticks"); - const exampleBackticks = graph.tables.filter( - (t: dataform.ITable) => - targetAsReadableString(t.target) === - "tada-analytics.df_integration_test.example_backticks" - )[0]; - expect(cleanSql(exampleBackticks.query)).equals( - "select * from `tada-analytics.df_integration_test.sample_data`" - ); - - // Check deferred calls to table resolve to the correct definitions file. - expect(tableNames).includes("tada-analytics.df_integration_test.example_deferred"); - const exampleDeferred = graph.tables.filter( - (t: dataform.ITable) => - targetAsReadableString(t.target) === "tada-analytics.df_integration_test.example_deferred" - )[0]; - expect(exampleDeferred.fileName).includes("definitions/example_deferred.js"); - - // Check inline tables - expect(tableNames).includes("tada-analytics.df_integration_test.example_inline"); - const exampleInline = graph.tables.filter( - (t: dataform.ITable) => - targetAsReadableString(t.target) === "tada-analytics.df_integration_test.example_inline" - )[0]; - expect(exampleInline.type).equals("inline"); - expect(exampleInline.query).equals( - "\nselect * from `tada-analytics.df_integration_test.sample_data`" - ); - expect(exampleInline.dependencyTargets).eql([ - dataform.Target.create({ - database: "tada-analytics", - schema: "df_integration_test", - name: "sample_data" - }) - ]); - - expect(tableNames).includes("tada-analytics.df_integration_test.example_using_inline"); - const exampleUsingInline = graph.tables.filter( - (t: dataform.ITable) => - targetAsReadableString(t.target) === - "tada-analytics.df_integration_test.example_using_inline" - )[0]; - expect(exampleUsingInline.type).equals("table"); - expect(exampleUsingInline.query).equals( - "\nselect * from (\nselect * from `tada-analytics.df_integration_test.sample_data`)\nwhere true" - ); - expect(exampleUsingInline.dependencyTargets).eql([ - dataform.Target.create({ - database: "tada-analytics", - schema: "df_integration_test", - name: "sample_data" - }) - ]); - - // Check view - expect(tableNames).includes("tada-analytics.df_integration_test.example_view"); - const exampleView = graph.tables.filter( - (t: dataform.ITable) => - targetAsReadableString(t.target) === "tada-analytics.df_integration_test.example_view" - )[0]; - expect(exampleView.type).equals("view"); - expect(exampleView.query).equals( - "\nselect * from `tada-analytics.df_integration_test.sample_data`" - ); - expect(exampleView.dependencyTargets).eql([ - dataform.Target.create({ - database: "tada-analytics", - schema: "df_integration_test", - name: "sample_data" - }) - ]); - - // Check table - expect(tableNames).includes("tada-analytics.df_integration_test.example_table"); - const exampleTable = graph.tables.filter( - (t: dataform.ITable) => - targetAsReadableString(t.target) === "tada-analytics.df_integration_test.example_table" - )[0]; - expect(exampleTable.type).equals("table"); - expect(exampleTable.query).equals( - "\nselect * from `tada-analytics.df_integration_test.sample_data`" - ); - expect(exampleTable.dependencyTargets).eql([ - dataform.Target.create({ - database: "tada-analytics", - schema: "df_integration_test", - name: "sample_data" - }) - ]); - - // Check sample data - expect(tableNames).includes("tada-analytics.df_integration_test.sample_data"); - const exampleSampleData = graph.tables.filter( - (t: dataform.ITable) => - targetAsReadableString(t.target) === "tada-analytics.df_integration_test.sample_data" - )[0]; - expect(exampleSampleData.type).equals("view"); - expect(exampleSampleData.query).equals( - "select 1 as sample union all\nselect 2 as sample union all\nselect 3 as sample" - ); - expect(exampleSampleData.dependencyTargets).eql([]); - }); - - test("bigquery compiles with schema override", async () => { - const graph = await compile({ - projectDir: path.resolve("examples/common_v1"), - projectConfigOverride: { - warehouse: "redshift", - schemaSuffix: "suffix" - } - }); - expect(graph.projectConfig.schemaSuffix).to.equal("suffix"); - graph.tables.forEach(table => - expect(table.target.schema).to.match( - /^(df_integration_test_suffix|override_schema_suffix)$/ - ) - ); - }); - - test("bigquery compiles with database override", async () => { - const graph = await compile({ - projectDir: path.resolve("examples/common_v2"), - projectConfigOverride: { - warehouse: "bigquery", - defaultDatabase: "overridden-database" - } - }); - expect(graph.projectConfig.defaultDatabase).to.equal("overridden-database"); - const exampleTable = graph.tables.find( - table => - targetAsReadableString(table.target) === - "overridden-database.df_integration_test.example_table" - ); - expect(exampleTable.target.database).equals("overridden-database"); - }); - - test("redshift compiles", () => { - return compile({ - projectDir: "examples/common_v1", - projectConfigOverride: { warehouse: "redshift" } - }).then(graph => { - const tableNames = graph.tables.map((t: dataform.ITable) => - targetAsReadableString(t.target) - ); - - // Check we can import and use an external package. - expect(tableNames).includes("df_integration_test.example_incremental"); - const exampleIncremental = graph.tables.filter( - (t: dataform.ITable) => - targetAsReadableString(t.target) === "df_integration_test.example_incremental" - )[0]; - expect(exampleIncremental.query).equals("select current_timestamp::timestamp as ts"); - - // Check inline tables - expect(tableNames).includes("df_integration_test.example_inline"); - const exampleInline = graph.tables.filter( - (t: dataform.ITable) => - targetAsReadableString(t.target) === "df_integration_test.example_inline" - )[0]; - expect(exampleInline.type).equals("inline"); - expect(exampleInline.query).equals('\nselect * from "df_integration_test"."sample_data"'); - expect(exampleInline.dependencyTargets).eql([ - dataform.Target.create({ - schema: "df_integration_test", - name: "sample_data" - }) - ]); - - expect(tableNames).includes("df_integration_test.example_using_inline"); - const exampleUsingInline = graph.tables.filter( - (t: dataform.ITable) => - targetAsReadableString(t.target) === "df_integration_test.example_using_inline" - )[0]; - expect(exampleUsingInline.type).equals("table"); - expect(exampleUsingInline.query).equals( - '\nselect * from (\nselect * from "df_integration_test"."sample_data")\nwhere true' - ); - expect(exampleUsingInline.dependencyTargets).eql([ - dataform.Target.create({ - schema: "df_integration_test", - name: "sample_data" - }) - ]); - }); - }); - - test("snowflake compiles", async () => { - const graph = await compile({ - projectDir: "examples/common_v1", - projectConfigOverride: { warehouse: "snowflake" } - }).catch(error => error); - expect(graph).to.not.be.an.instanceof(Error); - expect(graph.graphErrors).deep.equals(dataform.GraphErrors.create({})); - - const mNames = graph.tables.map((t: dataform.ITable) => targetAsReadableString(t.target)); - - expect(mNames).includes("DF_INTEGRATION_TEST.EXAMPLE_INCREMENTAL"); - const mIncremental = graph.tables.filter( - (t: dataform.ITable) => - targetAsReadableString(t.target) === "DF_INTEGRATION_TEST.EXAMPLE_INCREMENTAL" - )[0]; - expect(mIncremental.type).equals("incremental"); - expect(mIncremental.query).equals( - "select convert_timezone('UTC', current_timestamp())::timestamp as ts" - ); - - expect(mIncremental.dependencyTargets).eql([]); - - expect(mNames).includes("DF_INTEGRATION_TEST.EXAMPLE_TABLE"); - const mTable = graph.tables.filter( - (t: dataform.ITable) => - targetAsReadableString(t.target) === "DF_INTEGRATION_TEST.EXAMPLE_TABLE" - )[0]; - expect(mTable.type).equals("table"); - expect(mTable.query).equals('\nselect * from "DF_INTEGRATION_TEST"."SAMPLE_DATA"'); - expect(mTable.dependencyTargets).eql([ - dataform.Target.create({ - schema: "DF_INTEGRATION_TEST", - name: "SAMPLE_DATA" - }) - ]); - - expect(mNames).includes("DF_INTEGRATION_TEST.EXAMPLE_VIEW"); - const mView = graph.tables.filter( - (t: dataform.ITable) => - targetAsReadableString(t.target) === "DF_INTEGRATION_TEST.EXAMPLE_VIEW" - )[0]; - expect(mView.type).equals("view"); - expect(mView.query).equals('\nselect * from "DF_INTEGRATION_TEST"."SAMPLE_DATA"'); - expect(mView.dependencyTargets).eql([ - dataform.Target.create({ - schema: "DF_INTEGRATION_TEST", - name: "SAMPLE_DATA" - }) - ]); - - expect(mNames).includes("DF_INTEGRATION_TEST.SAMPLE_DATA"); - const mSampleData = graph.tables.filter( - (t: dataform.ITable) => - targetAsReadableString(t.target) === "DF_INTEGRATION_TEST.SAMPLE_DATA" - )[0]; - expect(mSampleData.type).equals("view"); - expect(mSampleData.query).equals( - "select 1 as sample union all\nselect 2 as sample union all\nselect 3 as sample" - ); - expect(mSampleData.dependencyTargets).eql([]); - - // Check inline tables - expect(mNames).includes("DF_INTEGRATION_TEST.EXAMPLE_INLINE"); - const exampleInline = graph.tables.filter( - (t: dataform.ITable) => - targetAsReadableString(t.target) === "DF_INTEGRATION_TEST.EXAMPLE_INLINE" - )[0]; - expect(exampleInline.type).equals("inline"); - expect(exampleInline.query).equals('\nselect * from "DF_INTEGRATION_TEST"."SAMPLE_DATA"'); - expect(exampleInline.dependencyTargets).eql([ - dataform.Target.create({ - schema: "DF_INTEGRATION_TEST", - name: "SAMPLE_DATA" - }) - ]); - - expect(mNames).includes("DF_INTEGRATION_TEST.EXAMPLE_USING_INLINE"); - const exampleUsingInline = graph.tables.filter( - (t: dataform.ITable) => - targetAsReadableString(t.target) === "DF_INTEGRATION_TEST.EXAMPLE_USING_INLINE" - )[0]; - expect(exampleUsingInline.type).equals("table"); - expect(exampleUsingInline.query).equals( - '\nselect * from (\nselect * from "DF_INTEGRATION_TEST"."SAMPLE_DATA")\nwhere true' - ); - expect(exampleUsingInline.dependencyTargets).eql([ - dataform.Target.create({ - schema: "DF_INTEGRATION_TEST", - name: "SAMPLE_DATA" - }) - ]); - - const aNames = graph.assertions.map((a: dataform.IAssertion) => - targetAsReadableString(a.target) - ); - - expect(aNames).includes("DF_INTEGRATION_TEST_ASSERTIONS.SAMPLE_DATA_ASSERTION"); - const assertion = graph.assertions.filter( - (a: dataform.IAssertion) => - targetAsReadableString(a.target) === - "DF_INTEGRATION_TEST_ASSERTIONS.SAMPLE_DATA_ASSERTION" - )[0]; - expect(assertion.query).equals( - 'select * from "DF_INTEGRATION_TEST"."SAMPLE_DATA" where sample > 3' - ); - expect(assertion.dependencyTargets).eql([ - dataform.Target.create({ - schema: "DF_INTEGRATION_TEST", - name: "EXAMPLE_BACKTICKS" - }), - dataform.Target.create({ - schema: "DF_INTEGRATION_TEST", - name: "EXAMPLE_DEFERRED" - }), - dataform.Target.create({ - schema: "DF_INTEGRATION_TEST", - name: "EXAMPLE_INCREMENTAL" - }), - dataform.Target.create({ - schema: "DF_INTEGRATION_TEST", - name: "EXAMPLE_JS_BLOCKS" - }), - dataform.Target.create({ - schema: "DF_INTEGRATION_TEST", - name: "EXAMPLE_TABLE" - }), - dataform.Target.create({ - schema: "DF_INTEGRATION_TEST", - name: "EXAMPLE_USING_INLINE" - }), - dataform.Target.create({ - schema: "DF_INTEGRATION_TEST", - name: "EXAMPLE_VIEW" - }), - dataform.Target.create({ - schema: "DF_INTEGRATION_TEST", - name: "SAMPLE_DATA" - }) - ]); - }); - }); - test("backwards_compatibility", async () => { const graph = await compile({ projectDir: "examples/backwards_compatibility" }); From f5a75c3f089c77043e8d32ef1a25f2562bcc750a Mon Sep 17 00:00:00 2001 From: Elias Kassell Date: Tue, 3 Jan 2023 14:34:05 +0000 Subject: [PATCH 2/5] Remove SQL package from common_v2 example --- examples/common_v2/BUILD | 1 - examples/common_v2/definitions/example_incremental.sqlx | 2 +- examples/common_v2/definitions/example_is_incremental.sqlx | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/examples/common_v2/BUILD b/examples/common_v2/BUILD index 3672e2d24..e1fafe2e3 100644 --- a/examples/common_v2/BUILD +++ b/examples/common_v2/BUILD @@ -13,6 +13,5 @@ node_modules( name = "node_modules", deps = [ "//packages/@dataform/core:package_tar", - "//packages/@dataform/sql:package_tar", ], ) diff --git a/examples/common_v2/definitions/example_incremental.sqlx b/examples/common_v2/definitions/example_incremental.sqlx index 01fdb2eba..602cd2529 100644 --- a/examples/common_v2/definitions/example_incremental.sqlx +++ b/examples/common_v2/definitions/example_incremental.sqlx @@ -2,7 +2,7 @@ config { type: "incremental", protected: true, hermetic: false } js { const sql = require("@dataform/sql"); } -select ${sql().timestamps.currentUTC()} as ts +select CURRENT_TIMESTAMP() as ts incremental_where { ts > (select max(ts) from ${self()}) or (select max(ts) from ${self()}) is null diff --git a/examples/common_v2/definitions/example_is_incremental.sqlx b/examples/common_v2/definitions/example_is_incremental.sqlx index 8d8378a51..4daaccaab 100644 --- a/examples/common_v2/definitions/example_is_incremental.sqlx +++ b/examples/common_v2/definitions/example_is_incremental.sqlx @@ -3,7 +3,7 @@ config { type: "incremental", protected: true, hermetic: false } js { const sql = require("@dataform/sql"); } select * from ( - select ${sql().timestamps.currentUTC()} as ts + select CURRENT_TIMESTAMP() as ts ) ${when(incremental(), `where ts > (select max(ts) from ${self()}) or (select max(ts) from ${self()}) is null`)} From 19990caf7cadb8920e766e645e87083de5710051 Mon Sep 17 00:00:00 2001 From: Elias Kassell Date: Tue, 3 Jan 2023 14:40:22 +0000 Subject: [PATCH 3/5] Remove build imports from tests of @dataform/sql that weren't doing anything --- examples/formatter/BUILD | 1 - examples/never_finishes_compiling/BUILD | 1 - tests/integration/bigquery_project/BUILD | 1 - tests/integration/postgres_project/BUILD | 1 - tests/integration/presto_project/BUILD | 1 - tests/integration/redshift_project/BUILD | 1 - tests/integration/snowflake_project/BUILD | 1 - tests/integration/sqldatawarehouse_project/BUILD | 1 - 8 files changed, 8 deletions(-) diff --git a/examples/formatter/BUILD b/examples/formatter/BUILD index 3672e2d24..e1fafe2e3 100644 --- a/examples/formatter/BUILD +++ b/examples/formatter/BUILD @@ -13,6 +13,5 @@ node_modules( name = "node_modules", deps = [ "//packages/@dataform/core:package_tar", - "//packages/@dataform/sql:package_tar", ], ) diff --git a/examples/never_finishes_compiling/BUILD b/examples/never_finishes_compiling/BUILD index 3672e2d24..e1fafe2e3 100644 --- a/examples/never_finishes_compiling/BUILD +++ b/examples/never_finishes_compiling/BUILD @@ -13,6 +13,5 @@ node_modules( name = "node_modules", deps = [ "//packages/@dataform/core:package_tar", - "//packages/@dataform/sql:package_tar", ], ) diff --git a/tests/integration/bigquery_project/BUILD b/tests/integration/bigquery_project/BUILD index f8ae395c4..af5ae241e 100644 --- a/tests/integration/bigquery_project/BUILD +++ b/tests/integration/bigquery_project/BUILD @@ -14,6 +14,5 @@ node_modules( deps = [ "//packages/@dataform/assertion_utils:package_tar", "//packages/@dataform/core:package_tar", - "//packages/@dataform/sql:package_tar", ], ) diff --git a/tests/integration/postgres_project/BUILD b/tests/integration/postgres_project/BUILD index f8ae395c4..af5ae241e 100644 --- a/tests/integration/postgres_project/BUILD +++ b/tests/integration/postgres_project/BUILD @@ -14,6 +14,5 @@ node_modules( deps = [ "//packages/@dataform/assertion_utils:package_tar", "//packages/@dataform/core:package_tar", - "//packages/@dataform/sql:package_tar", ], ) diff --git a/tests/integration/presto_project/BUILD b/tests/integration/presto_project/BUILD index f8ae395c4..af5ae241e 100644 --- a/tests/integration/presto_project/BUILD +++ b/tests/integration/presto_project/BUILD @@ -14,6 +14,5 @@ node_modules( deps = [ "//packages/@dataform/assertion_utils:package_tar", "//packages/@dataform/core:package_tar", - "//packages/@dataform/sql:package_tar", ], ) diff --git a/tests/integration/redshift_project/BUILD b/tests/integration/redshift_project/BUILD index f8ae395c4..af5ae241e 100644 --- a/tests/integration/redshift_project/BUILD +++ b/tests/integration/redshift_project/BUILD @@ -14,6 +14,5 @@ node_modules( deps = [ "//packages/@dataform/assertion_utils:package_tar", "//packages/@dataform/core:package_tar", - "//packages/@dataform/sql:package_tar", ], ) diff --git a/tests/integration/snowflake_project/BUILD b/tests/integration/snowflake_project/BUILD index f8ae395c4..af5ae241e 100644 --- a/tests/integration/snowflake_project/BUILD +++ b/tests/integration/snowflake_project/BUILD @@ -14,6 +14,5 @@ node_modules( deps = [ "//packages/@dataform/assertion_utils:package_tar", "//packages/@dataform/core:package_tar", - "//packages/@dataform/sql:package_tar", ], ) diff --git a/tests/integration/sqldatawarehouse_project/BUILD b/tests/integration/sqldatawarehouse_project/BUILD index f8ae395c4..af5ae241e 100644 --- a/tests/integration/sqldatawarehouse_project/BUILD +++ b/tests/integration/sqldatawarehouse_project/BUILD @@ -14,6 +14,5 @@ node_modules( deps = [ "//packages/@dataform/assertion_utils:package_tar", "//packages/@dataform/core:package_tar", - "//packages/@dataform/sql:package_tar", ], ) From 88b50055bb3640ed8e07713531733fb953273220 Mon Sep 17 00:00:00 2001 From: Elias Kassell Date: Tue, 3 Jan 2023 14:51:40 +0000 Subject: [PATCH 4/5] Inline timestamp definition in non BQ tests --- examples/common_v2/definitions/example_incremental.sqlx | 2 -- examples/common_v2/definitions/example_is_incremental.sqlx | 2 -- .../definitions/example_incremental.sqlx | 6 +----- 3 files changed, 1 insertion(+), 9 deletions(-) diff --git a/examples/common_v2/definitions/example_incremental.sqlx b/examples/common_v2/definitions/example_incremental.sqlx index 602cd2529..274d5c789 100644 --- a/examples/common_v2/definitions/example_incremental.sqlx +++ b/examples/common_v2/definitions/example_incremental.sqlx @@ -1,7 +1,5 @@ config { type: "incremental", protected: true, hermetic: false } -js { const sql = require("@dataform/sql"); } - select CURRENT_TIMESTAMP() as ts incremental_where { diff --git a/examples/common_v2/definitions/example_is_incremental.sqlx b/examples/common_v2/definitions/example_is_incremental.sqlx index 4daaccaab..ac0a87b5b 100644 --- a/examples/common_v2/definitions/example_is_incremental.sqlx +++ b/examples/common_v2/definitions/example_is_incremental.sqlx @@ -1,7 +1,5 @@ config { type: "incremental", protected: true, hermetic: false } -js { const sql = require("@dataform/sql"); } - select * from ( select CURRENT_TIMESTAMP() as ts ) diff --git a/tests/integration/sqldatawarehouse_project/definitions/example_incremental.sqlx b/tests/integration/sqldatawarehouse_project/definitions/example_incremental.sqlx index e265911c6..48314dad0 100644 --- a/tests/integration/sqldatawarehouse_project/definitions/example_incremental.sqlx +++ b/tests/integration/sqldatawarehouse_project/definitions/example_incremental.sqlx @@ -2,12 +2,8 @@ config { type: "incremental" } -js { - const sql = require("@dataform/sql"); -} - ${ when(incremental(), "select * from (") } -select ${sql("mssql").timestamps.currentUTC()} as ts +select CURRENT_TIMESTAMP as ts ${ when(incremental(), `) as subquery where ts > (select max(ts) from ${self()}) or (select max(ts) from ${self()}) is null`) } From 70e80596784c58c795071135f93c886a93e1553a Mon Sep 17 00:00:00 2001 From: Elias Kassell Date: Tue, 3 Jan 2023 15:16:01 +0000 Subject: [PATCH 5/5] Fix timestamp function case --- examples/common_v2/definitions/example_incremental.sqlx | 2 +- examples/common_v2/definitions/example_is_incremental.sqlx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/common_v2/definitions/example_incremental.sqlx b/examples/common_v2/definitions/example_incremental.sqlx index 274d5c789..aedba08a0 100644 --- a/examples/common_v2/definitions/example_incremental.sqlx +++ b/examples/common_v2/definitions/example_incremental.sqlx @@ -1,6 +1,6 @@ config { type: "incremental", protected: true, hermetic: false } -select CURRENT_TIMESTAMP() as ts +select current_timestamp() as ts incremental_where { ts > (select max(ts) from ${self()}) or (select max(ts) from ${self()}) is null diff --git a/examples/common_v2/definitions/example_is_incremental.sqlx b/examples/common_v2/definitions/example_is_incremental.sqlx index ac0a87b5b..a1320398d 100644 --- a/examples/common_v2/definitions/example_is_incremental.sqlx +++ b/examples/common_v2/definitions/example_is_incremental.sqlx @@ -1,7 +1,7 @@ config { type: "incremental", protected: true, hermetic: false } select * from ( - select CURRENT_TIMESTAMP() as ts + select current_timestamp() as ts ) ${when(incremental(), `where ts > (select max(ts) from ${self()}) or (select max(ts) from ${self()}) is null`)}