From ff5f687ec7354f3e2baf362858df9413246f2e5e Mon Sep 17 00:00:00 2001 From: Alexandre de Souza Date: Fri, 8 Jul 2022 14:04:48 -0300 Subject: [PATCH 01/13] Add support for aws_credentials with ReqAthena --- lib/assets/connection_cell/main.js | 10 +++-- lib/kino_db/connection_cell.ex | 57 ++++++++++++++++----------- test/kino_db/connection_cell_test.exs | 6 +-- 3 files changed, 45 insertions(+), 28 deletions(-) diff --git a/lib/assets/connection_cell/main.js b/lib/assets/connection_cell/main.js index 1b764d3..d3e8801 100644 --- a/lib/assets/connection_cell/main.js +++ b/lib/assets/connection_cell/main.js @@ -215,6 +215,10 @@ export function init(ctx, info) { type: Object, default: {} }, + hasAwsCredentials: { + type: Boolean, + default: false + } }, template: ` @@ -226,7 +230,7 @@ export function init(ctx, info) { v-model="fields.access_key_id" inputClass="input" :grow - :required + :required="!hasAwsCredentials" />
diff --git a/lib/kino_db/connection_cell.ex b/lib/kino_db/connection_cell.ex index bce4950..ec33261 100644 --- a/lib/kino_db/connection_cell.ex +++ b/lib/kino_db/connection_cell.ex @@ -32,8 +32,14 @@ defmodule KinoDB.ConnectionCell do "output_location" => attrs["output_location"] || "" } - {:ok, - assign(ctx, fields: fields, missing_dep: missing_dep(fields), help_box: help_box(fields))} + assigns = [ + fields: fields, + missing_dep: missing_dep(fields), + help_box: help_box(fields), + has_aws_credentials: Code.ensure_loaded?(:aws_credentials) + ] + + {:ok, assign(ctx, assigns)} end @impl true @@ -41,7 +47,8 @@ defmodule KinoDB.ConnectionCell do payload = %{ fields: ctx.assigns.fields, missing_dep: ctx.assigns.missing_dep, - help_box: ctx.assigns.help_box + help_box: ctx.assigns.help_box, + has_aws_credentials: ctx.assigns.has_aws_credentials } {:ok, payload, ctx} @@ -91,7 +98,7 @@ defmodule KinoDB.ConnectionCell do defp to_updates(_fields, field, value), do: %{field => value} - @default_keys ["type", "variable"] + @default_keys ["type", "variable", "_keys_"] @impl true def to_attrs(%{assigns: %{fields: fields}}) do @@ -110,7 +117,9 @@ defmodule KinoDB.ConnectionCell do ~w|database hostname port username password| end - Map.take(fields, @default_keys ++ connection_keys) + fields + |> Map.put("_keys_", connection_keys) + |> Map.take(@default_keys ++ connection_keys) end @impl true @@ -124,7 +133,9 @@ defmodule KinoDB.ConnectionCell do ~w|project_id| "athena" -> - ~w|access_key_id secret_access_key region output_location database| + if Code.ensure_loaded?(:aws_credentials), + do: ~w|output_location database|, + else: ~w|access_key_id secret_access_key region output_location database| type when type in ["postgres", "mysql"] -> ~w|hostname port| @@ -186,6 +197,24 @@ defmodule KinoDB.ConnectionCell do join_quoted([goth_opts_block, conn_block]) end + defp to_quoted(%{"type" => "athena"} = attrs) do + quote do + unquote(quoted_var(attrs["variable"])) = + Req.new(http_errors: :raise) + |> ReqAthena.attach( + access_key_id: unquote(attrs["access_key_id"]), + database: unquote(attrs["database"]), + output_location: unquote(attrs["output_location"]), + region: unquote(attrs["region"]), + secret_access_key: unquote(attrs["secret_access_key"]), + token: unquote(attrs["token"]), + workgroup: unquote(attrs["workgroup"]) + ) + + :ok + end + end + defp check_bigquery_credentials(attrs) do case attrs["credentials"] do %{"type" => "service_account"} -> @@ -217,22 +246,6 @@ defmodule KinoDB.ConnectionCell do end end - defp to_quoted(%{"type" => "athena"} = attrs) do - quote do - unquote(quoted_var(attrs["variable"])) = - Req.new(http_errors: :raise) - |> ReqAthena.attach( - access_key_id: unquote(attrs["access_key_id"]), - secret_access_key: unquote(attrs["secret_access_key"]), - region: unquote(attrs["region"]), - database: unquote(attrs["database"]), - output_location: unquote(attrs["output_location"]) - ) - - :ok - end - end - defp shared_options(attrs) do quote do [ diff --git a/test/kino_db/connection_cell_test.exs b/test/kino_db/connection_cell_test.exs index c36e184..8213f6a 100644 --- a/test/kino_db/connection_cell_test.exs +++ b/test/kino_db/connection_cell_test.exs @@ -181,10 +181,10 @@ defmodule KinoDB.ConnectionCellTest do Req.new(http_errors: :raise) |> ReqAthena.attach( access_key_id: "id", - secret_access_key: "secret", - region: "region", database: "default", - output_location: "s3://my-bucket" + output_location: "s3://my-bucket", + region: "region", + secret_access_key: "secret" ) :ok\ From 1af9d5afe3ffb32ef59eceec4ce39dad7ac708f6 Mon Sep 17 00:00:00 2001 From: Alexandre de Souza Date: Wed, 13 Jul 2022 14:45:33 -0300 Subject: [PATCH 02/13] Add conditional fields with new ReqAthena options --- lib/assets/connection_cell/main.js | 34 +++++++++++++++++++++++++-- lib/kino_db/connection_cell.ex | 28 +++++++++++++++++----- test/kino_db/connection_cell_test.exs | 22 ++++++++++++++++- 3 files changed, 75 insertions(+), 9 deletions(-) diff --git a/lib/assets/connection_cell/main.js b/lib/assets/connection_cell/main.js index d3e8801..21f57f4 100644 --- a/lib/assets/connection_cell/main.js +++ b/lib/assets/connection_cell/main.js @@ -221,6 +221,16 @@ export function init(ctx, info) { } }, + methods: { + areFieldsEmpty(currentField, otherField) { + if (currentField === "" && otherField === "") { + return true; + } + + return false; + }, + }, + template: `
+
+
+ +
` @@ -403,7 +432,7 @@ export function init(ctx, info) { - +
@@ -415,6 +444,7 @@ export function init(ctx, info) { fields: info.fields, missingDep: info.missing_dep, helpBox: info.help_box, + hasAwsCredentials: info.has_aws_credentials, availableDatabases: [ {label: "PostgreSQL", value: "postgres"}, {label: "MySQL", value: "mysql"}, diff --git a/lib/kino_db/connection_cell.ex b/lib/kino_db/connection_cell.ex index ec33261..8ccab51 100644 --- a/lib/kino_db/connection_cell.ex +++ b/lib/kino_db/connection_cell.ex @@ -28,7 +28,9 @@ defmodule KinoDB.ConnectionCell do "credentials" => attrs["credentials"] || %{}, "access_key_id" => attrs["access_key_id"] || "", "secret_access_key" => attrs["secret_access_key"] || "", + "token" => attrs["token"] || "", "region" => attrs["region"] || "", + "workgroup" => attrs["workgroup"] || "", "output_location" => attrs["output_location"] || "" } @@ -105,13 +107,14 @@ defmodule KinoDB.ConnectionCell do connection_keys = case fields["type"] do "sqlite" -> - ["database_path"] + ~w|database_path| "bigquery" -> ~w|project_id default_dataset_id credentials| "athena" -> - ~w|access_key_id secret_access_key region output_location database| + ~w|access_key_id secret_access_key token region + workgroup output_location database| type when type in ["postgres", "mysql"] -> ~w|database hostname port username password| @@ -127,21 +130,28 @@ defmodule KinoDB.ConnectionCell do required_keys = case attrs["type"] do "sqlite" -> - ["database_path"] + ~w|database_path| "bigquery" -> ~w|project_id| "athena" -> if Code.ensure_loaded?(:aws_credentials), - do: ~w|output_location database|, - else: ~w|access_key_id secret_access_key region output_location database| + do: ~w|database|, + else: ~w|access_key_id secret_access_key region database| type when type in ["postgres", "mysql"] -> ~w|hostname port| end - if required_fields_filled?(attrs, required_keys) do + conditional_keys = + case attrs["type"] do + "athena" -> ~w|workgroup output_location| + _ -> [] + end + + if required_fields_filled?(attrs, required_keys) and + conditional_fields_filled?(attrs, conditional_keys) do attrs |> to_quoted() |> Kino.SmartCell.quoted_to_string() else "" @@ -152,6 +162,12 @@ defmodule KinoDB.ConnectionCell do not Enum.any?(keys, fn key -> attrs[key] in [nil, ""] end) end + defp conditional_fields_filled?(_, []), do: true + + defp conditional_fields_filled?(attrs, keys) do + not Enum.all?(keys, fn key -> attrs[key] in [nil, ""] end) + end + defp to_quoted(%{"type" => "sqlite"} = attrs) do quote do opts = [database: unquote(attrs["database_path"])] diff --git a/test/kino_db/connection_cell_test.exs b/test/kino_db/connection_cell_test.exs index 8213f6a..1ed9427 100644 --- a/test/kino_db/connection_cell_test.exs +++ b/test/kino_db/connection_cell_test.exs @@ -184,13 +184,33 @@ defmodule KinoDB.ConnectionCellTest do database: "default", output_location: "s3://my-bucket", region: "region", - secret_access_key: "secret" + secret_access_key: "secret", + token: "", + workgroup: "" ) :ok\ """ end + test "doesn't restore source code with empty conditional fields" do + attrs = %{ + "variable" => "db", + "type" => "athena", + "access_key_id" => "id", + "secret_access_key" => "secret", + "region" => "region", + "database" => "default", + "token" => "token", + "workgroup" => "", + "output_location" => "" + } + + {_kino, source} = start_smart_cell!(ConnectionCell, attrs) + + assert source == "" + end + test "doesn't restore source code with empty required fields" do attrs = %{ "variable" => "db", From 0a5f976e1d50cb395fe6392bf8690fdc94a10975 Mon Sep 17 00:00:00 2001 From: Alexandre de Souza Date: Wed, 13 Jul 2022 14:50:38 -0300 Subject: [PATCH 03/13] Define default AWS region --- lib/kino_db/connection_cell.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/kino_db/connection_cell.ex b/lib/kino_db/connection_cell.ex index 8ccab51..984162a 100644 --- a/lib/kino_db/connection_cell.ex +++ b/lib/kino_db/connection_cell.ex @@ -29,7 +29,7 @@ defmodule KinoDB.ConnectionCell do "access_key_id" => attrs["access_key_id"] || "", "secret_access_key" => attrs["secret_access_key"] || "", "token" => attrs["token"] || "", - "region" => attrs["region"] || "", + "region" => attrs["region"] || "us-east-1", "workgroup" => attrs["workgroup"] || "", "output_location" => attrs["output_location"] || "" } From 9b85e42491560aa7306fedc1eae556d8eb54c469 Mon Sep 17 00:00:00 2001 From: Alexandre de Souza Date: Wed, 13 Jul 2022 16:46:12 -0300 Subject: [PATCH 04/13] Add `:cache_query` option to AWS Athena queries --- lib/assets/sql_cell/main.css | 89 +++++++- lib/assets/sql_cell/main.js | 385 +++++++++++++++++++++++++-------- lib/kino_db/sql_cell.ex | 18 +- test/kino_db/sql_cell_test.exs | 35 +++ 4 files changed, 432 insertions(+), 95 deletions(-) diff --git a/lib/assets/sql_cell/main.css b/lib/assets/sql_cell/main.css index 4d1335b..19ef7ef 100644 --- a/lib/assets/sql_cell/main.css +++ b/lib/assets/sql_cell/main.css @@ -81,7 +81,6 @@ button { .field { display: flex; flex-direction: column; - align-items: flex-start; } .inline-field { @@ -102,7 +101,6 @@ button { .help-box { padding: 16px; - white-space: pre-wrap; font-weight: 500; font-size: 0.875rem; background-color: var(--gray-100); @@ -151,7 +149,6 @@ button { .grow { flex-grow: 1; - flex-shrink: 3; } select.input { @@ -186,3 +183,89 @@ select.input { color: var(--gray-600); padding-left: 0; } + +.row { + display: flex; + align-items: center; + padding: 8px 16px; + gap: 8px; +} + +.help-box .row { + display: flex; + align-items: stretch; + justify-content: flex-start; + padding: 0px; + gap: 16px; +} + +@media only screen and (max-width: 750px) { + .mixed-row .field { + max-width: 32%; + } +} + +/* Switch */ + +.switch-button { + margin-top: 3px; + display: inline-block; + height: 1.75rem; + margin-right: 0.5rem; + position: relative; + transition-duration: 0.15s; + transition-property: color, background-color, border-color, fill, stroke, + opacity, box-shadow, transform, filter, -webkit-text-decoration-color, + -webkit-backdrop-filter; + transition-property: color, background-color, border-color, + text-decoration-color, fill, stroke, opacity, box-shadow, transform, filter, + backdrop-filter; + transition-property: color, background-color, border-color, + text-decoration-color, fill, stroke, opacity, box-shadow, transform, filter, + backdrop-filter, -webkit-text-decoration-color, -webkit-backdrop-filter; + transition-timing-function: cubic-bezier(0.4, 0, 0.2, 1); + -webkit-user-select: none; + -moz-user-select: none; + -ms-user-select: none; + user-select: none; + width: 3.5rem; +} + +.switch-checkbox { + -webkit-appearance: none; + -moz-appearance: none; + appearance: none; + background-color: rgb(145 164 183); + border-color: rgb(240 245 249); + border-width: 5px; + height: 1.25rem; + position: absolute; + width: 1.45rem; + top: 1px; +} + +.switch-button-bg, +.switch-checkbox { + border-radius: 9999px; + cursor: pointer; + display: block; + transition-duration: 0.3s; + transition-property: all; + transition-timing-function: cubic-bezier(0.4, 0, 0.2, 1); +} + +.switch-button-bg { + background-color: var(--gray-200); + height: 100%; + width: 100%; +} + +.switch-checkbox:checked { + background-color: rgb(255 255 255); + border-color: rgb(62 100 255); + transform: translateX(100%); +} + +.switch-checkbox:checked + .switch-button-bg { + background-color: rgb(62 100 255); +} diff --git a/lib/assets/sql_cell/main.js b/lib/assets/sql_cell/main.js index 90c0728..4e6150b 100644 --- a/lib/assets/sql_cell/main.js +++ b/lib/assets/sql_cell/main.js @@ -1,98 +1,342 @@ +import * as Vue from "https://cdn.jsdelivr.net/npm/vue@3.2.26/dist/vue.esm-browser.prod.js"; + export function init(ctx, payload) { ctx.importCSS("main.css"); ctx.importCSS("https://fonts.googleapis.com/css2?family=Inter:wght@400;500&display=swap"); ctx.importCSS("https://cdn.jsdelivr.net/npm/remixicon@2.5.0/fonts/remixicon.min.css"); - ctx.root.innerHTML = ` + const BaseSelect = { + name: "BaseSelect", + + props: { + label: { + type: String, + default: '' + }, + selectClass: { + type: String, + default: 'input' + }, + modelValue: { + type: String, + default: '' + }, + options: { + type: Array, + default: [], + required: true + }, + required: { + type: Boolean, + default: false + }, + inline: { + type: Boolean, + default: false + }, + existent: { + type: Boolean, + default: false + }, + disabled: { + type: Boolean, + default: false + } + }, + + template: ` +
+ + +
+ ` + }; + + const BaseInput = { + name: "BaseInput", + + props: { + label: { + type: String, + default: '' + }, + inputClass: { + type: String, + default: 'input' + }, + modelValue: { + type: [String, Number], + default: '' + }, + inline: { + type: Boolean, + default: false + }, + grow: { + type: Boolean, + default: false + }, + number: { + type: Boolean, + default: false + } + }, + + template: ` +
+ + +
+ ` + }; + + const BaseSwitch = { + name: "BaseSwitch", + + props: { + label: { + type: String, + default: '' + }, + modelValue: { + type: Boolean, + default: true + }, + inline: { + type: Boolean, + default: false + }, + grow: { + type: Boolean, + default: false + } + }, + + template: ` +
+ +
` @@ -146,7 +146,7 @@ export function init(ctx, payload) { @input="$emit('update:data', $event.target.checked)" v-bind="$attrs" class="switch-checkbox" - v-bind:class="[inputClass, number ? 'input-number' : '', ]" + v-bind:class="[inputClass, number ? 'input-number' : '']" >
@@ -236,7 +236,7 @@ export function init(ctx, payload) {
diff --git a/lib/kino_db/connection_cell.ex b/lib/kino_db/connection_cell.ex index e936066..3bfbca5 100644 --- a/lib/kino_db/connection_cell.ex +++ b/lib/kino_db/connection_cell.ex @@ -34,14 +34,15 @@ defmodule KinoDB.ConnectionCell do "output_location" => attrs["output_location"] || "" } - assigns = [ - fields: fields, - missing_dep: missing_dep(fields), - help_box: help_box(fields), - has_aws_credentials: Code.ensure_loaded?(:aws_credentials) - ] - - {:ok, assign(ctx, assigns)} + ctx = + assign(ctx, + fields: fields, + missing_dep: missing_dep(fields), + help_box: help_box(fields), + has_aws_credentials: Code.ensure_loaded?(:aws_credentials) + ) + + {:ok, ctx} end @impl true @@ -100,7 +101,7 @@ defmodule KinoDB.ConnectionCell do defp to_updates(_fields, field, value), do: %{field => value} - @default_keys ["type", "variable", "_keys_"] + @default_keys ["type", "variable"] @impl true def to_attrs(%{assigns: %{fields: fields}}) do @@ -120,9 +121,7 @@ defmodule KinoDB.ConnectionCell do ~w|database hostname port username password| end - fields - |> Map.put("_keys_", connection_keys) - |> Map.take(@default_keys ++ connection_keys) + Map.take(fields, @default_keys ++ connection_keys) end @impl true @@ -150,22 +149,22 @@ defmodule KinoDB.ConnectionCell do _ -> [] end - if required_fields_filled?(attrs, required_keys) and - conditional_fields_filled?(attrs, conditional_keys) do + if all_fields_filled?(attrs, required_keys) and + any_fields_filled?(attrs, conditional_keys) do attrs |> to_quoted() |> Kino.SmartCell.quoted_to_string() else "" end end - defp required_fields_filled?(attrs, keys) do + defp all_fields_filled?(attrs, keys) do not Enum.any?(keys, fn key -> attrs[key] in [nil, ""] end) end - defp conditional_fields_filled?(_, []), do: true + defp any_fields_filled?(_, []), do: true - defp conditional_fields_filled?(attrs, keys) do - not Enum.all?(keys, fn key -> attrs[key] in [nil, ""] end) + defp any_fields_filled?(attrs, keys) do + Enum.any?(keys, fn key -> attrs[key] not in [nil, ""] end) end defp to_quoted(%{"type" => "sqlite"} = attrs) do diff --git a/test/kino_db/sql_cell_test.exs b/test/kino_db/sql_cell_test.exs index d7698a3..77f55d9 100644 --- a/test/kino_db/sql_cell_test.exs +++ b/test/kino_db/sql_cell_test.exs @@ -312,7 +312,7 @@ defmodule KinoDB.SQLCellTest do """ end - test "passes cache_query option when a connection type is specified" do + test "passes cache_query option when supported" do attrs = %{ "connection" => %{"variable" => "conn", "type" => "postgres"}, "result_variable" => "result", From 84d9decccb4b868a3a02fa76501289c20b72c395 Mon Sep 17 00:00:00 2001 From: Alexandre de Souza Date: Thu, 14 Jul 2022 11:19:30 -0300 Subject: [PATCH 09/13] Improve connection cell tests --- test/kino_db/connection_cell_test.exs | 284 ++++++++------------------ 1 file changed, 84 insertions(+), 200 deletions(-) diff --git a/test/kino_db/connection_cell_test.exs b/test/kino_db/connection_cell_test.exs index 1ed9427..c56b838 100644 --- a/test/kino_db/connection_cell_test.exs +++ b/test/kino_db/connection_cell_test.exs @@ -25,207 +25,91 @@ defmodule KinoDB.ConnectionCellTest do {:ok, conn} = Kino.start_child({Postgrex, opts})\ """ end + end - test "restores source code from attrs" do - attrs = %{ - "variable" => "db", - "type" => "mysql", - "hostname" => "localhost", - "port" => 4444, - "username" => "admin", - "password" => "pass", - "database" => "default" - } - - {_kino, source} = start_smart_cell!(ConnectionCell, attrs) - - assert source == - """ - opts = [ - hostname: "localhost", - port: 4444, - username: "admin", - password: "pass", - database: "default", - socket_options: [:inet6] - ] - - {:ok, db} = Kino.start_child({MyXQL, opts})\ - """ - end - - test "restores source code from attrs with SQLite3" do - attrs = %{ - "variable" => "db", - "type" => "sqlite", - "database_path" => "/path/to/sqlite3.db" - } - - {_kino, source} = start_smart_cell!(ConnectionCell, attrs) - - assert source == - """ - opts = [database: "/path/to/sqlite3.db"] - {:ok, db} = Kino.start_child({Exqlite, opts})\ - """ - end - - test "restores source code from attrs with BigQuery" do - attrs = %{ - "variable" => "db", - "type" => "bigquery", - "project_id" => "foo", - "credentials" => %{}, - "default_dataset_id" => "" - } - - {_kino, source} = start_smart_cell!(ConnectionCell, attrs) - - assert source == - """ - opts = [name: ReqBigQuery.Goth, http_client: &Req.request/1] - {:ok, _pid} = Kino.start_child({Goth, opts}) - - db = - Req.new(http_errors: :raise) - |> ReqBigQuery.attach(goth: ReqBigQuery.Goth, project_id: "foo", default_dataset_id: "") - - :ok\ - """ - - credentials = %{ - "private_key" => "foo", - "client_email" => "alice@example.com", - "token_uri" => "/", - "type" => "service_account" - } - - {_kino, source} = - start_smart_cell!(ConnectionCell, put_in(attrs["credentials"], credentials)) - - assert source == - """ - credentials = %{ - "client_email" => "alice@example.com", - "private_key" => "foo", - "token_uri" => "/", - "type" => "service_account" - } - - opts = [ - name: ReqBigQuery.Goth, - http_client: &Req.request/1, - source: {:service_account, credentials} - ] - - {:ok, _pid} = Kino.start_child({Goth, opts}) - - db = - Req.new(http_errors: :raise) - |> ReqBigQuery.attach(goth: ReqBigQuery.Goth, project_id: "foo", default_dataset_id: "") - - :ok\ - """ - - credentials = %{ - "refresh_token" => "foo", - "client_id" => "alice@example.com", - "client_secret" => "bar", - "type" => "authorized_user" - } - - {_kino, source} = - start_smart_cell!(ConnectionCell, put_in(attrs["credentials"], credentials)) - - assert source == - """ - credentials = %{ - "client_id" => "alice@example.com", - "client_secret" => "bar", - "refresh_token" => "foo", - "type" => "authorized_user" - } - - opts = [ - name: ReqBigQuery.Goth, - http_client: &Req.request/1, - source: {:refresh_token, credentials} - ] - - {:ok, _pid} = Kino.start_child({Goth, opts}) - - db = - Req.new(http_errors: :raise) - |> ReqBigQuery.attach(goth: ReqBigQuery.Goth, project_id: "foo", default_dataset_id: "") - - :ok\ - """ - end - - test "restores source code from attrs with Athena" do - attrs = %{ - "variable" => "db", - "type" => "athena", - "access_key_id" => "id", - "secret_access_key" => "secret", - "region" => "region", - "database" => "default", - "output_location" => "s3://my-bucket" - } - - {_kino, source} = start_smart_cell!(ConnectionCell, attrs) - - assert source == - """ - db = - Req.new(http_errors: :raise) - |> ReqAthena.attach( - access_key_id: "id", - database: "default", - output_location: "s3://my-bucket", - region: "region", - secret_access_key: "secret", - token: "", - workgroup: "" - ) - - :ok\ - """ - end - - test "doesn't restore source code with empty conditional fields" do - attrs = %{ - "variable" => "db", - "type" => "athena", - "access_key_id" => "id", - "secret_access_key" => "secret", - "region" => "region", - "database" => "default", - "token" => "token", - "workgroup" => "", - "output_location" => "" - } - - {_kino, source} = start_smart_cell!(ConnectionCell, attrs) - - assert source == "" - end - - test "doesn't restore source code with empty required fields" do - attrs = %{ - "variable" => "db", - "type" => "mysql", - "hostname" => "", - "port" => nil, - "username" => "admin", - "password" => "pass", - "database" => "default" - } - - {_kino, source} = start_smart_cell!(ConnectionCell, attrs) - - assert source == "" - end + test "restores source code from attrs" do + attrs = %{ + "variable" => "db", + "type" => "postgres", + "hostname" => "localhost", + "port" => 4444, + "username" => "admin", + "password" => "pass", + "database" => "default", + "database_path" => "/path/to/sqlite3.db", + "project_id" => "foo", + "credentials" => %{}, + "default_dataset_id" => "", + "access_key_id" => "id", + "secret_access_key" => "secret", + "token" => "token", + "region" => "region", + "output_location" => "s3://my-bucket", + "workgroup" => "primary" + } + + conditional_attrs = Map.merge(attrs, %{"workgroup" => "", "output_location" => ""}) + + assert ConnectionCell.to_source(attrs) === ~s''' + opts = [ + hostname: "localhost", + port: 4444, + username: "admin", + password: "pass", + database: "default", + socket_options: [:inet6] + ] + + {:ok, db} = Kino.start_child({Postgrex, opts})\ + ''' + + assert ConnectionCell.to_source(put_in(attrs["type"], "mysql")) == ~s''' + opts = [ + hostname: "localhost", + port: 4444, + username: "admin", + password: "pass", + database: "default", + socket_options: [:inet6] + ] + + {:ok, db} = Kino.start_child({MyXQL, opts})\ + ''' + + assert ConnectionCell.to_source(put_in(attrs["type"], "sqlite")) == ~s''' + opts = [database: "/path/to/sqlite3.db"] + {:ok, db} = Kino.start_child({Exqlite, opts})\ + ''' + + assert ConnectionCell.to_source(put_in(attrs["type"], "bigquery")) == ~s''' + opts = [name: ReqBigQuery.Goth, http_client: &Req.request/1] + {:ok, _pid} = Kino.start_child({Goth, opts}) + + db = + Req.new(http_errors: :raise) + |> ReqBigQuery.attach(goth: ReqBigQuery.Goth, project_id: "foo", default_dataset_id: "") + + :ok\ + ''' + + assert ConnectionCell.to_source(put_in(attrs["type"], "athena")) == ~s''' + db = + Req.new(http_errors: :raise) + |> ReqAthena.attach( + access_key_id: "id", + database: "default", + output_location: "s3://my-bucket", + region: "region", + secret_access_key: "secret", + token: "token", + workgroup: "primary" + ) + + :ok\ + ''' + + assert ConnectionCell.to_source(put_in(attrs["port"], nil)) == "" + assert ConnectionCell.to_source(put_in(conditional_attrs["type"], "athena")) == "" end test "when a field changes, broadcasts the change and sends source update" do From 80a7726e20cedb3ec76c1245d16fc63497d32199 Mon Sep 17 00:00:00 2001 From: Alexandre de Souza Date: Thu, 14 Jul 2022 11:35:50 -0300 Subject: [PATCH 10/13] Apply review comments --- test/kino_db/connection_cell_test.exs | 191 +++++++++++++++----------- 1 file changed, 109 insertions(+), 82 deletions(-) diff --git a/test/kino_db/connection_cell_test.exs b/test/kino_db/connection_cell_test.exs index c56b838..c5a0cf5 100644 --- a/test/kino_db/connection_cell_test.exs +++ b/test/kino_db/connection_cell_test.exs @@ -27,89 +27,116 @@ defmodule KinoDB.ConnectionCellTest do end end - test "restores source code from attrs" do - attrs = %{ - "variable" => "db", - "type" => "postgres", - "hostname" => "localhost", - "port" => 4444, - "username" => "admin", - "password" => "pass", - "database" => "default", - "database_path" => "/path/to/sqlite3.db", - "project_id" => "foo", - "credentials" => %{}, - "default_dataset_id" => "", - "access_key_id" => "id", - "secret_access_key" => "secret", - "token" => "token", - "region" => "region", - "output_location" => "s3://my-bucket", - "workgroup" => "primary" - } - - conditional_attrs = Map.merge(attrs, %{"workgroup" => "", "output_location" => ""}) - - assert ConnectionCell.to_source(attrs) === ~s''' - opts = [ - hostname: "localhost", - port: 4444, - username: "admin", - password: "pass", - database: "default", - socket_options: [:inet6] - ] - - {:ok, db} = Kino.start_child({Postgrex, opts})\ - ''' - - assert ConnectionCell.to_source(put_in(attrs["type"], "mysql")) == ~s''' - opts = [ - hostname: "localhost", - port: 4444, - username: "admin", - password: "pass", - database: "default", - socket_options: [:inet6] - ] - - {:ok, db} = Kino.start_child({MyXQL, opts})\ - ''' - - assert ConnectionCell.to_source(put_in(attrs["type"], "sqlite")) == ~s''' - opts = [database: "/path/to/sqlite3.db"] - {:ok, db} = Kino.start_child({Exqlite, opts})\ - ''' - - assert ConnectionCell.to_source(put_in(attrs["type"], "bigquery")) == ~s''' - opts = [name: ReqBigQuery.Goth, http_client: &Req.request/1] - {:ok, _pid} = Kino.start_child({Goth, opts}) - - db = - Req.new(http_errors: :raise) - |> ReqBigQuery.attach(goth: ReqBigQuery.Goth, project_id: "foo", default_dataset_id: "") - - :ok\ - ''' - - assert ConnectionCell.to_source(put_in(attrs["type"], "athena")) == ~s''' - db = - Req.new(http_errors: :raise) - |> ReqAthena.attach( - access_key_id: "id", + describe "code generation" do + test "restores source code from attrs" do + attrs = %{ + "variable" => "db", + "type" => "postgres", + "hostname" => "localhost", + "port" => 4444, + "username" => "admin", + "password" => "pass", + "database" => "default", + "database_path" => "/path/to/sqlite3.db", + "project_id" => "foo", + "credentials" => %{}, + "default_dataset_id" => "", + "access_key_id" => "id", + "secret_access_key" => "secret", + "token" => "token", + "region" => "region", + "output_location" => "s3://my-bucket", + "workgroup" => "primary" + } + + assert ConnectionCell.to_source(attrs) === ~s''' + opts = [ + hostname: "localhost", + port: 4444, + username: "admin", + password: "pass", + database: "default", + socket_options: [:inet6] + ] + + {:ok, db} = Kino.start_child({Postgrex, opts})\ + ''' + + assert ConnectionCell.to_source(put_in(attrs["type"], "mysql")) == ~s''' + opts = [ + hostname: "localhost", + port: 4444, + username: "admin", + password: "pass", database: "default", - output_location: "s3://my-bucket", - region: "region", - secret_access_key: "secret", - token: "token", - workgroup: "primary" - ) - - :ok\ - ''' - - assert ConnectionCell.to_source(put_in(attrs["port"], nil)) == "" - assert ConnectionCell.to_source(put_in(conditional_attrs["type"], "athena")) == "" + socket_options: [:inet6] + ] + + {:ok, db} = Kino.start_child({MyXQL, opts})\ + ''' + + assert ConnectionCell.to_source(put_in(attrs["type"], "sqlite")) == ~s''' + opts = [database: "/path/to/sqlite3.db"] + {:ok, db} = Kino.start_child({Exqlite, opts})\ + ''' + + assert ConnectionCell.to_source(put_in(attrs["type"], "bigquery")) == ~s''' + opts = [name: ReqBigQuery.Goth, http_client: &Req.request/1] + {:ok, _pid} = Kino.start_child({Goth, opts}) + + db = + Req.new(http_errors: :raise) + |> ReqBigQuery.attach(goth: ReqBigQuery.Goth, project_id: "foo", default_dataset_id: "") + + :ok\ + ''' + + assert ConnectionCell.to_source(put_in(attrs["type"], "athena")) == ~s''' + db = + Req.new(http_errors: :raise) + |> ReqAthena.attach( + access_key_id: "id", + database: "default", + output_location: "s3://my-bucket", + region: "region", + secret_access_key: "secret", + token: "token", + workgroup: "primary" + ) + + :ok\ + ''' + end + + test "doesn't restore source code with empty required fields" do + attrs = %{ + "variable" => "db", + "type" => "postgres", + "hostname" => "localhost", + "port" => nil, + "username" => "admin", + "password" => "pass", + "database" => "default" + } + + assert ConnectionCell.to_source(attrs) == "" + end + + test "doesn't restore source code with empty conditional fields" do + attrs = %{ + "variable" => "db", + "type" => "postgres", + "database" => "default", + "access_key_id" => "id", + "secret_access_key" => "secret", + "token" => "token", + "region" => "region", + "output_location" => "", + "workgroup" => "" + } + + assert ConnectionCell.to_source(attrs) == "" + end end test "when a field changes, broadcasts the change and sends source update" do From 47187047211a238672e958a86f348eefec8189af Mon Sep 17 00:00:00 2001 From: Alexandre de Souza Date: Thu, 14 Jul 2022 11:48:46 -0300 Subject: [PATCH 11/13] Apply review comments --- test/kino_db/connection_cell_test.exs | 94 +++++++++++++-------------- 1 file changed, 45 insertions(+), 49 deletions(-) diff --git a/test/kino_db/connection_cell_test.exs b/test/kino_db/connection_cell_test.exs index c5a0cf5..03dde48 100644 --- a/test/kino_db/connection_cell_test.exs +++ b/test/kino_db/connection_cell_test.exs @@ -7,6 +7,38 @@ defmodule KinoDB.ConnectionCellTest do setup :configure_livebook_bridge + @attrs %{ + "variable" => "db", + "type" => "postgres", + "hostname" => "localhost", + "port" => 4444, + "username" => "admin", + "password" => "pass", + "database" => "default", + "database_path" => "/path/to/sqlite3.db", + "project_id" => "foo", + "credentials" => %{}, + "default_dataset_id" => "", + "access_key_id" => "id", + "secret_access_key" => "secret", + "token" => "token", + "region" => "region", + "output_location" => "s3://my-bucket", + "workgroup" => "primary" + } + + @empty_required_fields %{ + "variable" => "db", + "type" => "postgres", + "hostname" => "", + "port" => nil, + "database_path" => "", + "project_id" => "", + "access_key_id" => "", + "secret_access_key" => "", + "region" => "" + } + describe "initialization" do test "returns default source when started with missing attrs" do {_kino, source} = start_smart_cell!(ConnectionCell, %{"variable" => "conn"}) @@ -29,27 +61,7 @@ defmodule KinoDB.ConnectionCellTest do describe "code generation" do test "restores source code from attrs" do - attrs = %{ - "variable" => "db", - "type" => "postgres", - "hostname" => "localhost", - "port" => 4444, - "username" => "admin", - "password" => "pass", - "database" => "default", - "database_path" => "/path/to/sqlite3.db", - "project_id" => "foo", - "credentials" => %{}, - "default_dataset_id" => "", - "access_key_id" => "id", - "secret_access_key" => "secret", - "token" => "token", - "region" => "region", - "output_location" => "s3://my-bucket", - "workgroup" => "primary" - } - - assert ConnectionCell.to_source(attrs) === ~s''' + assert ConnectionCell.to_source(@attrs) === ~s''' opts = [ hostname: "localhost", port: 4444, @@ -62,7 +74,7 @@ defmodule KinoDB.ConnectionCellTest do {:ok, db} = Kino.start_child({Postgrex, opts})\ ''' - assert ConnectionCell.to_source(put_in(attrs["type"], "mysql")) == ~s''' + assert ConnectionCell.to_source(put_in(@attrs["type"], "mysql")) == ~s''' opts = [ hostname: "localhost", port: 4444, @@ -75,12 +87,12 @@ defmodule KinoDB.ConnectionCellTest do {:ok, db} = Kino.start_child({MyXQL, opts})\ ''' - assert ConnectionCell.to_source(put_in(attrs["type"], "sqlite")) == ~s''' + assert ConnectionCell.to_source(put_in(@attrs["type"], "sqlite")) == ~s''' opts = [database: "/path/to/sqlite3.db"] {:ok, db} = Kino.start_child({Exqlite, opts})\ ''' - assert ConnectionCell.to_source(put_in(attrs["type"], "bigquery")) == ~s''' + assert ConnectionCell.to_source(put_in(@attrs["type"], "bigquery")) == ~s''' opts = [name: ReqBigQuery.Goth, http_client: &Req.request/1] {:ok, _pid} = Kino.start_child({Goth, opts}) @@ -91,7 +103,7 @@ defmodule KinoDB.ConnectionCellTest do :ok\ ''' - assert ConnectionCell.to_source(put_in(attrs["type"], "athena")) == ~s''' + assert ConnectionCell.to_source(put_in(@attrs["type"], "athena")) == ~s''' db = Req.new(http_errors: :raise) |> ReqAthena.attach( @@ -108,32 +120,16 @@ defmodule KinoDB.ConnectionCellTest do ''' end - test "doesn't restore source code with empty required fields" do - attrs = %{ - "variable" => "db", - "type" => "postgres", - "hostname" => "localhost", - "port" => nil, - "username" => "admin", - "password" => "pass", - "database" => "default" - } - - assert ConnectionCell.to_source(attrs) == "" + test "generates empty source code when all required fields are missing" do + assert ConnectionCell.to_source(put_in(@empty_required_fields["type"], "postgres")) == "" + assert ConnectionCell.to_source(put_in(@empty_required_fields["type"], "mysql")) == "" + assert ConnectionCell.to_source(put_in(@empty_required_fields["type"], "sqlite")) == "" + assert ConnectionCell.to_source(put_in(@empty_required_fields["type"], "bigquery")) == "" + assert ConnectionCell.to_source(put_in(@empty_required_fields["type"], "athena")) == "" end - test "doesn't restore source code with empty conditional fields" do - attrs = %{ - "variable" => "db", - "type" => "postgres", - "database" => "default", - "access_key_id" => "id", - "secret_access_key" => "secret", - "token" => "token", - "region" => "region", - "output_location" => "", - "workgroup" => "" - } + test "generates empty source code when all conditional fields are missing" do + attrs = Map.merge(@attrs, %{"type" => "athena", "workgroup" => "", "output_location" => ""}) assert ConnectionCell.to_source(attrs) == "" end From d241638c775b928032756aa388e78ae6b53087d2 Mon Sep 17 00:00:00 2001 From: Alexandre de Souza Date: Thu, 14 Jul 2022 11:51:55 -0300 Subject: [PATCH 12/13] Apply review comments --- test/kino_db/connection_cell_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/kino_db/connection_cell_test.exs b/test/kino_db/connection_cell_test.exs index 03dde48..e5a5c4e 100644 --- a/test/kino_db/connection_cell_test.exs +++ b/test/kino_db/connection_cell_test.exs @@ -120,7 +120,7 @@ defmodule KinoDB.ConnectionCellTest do ''' end - test "generates empty source code when all required fields are missing" do + test "generates empty source code when required fields are missing" do assert ConnectionCell.to_source(put_in(@empty_required_fields["type"], "postgres")) == "" assert ConnectionCell.to_source(put_in(@empty_required_fields["type"], "mysql")) == "" assert ConnectionCell.to_source(put_in(@empty_required_fields["type"], "sqlite")) == "" From 877145c2b6b443d5970105334bbe4f136923175d Mon Sep 17 00:00:00 2001 From: Alexandre de Souza Date: Thu, 14 Jul 2022 11:57:37 -0300 Subject: [PATCH 13/13] Update AWS Athena help text --- lib/assets/connection_cell/main.js | 13 +++++++------ lib/kino_db/connection_cell.ex | 6 ++++++ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/lib/assets/connection_cell/main.js b/lib/assets/connection_cell/main.js index 75b4abb..69c90f0 100644 --- a/lib/assets/connection_cell/main.js +++ b/lib/assets/connection_cell/main.js @@ -215,6 +215,10 @@ export function init(ctx, info) { type: Object, default: {} }, + helpBox: { + type: String, + default: "" + }, hasAwsCredentials: { type: Boolean, default: false @@ -300,9 +304,7 @@ export function init(ctx, info) { :required="!!areFieldsEmpty(fields.output_location, fields.workgroup)" /> - - You must use your AWS Credentials or authenticate your machine with aws CLI authentication. - + ` }; @@ -318,11 +320,10 @@ export function init(ctx, info) { type: Object, default: {} }, - helpBox: { type: String, default: "" - }, + } }, methods: { @@ -435,7 +436,7 @@ export function init(ctx, info) { - + diff --git a/lib/kino_db/connection_cell.ex b/lib/kino_db/connection_cell.ex index 3bfbca5..5a0c450 100644 --- a/lib/kino_db/connection_cell.ex +++ b/lib/kino_db/connection_cell.ex @@ -342,6 +342,12 @@ defmodule KinoDB.ConnectionCell do end end + defp help_box(%{"type" => "athena"}) do + if Code.ensure_loaded?(:aws_credentials) do + "You must fill in the fields above accordingly or authenticate your machine with AWS CLI authentication." + end + end + defp help_box(_ctx), do: nil defp running_on_google_metadata? do