Skip to content

Commit

Permalink
Saving data links to a DB_Table (#11371)
Browse files Browse the repository at this point in the history
- Closes #11295
  • Loading branch information
radeusgd authored Oct 24, 2024
1 parent fe45da9 commit ca9df70
Show file tree
Hide file tree
Showing 32 changed files with 592 additions and 157 deletions.
10 changes: 7 additions & 3 deletions .github/workflows/extra-nightly-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,9 @@ on:
default: false
jobs:
enso-build-ci-gen-job-snowflake-tests-linux-amd64:
name: Snowflake Tests (linux, amd64)
name: Snowflake Tests (LinuxLatest)
runs-on:
- self-hosted
- Linux
- ubuntu-latest
steps:
- if: startsWith(runner.name, 'GitHub Actions') || startsWith(runner.name, 'Hosted Agent')
name: Installing wasm-pack
Expand All @@ -44,6 +43,11 @@ jobs:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: ./run backend test std-snowflake
env:
ENSO_CLOUD_COGNITO_REGION: ${{ vars.ENSO_CLOUD_COGNITO_REGION }}
ENSO_CLOUD_COGNITO_USER_POOL_ID: ${{ vars.ENSO_CLOUD_COGNITO_USER_POOL_ID }}
ENSO_CLOUD_COGNITO_USER_POOL_WEB_CLIENT_ID: ${{ vars.ENSO_CLOUD_COGNITO_USER_POOL_WEB_CLIENT_ID }}
ENSO_CLOUD_TEST_ACCOUNT_PASSWORD: ${{ secrets.ENSO_CLOUD_TEST_ACCOUNT_PASSWORD }}
ENSO_CLOUD_TEST_ACCOUNT_USERNAME: ${{ secrets.ENSO_CLOUD_TEST_ACCOUNT_USERNAME }}
ENSO_SNOWFLAKE_ACCOUNT: ${{ secrets.ENSO_SNOWFLAKE_ACCOUNT }}
ENSO_SNOWFLAKE_DATABASE: ${{ secrets.ENSO_SNOWFLAKE_DATABASE }}
ENSO_SNOWFLAKE_PASSWORD: ${{ secrets.ENSO_SNOWFLAKE_PASSWORD }}
Expand Down
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,11 @@
cloud.][11235]
- [The user may set description and labels of an Enso Cloud asset
programmatically.][11255]
- [DB_Table may be saved as a Data Link.][11371]

[11235]: https://github.com/enso-org/enso/pull/11235
[11255]: https://github.com/enso-org/enso/pull/11255
[11371]: https://github.com/enso-org/enso/pull/11371

#### Enso Language & Runtime

Expand Down
7 changes: 6 additions & 1 deletion app/gui/src/dashboard/data/__tests__/dataLinkSchema.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,12 @@ v.test('correctly validates example Table .datalink files with the schema', () =
})

v.test('correctly validates example Database .datalink files with the schema', () => {
const schemas = ['postgres-db.datalink', 'postgres-table.datalink']
const schemas = [
'postgres-db.datalink',
'postgres-table.datalink',
'postgres-simple-query.datalink',
'postgres-serialized-query.datalink',
]
for (const schema of schemas) {
const json = loadDataLinkFile(path.resolve(TABLE_DATA_LINKS_ROOT, schema))
testSchema(json, schema)
Expand Down
35 changes: 32 additions & 3 deletions app/gui/src/dashboard/data/datalinkSchema.json
Original file line number Diff line number Diff line change
Expand Up @@ -188,7 +188,7 @@
},
"required": ["username", "password"]
},
"table": { "title": "Table to access", "type": "string" }
"table": { "title": "Table to access", "$ref": "#/$defs/DatabaseTableDefinition" }
},
"required": ["type", "libraryName", "host", "port", "database_name"]
},
Expand Down Expand Up @@ -233,7 +233,7 @@
},
"required": ["username", "password"]
},
"table": { "title": "Table to access", "type": "string" }
"table": { "title": "Table to access", "$ref": "#/$defs/DatabaseTableDefinition" }
},
"required": ["type", "libraryName", "account", "database_name", "credentials"]
},
Expand Down Expand Up @@ -277,7 +277,7 @@
},
"required": ["username", "password"]
},
"table": { "title": "Table to access", "type": "string" }
"table": { "title": "Table to access", "$ref": "#/$defs/DatabaseTableDefinition" }
},
"required": ["type", "libraryName", "host", "port", "database_name"]
},
Expand Down Expand Up @@ -458,6 +458,35 @@
}
},
"required": ["type", "subType"]
},

"DatabaseTableDefinition": {
"title": "Query to read",
"anyOf": [
{ "title": "Table", "type": "string" },
{
"title": "SQL Query",
"type": "object",
"properties": {
"query": {
"title": "SQL",
"type": "string"
}
},
"required": ["query"]
},
{
"title": "(Advanced) JSON serialized interpolated SQL statement",
"type": "object",
"properties": {
"sql_statement": {
"title": "SQL_Statement (JSON)",
"type": "string"
}
},
"required": ["sql_statement"]
}
]
}
}
}
20 changes: 14 additions & 6 deletions build/build/src/ci_gen/job.rs
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
use core::panic;

use crate::prelude::*;

use crate::ci_gen::not_default_branch;
Expand Down Expand Up @@ -307,6 +309,10 @@ fn build_job_ensuring_cloud_tests_run_on_github(
cloud_tests_enabled: bool,
) -> Job {
if cloud_tests_enabled {
if target.0 != OS::Linux {
panic!("If the Cloud tests are enabled, they require GitHub hosted runner for Cloud auth, so they only run on Linux.");
}

run_steps_builder.build_job(job_name, RunnerLabel::LinuxLatest)
} else {
run_steps_builder.build_job(job_name, target)
Expand All @@ -320,6 +326,9 @@ const GRAAL_EDITION_FOR_EXTRA_TESTS: graalvm::Edition = graalvm::Edition::Commun

impl JobArchetype for SnowflakeTests {
fn job(&self, target: Target) -> Job {
if target.0 != OS::Linux {
panic!("Snowflake tests currently require GitHub hosted runner for Cloud auth, so they only run on Linux.");
}
let job_name = "Snowflake Tests";
let mut job = RunStepsBuilder::new("backend test std-snowflake")
.customize(move |step| {
Expand Down Expand Up @@ -349,17 +358,16 @@ impl JobArchetype for SnowflakeTests {
crate::libraries_tests::snowflake::env::ENSO_SNOWFLAKE_WAREHOUSE,
);

// Temporarily disabled until we can get the Cloud auth fixed.
// Snowflake does not rely on cloud anyway, so it can be disabled.
// But it will rely once we add datalink tests, so this should be fixed soon.
// let updated_main_step = enable_cloud_tests(main_step);
// Snowflake tests are run only in the 'Extra' job, so it is okay to run it with
// Enso Cloud as well. They need it to test data link integration.
let updated_main_step = enable_cloud_tests(main_step);

vec![
main_step,
updated_main_step,
step::extra_stdlib_test_reporter(target, GRAAL_EDITION_FOR_EXTRA_TESTS),
]
})
.build_job(job_name, target)
.build_job(job_name, RunnerLabel::LinuxLatest)
.with_permission(Permission::Checks, Access::Write);
job.env(env::GRAAL_EDITION, GRAAL_EDITION_FOR_EXTRA_TESTS);
job
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -267,7 +267,7 @@ Text.characters self =
## This matches `aBc` @ character 11
"aabbbbccccaaBcaaaa".find "a[ab]c" Case_Sensitivity.Insensitive
Text.find : (Regex | Text) -> Case_Sensitivity -> Match | Nothing ! Regex_Syntax_Error | Illegal_Argument
Text.find self pattern:(Regex | Text)=".*" case_sensitivity=Case_Sensitivity.Sensitive =
Text.find self pattern:(Regex | Text)=".*" case_sensitivity:Case_Sensitivity=..Sensitive =
case_insensitive = case_sensitivity.is_case_insensitive_in_memory
compiled_pattern = Regex.compile pattern case_insensitive=case_insensitive
compiled_pattern.match self
Expand Down Expand Up @@ -299,8 +299,8 @@ Text.find self pattern:(Regex | Text)=".*" case_sensitivity=Case_Sensitivity.Sen
example_find_all_insensitive =
## This matches `aABbbbc` @ character 0 and `aBC` @ character 11
"aABbbbccccaaBCaaaa".find_all "a[ab]+c" Case_Sensitivity.Insensitive
Text.find_all : Text -> Case_Sensitivity -> Vector Match ! Regex_Syntax_Error | Illegal_Argument
Text.find_all self pattern=".*" case_sensitivity=Case_Sensitivity.Sensitive =
Text.find_all : Text|Regex -> Case_Sensitivity -> Vector Match ! Regex_Syntax_Error | Illegal_Argument
Text.find_all self pattern:Text|Regex=".*" case_sensitivity:Case_Sensitivity=..Sensitive =
case_insensitive = case_sensitivity.is_case_insensitive_in_memory
compiled_pattern = Regex.compile pattern case_insensitive=case_insensitive
compiled_pattern.match_all self
Expand Down Expand Up @@ -334,8 +334,8 @@ Text.find_all self pattern=".*" case_sensitivity=Case_Sensitivity.Sensitive =
regex = ".+ct@.+"
# Evaluates to true
"[email protected]".match regex Case_Sensitivity.Insensitive
Text.match : Text -> Case_Sensitivity -> Boolean ! Regex_Syntax_Error | Illegal_Argument
Text.match self pattern=".*" case_sensitivity=Case_Sensitivity.Sensitive =
Text.match : Text|Regex -> Case_Sensitivity -> Boolean ! Regex_Syntax_Error | Illegal_Argument
Text.match self pattern:Text|Regex=".*" case_sensitivity:Case_Sensitivity=..Sensitive =
case_insensitive = case_sensitivity.is_case_insensitive_in_memory
compiled_pattern = Regex.compile pattern case_insensitive=case_insensitive
compiled_pattern.matches self
Expand Down Expand Up @@ -394,7 +394,7 @@ Text.to_regex self case_insensitive=False = Regex.compile self case_insensitive
'azbzczdzezfzg'.split ['b', 'zez'] == ['az', 'zczd', 'fzg']
@delimiter make_delimiter_selector
Text.split : Text | Vector Text -> Case_Sensitivity -> Boolean -> Vector Text ! Illegal_Argument
Text.split self delimiter="," case_sensitivity=Case_Sensitivity.Sensitive use_regex=False =
Text.split self delimiter="," case_sensitivity:Case_Sensitivity=..Sensitive use_regex=False =
delimiter_is_empty = case delimiter of
_ : Text -> delimiter.is_empty
_ : Vector -> delimiter.is_empty || delimiter.any (.is_empty)
Expand Down Expand Up @@ -452,8 +452,8 @@ Text.split self delimiter="," case_sensitivity=Case_Sensitivity.Sensitive use_re

'Hello Big\r\nWide\tWorld\nGoodbye!' . tokenize "(\S+)(?:\s+|$)"
== ["Hello","Big","Wide","World","Goodbye!"]
Text.tokenize : Text -> Case_Sensitivity -> Vector Text
Text.tokenize self pattern:Text=(Missing_Argument.throw "pattern") case_sensitivity:Case_Sensitivity=..Sensitive =
Text.tokenize : Text|Regex -> Case_Sensitivity -> Vector Text
Text.tokenize self pattern:Text|Regex=(Missing_Argument.throw "pattern") case_sensitivity:Case_Sensitivity=..Sensitive =
case_insensitive = case_sensitivity.is_case_insensitive_in_memory
compiled_pattern = Regex.compile pattern case_insensitive=case_insensitive
compiled_pattern.tokenize self
Expand Down Expand Up @@ -614,7 +614,7 @@ Cleansable_Text.from (that:Text) = Cleansable_Text.Value (pattern->replace_with-

"แมวมีสี่ขา".words == ['แมว', 'มี', 'สี่', 'ขา']
Text.words : Boolean -> Vector Text
Text.words self keep_whitespace=False =
Text.words self keep_whitespace:Boolean=False =
iterator = BreakIterator.getWordInstance
iterator.setText self
Vector.build builder->
Expand Down Expand Up @@ -657,7 +657,7 @@ Text.words self keep_whitespace=False =

'\na\nb\n'.lines keep_endings=True == ['\n', 'a\n', 'b\n']
Text.lines : Boolean -> Vector Text
Text.lines self keep_endings=False =
Text.lines self keep_endings:Boolean=False =
Vector.from_polyglot_array (Text_Utils.split_on_lines self keep_endings)

## GROUP Text
Expand All @@ -684,7 +684,7 @@ Text.lines self keep_endings=False =
"Hello World!".insert 5 " Cruel" == "Hello Cruel World!"
"Hello World!".insert -1 " Cruel" == "Hello World! Cruel"
Text.insert : Integer -> Text -> Text ! Index_Out_Of_Bounds
Text.insert self index that =
Text.insert self index:Integer that:Text =
len = self.length
idx = if index < 0 then len + index + 1 else index
if (idx < 0) || (idx > len) then Error.throw (Index_Out_Of_Bounds.Error index len) else
Expand Down Expand Up @@ -718,7 +718,7 @@ Text.insert self index that =
"A0".is_digit 1 == True
"건반(Korean)".is_digit 1 == False
Text.is_digit : Integer -> Boolean ! Index_Out_Of_Bounds
Text.is_digit self (index=0) =
Text.is_digit self index:Integer=0 =
grapheme = self.at index
char = (Text_Utils.get_chars grapheme).at 0
char>=48 && char<=57
Expand Down Expand Up @@ -903,7 +903,7 @@ Text.from_codepoints codepoints = Text_Utils.from_codepoints codepoints
"Hello!".starts_with "hello" == False
"Hello!".starts_with "hello" Case_Sensitivity.Insensitive == True
Text.starts_with : Text -> Case_Sensitivity -> Boolean
Text.starts_with self prefix case_sensitivity=Case_Sensitivity.Sensitive = case case_sensitivity of
Text.starts_with self prefix:Text case_sensitivity:Case_Sensitivity=..Sensitive = case case_sensitivity of
Case_Sensitivity.Default -> self.starts_with prefix Case_Sensitivity.Sensitive
Case_Sensitivity.Sensitive -> Text_Utils.starts_with self prefix
Case_Sensitivity.Insensitive locale ->
Expand Down Expand Up @@ -933,7 +933,7 @@ Text.starts_with self prefix case_sensitivity=Case_Sensitivity.Sensitive = case
"Hello World".ends_with "world" == False
"Hello World".ends_with "world" Case_Sensitivity.Insensitive == True
Text.ends_with : Text -> Case_Sensitivity -> Boolean
Text.ends_with self suffix case_sensitivity=Case_Sensitivity.Sensitive = case case_sensitivity of
Text.ends_with self suffix:Text case_sensitivity:Case_Sensitivity=..Sensitive = case case_sensitivity of
Case_Sensitivity.Default -> self.ends_with suffix Case_Sensitivity.Sensitive
Case_Sensitivity.Sensitive -> Text_Utils.ends_with self suffix
Case_Sensitivity.Insensitive locale ->
Expand Down Expand Up @@ -979,7 +979,7 @@ Text.ends_with self suffix case_sensitivity=Case_Sensitivity.Sensitive = case ca

"Hello!".contains "LO" Case_Sensitivity.Insensitive
Text.contains : Text -> Case_Sensitivity -> Boolean
Text.contains self term="" case_sensitivity=Case_Sensitivity.Sensitive = case case_sensitivity of
Text.contains self term:Text="" case_sensitivity:Case_Sensitivity=..Sensitive = case case_sensitivity of
Case_Sensitivity.Default -> self.contains term Case_Sensitivity.Sensitive
Case_Sensitivity.Sensitive -> Text_Utils.contains self term
Case_Sensitivity.Insensitive locale ->
Expand All @@ -1004,7 +1004,7 @@ Text.contains self term="" case_sensitivity=Case_Sensitivity.Sensitive = case ca

"Hello " * 2 == "Hello Hello "
Text.* : Integer -> Text
Text.* self count = self.repeat count
Text.* self count:Integer = self.repeat count

## GROUP Calculations
ICON text
Expand All @@ -1025,7 +1025,7 @@ Text.* self count = self.repeat count

"Hello ".repeat 2 == "Hello Hello "
Text.repeat : Integer -> Text
Text.repeat self count=1 =
Text.repeat self count:Integer=1 =
0.up_to count . fold "" acc-> _-> acc + self

## ALIAS first, head, keep, last, left, limit, mid, right, slice, substring, tail, top
Expand Down Expand Up @@ -1343,7 +1343,7 @@ Text.trim self where:Location=..Both what=_.is_whitespace =
match_1 == match_2

Text.locate : Text -> Matching_Mode -> Case_Sensitivity -> Span | Nothing
Text.locate self term="" mode=Matching_Mode.First case_sensitivity=Case_Sensitivity.Sensitive = case case_sensitivity of
Text.locate self term:Text="" mode=Matching_Mode.First case_sensitivity:Case_Sensitivity=..Sensitive = case case_sensitivity of
Case_Sensitivity.Default -> self.locate term mode Case_Sensitivity.Sensitive
Case_Sensitivity.Sensitive ->
codepoint_span = case mode of
Expand Down Expand Up @@ -1434,7 +1434,7 @@ Text.locate self term="" mode=Matching_Mode.First case_sensitivity=Case_Sensitiv
match_2 = ligatures . locate_all "ffiff" case_sensitivity=Case_Sensitive.Insensitive
match_2 . map .length == [2, 5]
Text.locate_all : Text -> Case_Sensitivity -> Vector Span
Text.locate_all self term="" case_sensitivity=Case_Sensitivity.Sensitive = if term.is_empty then Vector.new (self.length + 1) (ix -> Span.Value (ix.up_to ix) self) else case case_sensitivity of
Text.locate_all self term:Text="" case_sensitivity:Case_Sensitivity=..Sensitive = if term.is_empty then Vector.new (self.length + 1) (ix -> Span.Value (ix.up_to ix) self) else case case_sensitivity of
Case_Sensitivity.Default -> self.locate term Case_Sensitivity.Sensitive
Case_Sensitivity.Sensitive ->
codepoint_spans = Vector.from_polyglot_array <| Text_Utils.span_of_all self term
Expand Down Expand Up @@ -1479,7 +1479,7 @@ Text.locate_all self term="" case_sensitivity=Case_Sensitivity.Sensitive = if te
"Hello World!".index_of "J" == Nothing
"Hello World!".index_of "o" == 4
Text.index_of : Text -> Integer -> Case_Sensitivity -> Integer | Nothing
Text.index_of self term="" (start : Integer = 0) case_sensitivity=Case_Sensitivity.Sensitive =
Text.index_of self term:Text="" (start : Integer = 0) case_sensitivity:Case_Sensitivity=..Sensitive =
used_start = if start < 0 then start+self.length else start
if used_start < 0 || used_start > self.length then Error.throw (Index_Out_Of_Bounds.Error start self.length+1) else
used = if used_start == 0 then self else self.drop used_start
Expand Down Expand Up @@ -1514,7 +1514,7 @@ Text.index_of self term="" (start : Integer = 0) case_sensitivity=Case_Sensitivi
"Hello World!".last_index_of "J" == Nothing
"Hello World!".last_index_of "o" == 7
Text.last_index_of : Text -> Integer -> Case_Sensitivity -> Integer | Nothing
Text.last_index_of self term="" start=-1 case_sensitivity=Case_Sensitivity.Sensitive =
Text.last_index_of self term:Text="" start=-1 case_sensitivity:Case_Sensitivity=..Sensitive =
used_start = if start < 0 then start+self.length else start
if used_start < 0 || used_start >= self.length then Error.throw (Index_Out_Of_Bounds.Error start self.length) else
used = if used_start == self.length-1 then self else self.take used_start+1
Expand Down
Loading

0 comments on commit ca9df70

Please sign in to comment.