Skip to content

Commit

Permalink
feat: make struct form fields ordered (#1181)
Browse files Browse the repository at this point in the history
  • Loading branch information
czgu authored Feb 25, 2023
1 parent 8ab460f commit 4a7c8da
Show file tree
Hide file tree
Showing 11 changed files with 171 additions and 112 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -72,8 +72,8 @@ def dag_exporter_engines(self) -> str:
@property
def dag_exporter_meta(self):
return StructFormField(
name=FormField(description="dag name"),
description=FormField(description="dag description"),
("name", FormField(description="dag name")),
("description", FormField(description="dag description")),
)

@with_session
Expand Down
25 changes: 17 additions & 8 deletions querybook/server/lib/export/exporters/gspread_exporter.py
Original file line number Diff line number Diff line change
Expand Up @@ -94,16 +94,25 @@ def requires_auth(self):
@property
def export_form(self):
return StructFormField(
sheet_url=FormField(
description="Optional, if not provided a new sheet will be created."
(
"sheet_url",
FormField(
description="Optional, if not provided a new sheet will be created."
),
),
worksheet_title=FormField(
description='Defaults to "Sheet1"',
helper="Title of the worksheet, if not found then a sheet will be created",
(
"worksheet_title",
FormField(
description='Defaults to "Sheet1"',
helper="Title of the worksheet, if not found then a sheet will be created",
),
),
start_cell=FormField(
description="The top left cell position where data will be filled. Defaults to A1",
regex="^[A-Z]{1,3}[1-9][0-9]*$",
(
"start_cell",
FormField(
description="The top left cell position where data will be filled. Defaults to A1",
regex="^[A-Z]{1,3}[1-9][0-9]*$",
),
),
)

Expand Down
20 changes: 15 additions & 5 deletions querybook/server/lib/form/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -91,20 +91,29 @@ def to_dict(self):


class StructFormField(AbstractFormField):
def __init__(self, **kwargs: Dict[str, "AllFormField"]):
self.kwargs = kwargs
def __init__(
self,
*fields: list[tuple[str, "AllFormField"]],
# deprecated do not use
**kwargs: Dict[str, "AllFormField"]
):
self.fields = list(fields) + list(kwargs.items())

def to_dict(self):
return {
"field_type": CompositeFormFieldType.Struct.value,
"fields": {key: value.to_dict() for key, value in self.kwargs.items()},
"fields": [(field[0], field[1].to_dict()) for field in self.fields],
}

@property
def dict_fields(self):
return dict(self.fields)


AllFormField = Union[FormField, ExpandableFormField, StructFormField]


def validate_form(form: AllFormField, form_value) -> [bool, str]:
def validate_form(form: AllFormField, form_value) -> tuple[bool, str]:
"""Checks if the form is valid
Arguments:
Expand All @@ -117,7 +126,8 @@ def validate_form(form: AllFormField, form_value) -> [bool, str]:
if isinstance(form, StructFormField):
if not isinstance(form_value, dict):
return False, "Field value is not a dictionary"
for key, subform in form.kwargs.items():

for key, subform in form.fields:
valid, reason = validate_form(subform, form_value.get(key, None))
if not valid:
return valid, reason
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,13 +26,16 @@ def __init__(self, metastore_dict: Dict):
@classmethod
def get_metastore_params_template(cls):
return StructFormField(
catalog_id=FormField(
required=True,
description="Enter the Glue Data Catalog ID",
regex=r"^\d{12}$",
(
"catalog_id",
FormField(
required=True,
description="Enter the Glue Data Catalog ID",
regex=r"^\d{12}$",
),
),
region=FormField(required=True, description="Enter the AWS Region"),
load_partitions=load_partitions_field,
("region", FormField(required=True, description="Enter the AWS Region")),
("load_partitions", load_partitions_field),
)

def get_all_schema_names(self) -> List[str]:
Expand Down
17 changes: 10 additions & 7 deletions querybook/server/lib/metastore/loaders/hive_metastore_loader.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,15 +27,18 @@ def __del__(self):
@classmethod
def get_metastore_params_template(cls):
return StructFormField(
hms_connection=ExpandableFormField(
of=FormField(
required=True,
description="Put url to hive metastore server here",
field_type=FormFieldType.String,
(
"hms_connection",
ExpandableFormField(
of=FormField(
required=True,
description="Put url to hive metastore server here",
field_type=FormFieldType.String,
),
min=1,
),
min=1,
),
load_partitions=load_partitions_field,
("load_partitions", load_partitions_field),
)

def get_all_schema_names(self) -> List[str]:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,31 +17,41 @@ def __init__(self, metastore_dict: Dict):
@classmethod
def get_metastore_params_template(cls):
return StructFormField(
connection_string=FormField(
required=True,
description="Put your JDBC connection string here",
regex="^(?:jdbc:)?hive2:\\/\\/([\\w.-]+(?:\\:\\d+)?(?:,[\\w.-]+(?:\\:\\d+)?)*)\\/(\\w*)((?:;[\\w.-]+=[\\w.-]+)*)(\\?[\\w.-]+=[\\w.-]+(?:;[\\w.-]+=[\\w.-]+)*)?(\\#[\\w.-]+=[\\w.-]+(?:;[\\w.-]+=[\\w.-]+)*)?$", # noqa: E501
helper="""
<p>
Format
jdbc:hive2://&lt;host1&gt;:&lt;port1&gt;,&lt;host2&gt;:&lt;port2&gt;/dbName;sess_var_list?hive_conf_list#hive_var_list
</p>
<p>Currently support zookeeper in session var, and will pass any conf variables to HS2</p>
<p>See
<a href="https://cwiki.apache.org/confluence/display/Hive/HiveServer2+Clients#HiveServer2Clients-JDBC">
here
</a> for more details.
</p>""",
(
"hive_resource_manager",
FormField(
description="Provide resource manager link here to provide insights"
),
),
(
"connection_string",
FormField(
required=True,
description="Put your JDBC connection string here",
regex="^(?:jdbc:)?hive2:\\/\\/([\\w.-]+(?:\\:\\d+)?(?:,[\\w.-]+(?:\\:\\d+)?)*)\\/(\\w*)((?:;[\\w.-]+=[\\w.-]+)*)(\\?[\\w.-]+=[\\w.-]+(?:;[\\w.-]+=[\\w.-]+)*)?(\\#[\\w.-]+=[\\w.-]+(?:;[\\w.-]+=[\\w.-]+)*)?$", # noqa: E501
helper="""
<p>
Format
jdbc:hive2://&lt;host1&gt;:&lt;port1&gt;,&lt;host2&gt;:&lt;port2&gt;/dbName;sess_var_list?hive_conf_list#hive_var_list
</p>
<p>Currently support zookeeper in session var, and will pass any conf variables to HS2</p>
<p>See [here](https://cwiki.apache.org/confluence/display/Hive/HiveServer2+Clients#HiveServer2Clients-JDBC) for more details.
</p>""",
),
),
username=FormField(regex="\\w+"),
password=FormField(hidden=True),
hms_connection=ExpandableFormField(
of=FormField(
required=True, description="Put url to hive metastore server here"
("username", FormField(regex="\\w+")),
("password", FormField(hidden=True)),
(
"hms_connection",
ExpandableFormField(
of=FormField(
required=True,
description="Put url to hive metastore server here",
),
min=1,
),
min=1,
),
load_partitions=load_partitions_field,
("load_partitions", load_partitions_field),
)

def get_table_and_columns(
Expand Down
114 changes: 71 additions & 43 deletions querybook/server/lib/query_executor/executor_template/templates.py
Original file line number Diff line number Diff line change
@@ -1,90 +1,118 @@
from lib.form import FormField, StructFormField, FormFieldType, ExpandableFormField

hive_executor_template = StructFormField(
hive_resource_manager=FormField(
description="Provide resource manager link here to provide insights"
(
"hive_resource_manager",
FormField(description="Provide resource manager link here to provide insights"),
),
connection_string=FormField(
required=True,
description="Put your JDBC connection string here",
regex="^(?:jdbc:)?hive2:\\/\\/([\\w.-]+(?:\\:\\d+)?(?:,[\\w.-]+(?:\\:\\d+)?)*)\\/(\\w*)((?:;[\\w.-]+=[\\w.-]+)*)(\\?[\\w.-]+=[\\w.-]+(?:;[\\w.-]+=[\\w.-]+)*)?(\\#[\\w.-]+=[\\w.-]+(?:;[\\w.-]+=[\\w.-]+)*)?$", # noqa: E501
helper="""
(
"connection_string",
FormField(
required=True,
description="Put your JDBC connection string here",
regex="^(?:jdbc:)?hive2:\\/\\/([\\w.-]+(?:\\:\\d+)?(?:,[\\w.-]+(?:\\:\\d+)?)*)\\/(\\w*)((?:;[\\w.-]+=[\\w.-]+)*)(\\?[\\w.-]+=[\\w.-]+(?:;[\\w.-]+=[\\w.-]+)*)?(\\#[\\w.-]+=[\\w.-]+(?:;[\\w.-]+=[\\w.-]+)*)?$", # noqa: E501
helper="""
<p>
Format
jdbc:hive2://&lt;host1&gt;:&lt;port1&gt;,&lt;host2&gt;:&lt;port2&gt;/dbName;sess_var_list?hive_conf_list#hive_var_list
</p>
<p>Currently support zookeeper in session var, and will pass any conf variables to HS2</p>
<p>See [here](https://cwiki.apache.org/confluence/display/Hive/HiveServer2+Clients#HiveServer2Clients-JDBC) for more details.
</p>""",
),
),
username=FormField(regex="\\w+"),
password=FormField(hidden=True),
impersonate=FormField(field_type=FormFieldType.Boolean),
("username", FormField(regex="\\w+")),
("password", FormField(hidden=True)),
("impersonate", FormField(field_type=FormFieldType.Boolean)),
)

presto_executor_template = StructFormField(
connection_string=FormField(
required=True,
regex="^(?:jdbc:)?presto:\\/\\/([\\w.-]+(?:\\:\\d+)?(?:,[\\w.-]+(?:\\:\\d+)?)*)(\\/\\w+)?(\\/\\w+)?(\\?[\\w]+=[^&]+(?:&[\\w]+=[^&]+)*)?$", # noqa: E501
helper="""
(
"connection_string",
FormField(
required=True,
regex="^(?:jdbc:)?presto:\\/\\/([\\w.-]+(?:\\:\\d+)?(?:,[\\w.-]+(?:\\:\\d+)?)*)(\\/\\w+)?(\\/\\w+)?(\\?[\\w]+=[^&]+(?:&[\\w]+=[^&]+)*)?$", # noqa: E501
helper="""
<p>Format jdbc:presto://&lt;host:port&gt;/&lt;catalog&gt;/&lt;schema&gt;?presto_conf_list</p>
<p>Catalog and schema are optional. We only support SSL as the conf option.</p>
<p>See [here](https://prestodb.github.io/docs/current/installation/jdbc.html) for more details.</p>""",
),
),
username=FormField(regex="\\w+"),
password=FormField(hidden=True),
impersonate=FormField(field_type=FormFieldType.Boolean),
proxy_user_id=FormField(
field_type=FormFieldType.String,
helper="""
("username", FormField(regex="\\w+")),
("password", FormField(hidden=True)),
("impersonate", FormField(field_type=FormFieldType.Boolean)),
(
"proxy_user_id",
FormField(
field_type=FormFieldType.String,
helper="""
<p>User field used as proxy_user. proxy_user will be forwaded to Presto as the session user.</p>
<p>Defaults to username. Possible values are username, email, fullname </p>
<p>See [here](https://prestodb.github.io/docs/current/installation/jdbc.html) for more details.</p>""",
),
),
)

trino_executor_template = StructFormField(
connection_string=FormField(
required=True,
regex="^(?:jdbc:)?trino:\\/\\/([\\w.-]+(?:\\:\\d+)?(?:,[\\w.-]+(?:\\:\\d+)?)*)(\\/\\w+)?(\\/\\w+)?(\\?[\\w]+=[^&]+(?:&[\\w]+=[^&]+)*)?$", # noqa: E501
helper="""
(
"connection_string",
FormField(
required=True,
regex="^(?:jdbc:)?trino:\\/\\/([\\w.-]+(?:\\:\\d+)?(?:,[\\w.-]+(?:\\:\\d+)?)*)(\\/\\w+)?(\\/\\w+)?(\\?[\\w]+=[^&]+(?:&[\\w]+=[^&]+)*)?$", # noqa: E501
helper="""
<p>Format jdbc:trino://&lt;host:port&gt;/&lt;catalog&gt;/&lt;schema&gt;?trino_conf_list</p>
<p>Catalog and schema are optional. We only support SSL as the conf option.</p>
<p>See [here](https://trino.io/docs/current/installation/jdbc.html) for more details.</p>""",
),
),
username=FormField(required=True, regex="\\w+"),
impersonate=FormField(field_type=FormFieldType.Boolean),
proxy_user_id=FormField(
field_type=FormFieldType.String,
helper="""
("username", FormField(regex="\\w+")),
("impersonate", FormField(field_type=FormFieldType.Boolean)),
(
"proxy_user_id",
FormField(
field_type=FormFieldType.String,
helper="""
<p>User field used as proxy_user. proxy_user will be forwaded to Trino as the session user.</p>
<p>Defaults to username. Possible values are username, email, fullname </p>
<p>See [here](https://trino.io/docs/current/installation/jdbc.html) for more details.</p>""",
),
),
)

sqlalchemy_template = StructFormField(
connection_string=FormField(
required=True,
helper="""
(
"connection_string",
FormField(
required=True,
helper="""
<p>
See [here](https://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls) for more details.
</p>""",
),
),
connect_args=ExpandableFormField(
of=StructFormField(
key=FormField(required=True),
value=FormField(required=True),
isJson=FormField(
field_type=FormFieldType.Boolean,
helper="If true, then the value will be parsed as JSON",
),
)
(
"connect_args",
ExpandableFormField(
of=StructFormField(
("key", FormField(required=True)),
("value", FormField(required=True)),
(
"isJson",
FormField(
field_type=FormFieldType.Boolean,
helper="If true, then the value will be parsed as JSON",
),
),
)
),
),
)

bigquery_template = StructFormField(
google_credentials_json=FormField(
helper="The JSON string used to log in as service account. If not provided then **GOOGLE_CREDS** from settings will be used.",
(
"google_credentials_json",
FormField(
helper="The JSON string used to log in as service account. If not provided then **GOOGLE_CREDS** from settings will be used.",
),
)
)
4 changes: 2 additions & 2 deletions querybook/tests/test_lib/test_form/test__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -85,8 +85,8 @@ def test_array_field(self):

def test_dict_field(self):
form = StructFormField(
name=FormField(),
phone_numbers=ExpandableFormField(of=FormField(), min=1, max=2),
("name", FormField()),
("phone_numbers", ExpandableFormField(of=FormField(), min=1, max=2)),
)
self.assertEqual(
validate_form(form, "123"),
Expand Down
Loading

0 comments on commit 4a7c8da

Please sign in to comment.