Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: Unable to parse escaped tables #30560

Merged
merged 2 commits into from
Oct 9, 2024
Merged

fix: Unable to parse escaped tables #30560

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7494de5
fix: Unable to parse escaped tables
michael-s-molina Oct 9, 2024
81bcbf5
Move import
michael-s-molina Oct 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions superset/config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@

if TYPE_CHECKING:
from flask_appbuilder.security.sqla import models
from sqlglot import Dialect, Dialects

Check warning on line 71 in superset/config.py

View check run for this annotation

Codecov / codecov/patch

superset/config.py#L71 

Added line #L71 was not covered by tests

from superset.connectors.sqla.models import SqlaTable
from superset.models.core import Database
Expand Down Expand Up @@ -249,6 +250,10 @@
SQLALCHEMY_ENCRYPTED_FIELD_TYPE_ADAPTER = ( # pylint: disable=invalid-name
SQLAlchemyUtilsAdapter
)

# Extends the default SQLGlot dialects with additional dialects
SQLGLOT_DIALECTS_EXTENSIONS: map[str, Dialects | type[Dialect]] = {}

# The limit of queries fetched for query search
QUERY_SEARCH_LIMIT = 1000

Expand Down
5 changes: 5 additions & 0 deletions superset/initialization/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@
talisman,
)
from superset.security import SupersetSecurityManager
from superset.sql.parse import SQLGLOT_DIALECTS
from superset.superset_typing import FlaskResponse
from superset.tags.core import register_sqla_event_listeners
from superset.utils.core import is_test, pessimistic_connection_handling
Expand Down Expand Up @@ -484,6 +485,7 @@ def init_app(self) -> None:
self.configure_middlewares()
self.configure_cache()
self.set_db_default_isolation()
self.configure_sqlglot_dialects()

with self.superset_app.app_context():
self.init_app_in_ctx()
Expand Down Expand Up @@ -544,6 +546,9 @@ def configure_cache(self) -> None:
def configure_feature_flags(self) -> None:
feature_flag_manager.init_app(self.superset_app)

def configure_sqlglot_dialects(self) -> None:
SQLGLOT_DIALECTS.update(self.config["SQLGLOT_DIALECTS_EXTENSIONS"])

@transaction()
def configure_fab(self) -> None:
if self.config["SILENCE_FAB"]:
Expand Down
63 changes: 6 additions & 57 deletions superset/sql_parse.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@
from flask_babel import gettext as __
from jinja2 import nodes
from sqlalchemy import and_
from sqlglot.dialects.dialect import Dialects
from sqlparse import keywords
from sqlparse.lexer import Lexer
from sqlparse.sql import (
Expand Down Expand Up @@ -61,7 +60,12 @@
SupersetParseError,
SupersetSecurityException,
)
from superset.sql.parse import extract_tables_from_statement, SQLScript, Table
from superset.sql.parse import (
extract_tables_from_statement,
SQLGLOT_DIALECTS,
SQLScript,
Table,
)
from superset.utils.backports import StrEnum

try:
Expand All @@ -88,61 +92,6 @@
lex.set_SQL_REGEX(sqlparser_sql_regex)


# mapping between DB engine specs and sqlglot dialects
michael-s-molina marked this conversation as resolved.
Show resolved Hide resolved
SQLGLOT_DIALECTS = {
"ascend": Dialects.HIVE,
"awsathena": Dialects.PRESTO,
"bigquery": Dialects.BIGQUERY,
"clickhouse": Dialects.CLICKHOUSE,
"clickhousedb": Dialects.CLICKHOUSE,
"cockroachdb": Dialects.POSTGRES,
"couchbase": Dialects.MYSQL,
# "crate": ???
# "databend": ???
"databricks": Dialects.DATABRICKS,
# "db2": ???
# "dremio": ???
"drill": Dialects.DRILL,
# "druid": ???
"duckdb": Dialects.DUCKDB,
# "dynamodb": ???
# "elasticsearch": ???
# "exa": ???
# "firebird": ???
# "firebolt": ???
"gsheets": Dialects.SQLITE,
"hana": Dialects.POSTGRES,
"hive": Dialects.HIVE,
# "ibmi": ???
# "impala": ???
# "kustokql": ???
# "kylin": ???
"mssql": Dialects.TSQL,
"mysql": Dialects.MYSQL,
"netezza": Dialects.POSTGRES,
# "ocient": ???
# "odelasticsearch": ???
"oracle": Dialects.ORACLE,
# "pinot": ???
"postgresql": Dialects.POSTGRES,
"presto": Dialects.PRESTO,
"pydoris": Dialects.DORIS,
"redshift": Dialects.REDSHIFT,
# "risingwave": ???
# "rockset": ???
"shillelagh": Dialects.SQLITE,
"snowflake": Dialects.SNOWFLAKE,
# "solr": ???
"spark": Dialects.SPARK,
"sqlite": Dialects.SQLITE,
"starrocks": Dialects.STARROCKS,
"superset": Dialects.SQLITE,
"teradatasql": Dialects.TERADATA,
"trino": Dialects.TRINO,
"vertica": Dialects.POSTGRES,
}


class CtasMethod(StrEnum):
TABLE = "TABLE"
VIEW = "VIEW"
Expand Down
13 changes: 13 additions & 0 deletions tests/unit_tests/sql/parse_tests.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@


import pytest
from sqlglot import Dialects

from superset.exceptions import SupersetParseError
from superset.sql.parse import (
Expand Down Expand Up @@ -932,3 +933,15 @@ def test_get_settings() -> None:
SELECT * FROM some_table;
"""
assert SQLScript(sql, "postgresql").get_settings() == {"search_path": "bar"}


@pytest.mark.parametrize(
"app",
[{"SQLGLOT_DIALECTS_EXTENSIONS": {"custom": Dialects.MYSQL}}],
indirect=True,
)
def test_custom_dialect(app: None) -> None:
"""
Test that custom dialects are loaded correctly.
"""
assert SQLGLOT_DIALECTS.get("custom") == Dialects.MYSQL
Loading