From c9b0b63e33c071e84cd3d170706484a42765a8ba Mon Sep 17 00:00:00 2001 From: khtruong Date: Mon, 6 May 2019 17:04:08 -0700 Subject: [PATCH 1/5] fix: alter sql columns to long text --- ...a_update_sql_column_data_type_in_query_.py | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 superset/migrations/versions/afc69274c25a_update_sql_column_data_type_in_query_.py diff --git a/superset/migrations/versions/afc69274c25a_update_sql_column_data_type_in_query_.py b/superset/migrations/versions/afc69274c25a_update_sql_column_data_type_in_query_.py new file mode 100644 index 0000000000000..8db193e25473c --- /dev/null +++ b/superset/migrations/versions/afc69274c25a_update_sql_column_data_type_in_query_.py @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""update the sql, select_sql, and executed_sql columns in the + query table to support larger text + +Revision ID: afc69274c25a +Revises: e9df189e5c7e +Create Date: 2019-05-06 14:30:26.181449 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = 'afc69274c25a' +down_revision = 'e9df189e5c7e' + + +def upgrade(): + with op.batch_alter_table('query') as batch_op: + batch_op.alter_column( + 'sql', existing_type=sa.Text, type_=sa.Text(2 ** 32 - 1)) + batch_op.alter_column( + 'select_sql', existing_type=sa.Text, type_=sa.Text(2 ** 32 - 1)) + batch_op.alter_column( + 'executed_sql', existing_type=sa.Text, type_=sa.Text(2 ** 32 - 1)) + + +def downgrade(): + with op.batch_alter_table('query') as batch_op: + batch_op.alter_column( + 'sql', existing_type=sa.Text(2 ** 32 - 1), type_=sa.Text) + batch_op.alter_column( + 'select_sql', existing_type=sa.Text(2 ** 32 - 1), type_=sa.Text) + batch_op.alter_column( + 'executed_sql', existing_type=sa.Text(2 ** 32 - 1), type_=sa.Text) From 52c8395c641ca62f434e711acaece3e0a6397442 Mon Sep 17 00:00:00 2001 From: khtruong Date: Tue, 7 May 2019 10:40:21 -0700 Subject: [PATCH 2/5] fix: handle dbs that don't require text length --- ...a_update_sql_column_data_type_in_query_.py | 39 ++++++++++++------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/superset/migrations/versions/afc69274c25a_update_sql_column_data_type_in_query_.py b/superset/migrations/versions/afc69274c25a_update_sql_column_data_type_in_query_.py index 8db193e25473c..20a03768015eb 100644 --- a/superset/migrations/versions/afc69274c25a_update_sql_column_data_type_in_query_.py +++ b/superset/migrations/versions/afc69274c25a_update_sql_column_data_type_in_query_.py @@ -29,22 +29,33 @@ revision = 'afc69274c25a' down_revision = 'e9df189e5c7e' +text_len = 2 ** 32 - 1 + def upgrade(): - with op.batch_alter_table('query') as batch_op: - batch_op.alter_column( - 'sql', existing_type=sa.Text, type_=sa.Text(2 ** 32 - 1)) - batch_op.alter_column( - 'select_sql', existing_type=sa.Text, type_=sa.Text(2 ** 32 - 1)) - batch_op.alter_column( - 'executed_sql', existing_type=sa.Text, type_=sa.Text(2 ** 32 - 1)) + try: + # Set text length if database accepts it + with op.batch_alter_table('query') as batch_op: + batch_op.alter_column( + 'sql', existing_type=sa.Text, type_=sa.Text(length=text_len)) + batch_op.alter_column( + 'select_sql', existing_type=sa.Text, type_=sa.Text(length=text_len)) + batch_op.alter_column( + 'executed_sql', existing_type=sa.Text, type_=sa.Text(length=text_len)) + except: + # Many databases do not have a length on text objects + # so skip altering for those databases + pass def downgrade(): - with op.batch_alter_table('query') as batch_op: - batch_op.alter_column( - 'sql', existing_type=sa.Text(2 ** 32 - 1), type_=sa.Text) - batch_op.alter_column( - 'select_sql', existing_type=sa.Text(2 ** 32 - 1), type_=sa.Text) - batch_op.alter_column( - 'executed_sql', existing_type=sa.Text(2 ** 32 - 1), type_=sa.Text) + try: + with op.batch_alter_table('query') as batch_op: + batch_op.alter_column( + 'sql', existing_type=sa.Text(length=text_len), type_=sa.Text) + batch_op.alter_column( + 'select_sql', existing_type=sa.Text(length=text_len), type_=sa.Text) + batch_op.alter_column( + 'executed_sql', existing_type=sa.Text(length=text_len), type_=sa.Text) + except: + pass From da0944e9a89302336fb0d5e11f1d295de5a1b4b1 Mon Sep 17 00:00:00 2001 From: khtruong Date: Tue, 7 May 2019 11:27:07 -0700 Subject: [PATCH 3/5] fix: 1GB limit --- .../afc69274c25a_update_sql_column_data_type_in_query_.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/superset/migrations/versions/afc69274c25a_update_sql_column_data_type_in_query_.py b/superset/migrations/versions/afc69274c25a_update_sql_column_data_type_in_query_.py index 20a03768015eb..4f7698ed6795b 100644 --- a/superset/migrations/versions/afc69274c25a_update_sql_column_data_type_in_query_.py +++ b/superset/migrations/versions/afc69274c25a_update_sql_column_data_type_in_query_.py @@ -29,7 +29,7 @@ revision = 'afc69274c25a' down_revision = 'e9df189e5c7e' -text_len = 2 ** 32 - 1 +text_len = 10 ** 9 - 1 def upgrade(): From 1439c8a77a3f233d1ceb5c6be548232fb88ed3f8 Mon Sep 17 00:00:00 2001 From: khtruong Date: Tue, 7 May 2019 15:25:16 -0700 Subject: [PATCH 4/5] fix: only handle mysql --- ..._column_data_type_in_query_mysql_table.py} | 31 ++++++++----------- 1 file changed, 13 insertions(+), 18 deletions(-) rename superset/migrations/versions/{afc69274c25a_update_sql_column_data_type_in_query_.py => afc69274c25a_alter_sql_column_data_type_in_query_mysql_table.py} (64%) diff --git a/superset/migrations/versions/afc69274c25a_update_sql_column_data_type_in_query_.py b/superset/migrations/versions/afc69274c25a_alter_sql_column_data_type_in_query_mysql_table.py similarity index 64% rename from superset/migrations/versions/afc69274c25a_update_sql_column_data_type_in_query_.py rename to superset/migrations/versions/afc69274c25a_alter_sql_column_data_type_in_query_mysql_table.py index 4f7698ed6795b..4f443e7a99dcb 100644 --- a/superset/migrations/versions/afc69274c25a_update_sql_column_data_type_in_query_.py +++ b/superset/migrations/versions/afc69274c25a_alter_sql_column_data_type_in_query_mysql_table.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. """update the sql, select_sql, and executed_sql columns in the - query table to support larger text + query table in mysql dbs to be long text columns Revision ID: afc69274c25a Revises: e9df189e5c7e @@ -23,39 +23,34 @@ """ from alembic import op +from sqlalchemy.databases import mysql +from sqlalchemy.dialects.mysql.base import MySQLDialect import sqlalchemy as sa # revision identifiers, used by Alembic. revision = 'afc69274c25a' down_revision = 'e9df189e5c7e' -text_len = 10 ** 9 - 1 - def upgrade(): - try: - # Set text length if database accepts it + bind = op.get_bind() + if isinstance(bind.dialect, MySQLDialect): with op.batch_alter_table('query') as batch_op: batch_op.alter_column( - 'sql', existing_type=sa.Text, type_=sa.Text(length=text_len)) + 'sql', existing_type=sa.Text, type_=mysql.LONGTEXT) batch_op.alter_column( - 'select_sql', existing_type=sa.Text, type_=sa.Text(length=text_len)) + 'select_sql', existing_type=sa.Text, type_=mysql.LONGTEXT) batch_op.alter_column( - 'executed_sql', existing_type=sa.Text, type_=sa.Text(length=text_len)) - except: - # Many databases do not have a length on text objects - # so skip altering for those databases - pass + 'executed_sql', existing_type=sa.Text, type_=mysql.LONGTEXT) def downgrade(): - try: + bind = op.get_bind() + if isinstance(bind.dialect, MySQLDialect): with op.batch_alter_table('query') as batch_op: batch_op.alter_column( - 'sql', existing_type=sa.Text(length=text_len), type_=sa.Text) + 'sql', existing_type=mysql.LONGTEXT, type_=sa.Text) batch_op.alter_column( - 'select_sql', existing_type=sa.Text(length=text_len), type_=sa.Text) + 'select_sql', existing_type=mysql.LONGTEXT, type_=sa.Text) batch_op.alter_column( - 'executed_sql', existing_type=sa.Text(length=text_len), type_=sa.Text) - except: - pass + 'executed_sql', existing_type=mysql.LONGTEXT, type_=sa.Text) From 35d41f5bc504482d4184bb3bd79bea9a8995c520 Mon Sep 17 00:00:00 2001 From: khtruong Date: Tue, 7 May 2019 15:57:25 -0700 Subject: [PATCH 5/5] Empty Commit