From bb96a8d289c73bba769f7c4f3ab3ed8dd7abbf9e Mon Sep 17 00:00:00 2001 From: Daniel Gaspar Date: Thu, 18 Apr 2019 14:08:53 +0100 Subject: [PATCH 1/8] Bump FAB to 2.0.0 --- docs/faq.rst | 4 ++-- docs/installation.rst | 3 ++- requirements.txt | 17 +++++++++++++---- setup.py | 2 +- 4 files changed, 18 insertions(+), 8 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index d1f781f4a4172..426e0ab13b2d9 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -186,8 +186,8 @@ by setting the ``stagger_refresh`` to ``false`` and modify the stagger period by Here, the entire dashboard will refresh at once if periodic refresh is on. The stagger time of 2.5 seconds is ignored. -Why does fabmanager or superset freezed/hung/not responding when started (my home directory is NFS mounted)? ------------------------------------------------------------------------------------------------------------- +Why does 'flask fab' or superset freezed/hung/not responding when started (my home directory is NFS mounted)? +------------------------------------------------------------------------------------------------------------- By default, superset creates and uses an sqlite database at ``~/.superset/superset.db``. Sqlite is known to `don't work well if used on NFS`__ due to broken file locking implementation on NFS. __ https://www.sqlite.org/lockingv3.html diff --git a/docs/installation.rst b/docs/installation.rst index 7bc9f61f3ea60..0c6fee1abd86b 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -171,7 +171,8 @@ Follow these few simple steps to install Superset.:: pip install superset # Create an admin user (you will be prompted to set a username, first and last name before setting a password) - fabmanager create-admin --app superset + $ export FLASK_APP=superset + flask fab create-admin # Initialize the database superset db upgrade diff --git a/requirements.txt b/requirements.txt index 4b6b4f56edcab..f35a8fa251aad 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,9 @@ # alembic==1.0.0 # via flask-migrate amqp==2.3.2 # via kombu +apispec[yaml]==1.2.0 # via flask-appbuilder asn1crypto==0.24.0 # via cryptography +attrs==19.1.0 # via jsonschema babel==2.6.0 # via flask-babel billiard==3.5.0.4 # via celery bleach==3.0.2 @@ -21,10 +23,11 @@ croniter==0.3.29 cryptography==2.4.2 decorator==4.3.0 # via retry defusedxml==0.5.0 # via python3-openid -flask-appbuilder==1.12.5 +flask-appbuilder==2.0.0 flask-babel==0.11.1 # via flask-appbuilder flask-caching==1.4.0 flask-compress==1.4.0 +flask-jwt-extended==3.18.1 # via flask-appbuilder flask-login==0.4.1 # via flask-appbuilder flask-migrate==2.1.1 flask-openid==1.2.5 # via flask-appbuilder @@ -38,19 +41,25 @@ idna==2.6 isodate==0.6.0 itsdangerous==0.24 # via flask jinja2==2.10 # via flask, flask-babel +jsonschema==3.0.1 # via flask-appbuilder kombu==4.2.1 # via celery mako==1.0.7 # via alembic markdown==3.0 markupsafe==1.0 # via jinja2, mako +marshmallow-enum==1.4.1 # via flask-appbuilder +marshmallow-sqlalchemy==0.16.2 # via flask-appbuilder +marshmallow==2.19.2 # via flask-appbuilder, marshmallow-enum, marshmallow-sqlalchemy numpy==1.15.2 # via pandas pandas==0.23.4 parsedatetime==2.0.0 pathlib2==2.3.0 polyline==1.3.2 +prison==0.1.0 # via flask-appbuilder py==1.7.0 # via retry pycparser==2.19 # via cffi pydruid==0.5.0 -pyjwt==1.7.1 # via flask-appbuilder +pyjwt==1.7.1 # via flask-appbuilder, flask-jwt-extended +pyrsistent==0.14.11 # via jsonschema python-dateutil==2.6.1 python-editor==1.0.3 # via alembic python-geohash==0.8.5 @@ -61,7 +70,7 @@ requests==2.20.0 retry==0.9.2 selenium==3.141.0 simplejson==3.15.0 -six==1.11.0 # via bleach, cryptography, isodate, pathlib2, polyline, pydruid, python-dateutil, sqlalchemy-utils, wtforms-json +six==1.11.0 # via bleach, cryptography, flask-jwt-extended, isodate, jsonschema, pathlib2, polyline, prison, pydruid, pyrsistent, python-dateutil, sqlalchemy-utils, wtforms-json sqlalchemy-utils==0.32.21 sqlalchemy==1.3.1 sqlparse==0.2.4 @@ -69,6 +78,6 @@ unicodecsv==0.14.1 urllib3==1.22 # via requests, selenium vine==1.1.4 # via amqp webencodings==0.5.1 # via bleach -werkzeug==0.14.1 # via flask +werkzeug==0.14.1 # via flask, flask-jwt-extended wtforms-json==0.3.3 wtforms==2.2.1 # via flask-wtf, wtforms-json diff --git a/setup.py b/setup.py index 9c6278e75ce74..b97b49ad5b114 100644 --- a/setup.py +++ b/setup.py @@ -77,7 +77,7 @@ def get_git_sha(): 'croniter>=0.3.28', 'cryptography>=2.4.2', 'flask>=1.0.0, <2.0.0', - 'flask-appbuilder>=1.12.5, <2.0.0', + 'flask-appbuilder>=2.0.0, <2.3.0', 'flask-caching', 'flask-compress', 'flask-migrate', From c9c2dd06ef8cc7aca0a55568d47830eb63b09f6a Mon Sep 17 00:00:00 2001 From: Daniel Gaspar Date: Thu, 18 Apr 2019 15:14:26 +0100 Subject: [PATCH 2/8] [tests] whitelist SecurityApi login and refresh endpoints --- tests/security_tests.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/security_tests.py b/tests/security_tests.py index 063f1e8abf98b..4bca1a3b5decb 100644 --- a/tests/security_tests.py +++ b/tests/security_tests.py @@ -247,6 +247,8 @@ def test_views_are_secured(self): ['Superset', 'log'], ['Superset', 'theme'], ['Superset', 'welcome'], + ['SecurityApi', 'login'], + ['SecurityApi', 'refresh'] ] unsecured_views = [] for view_class in appbuilder.baseviews: From 21fa309c183f59887e5896b8b9613f7a583b27c8 Mon Sep 17 00:00:00 2001 From: Daniel Gaspar Date: Thu, 18 Apr 2019 15:51:28 +0100 Subject: [PATCH 3/8] [style] Fix, C812 missing trailing commas --- tests/security_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/security_tests.py b/tests/security_tests.py index 4bca1a3b5decb..466761177925a 100644 --- a/tests/security_tests.py +++ b/tests/security_tests.py @@ -248,7 +248,7 @@ def test_views_are_secured(self): ['Superset', 'theme'], ['Superset', 'welcome'], ['SecurityApi', 'login'], - ['SecurityApi', 'refresh'] + ['SecurityApi', 'refresh'], ] unsecured_views = [] for view_class in appbuilder.baseviews: From e2eec2fce9877dddfa9bbccaf686d28da9066b71 Mon Sep 17 00:00:00 2001 From: Daniel Gaspar Date: Fri, 19 Apr 2019 12:57:25 +0100 Subject: [PATCH 4/8] [security] Remove SUPERSET_UPDATE_PERMS flag Registering sources needs to be performed after the views are initialized on UPDATE_PERMS=False configuration --- superset/__init__.py | 14 ++++++-------- superset/utils/core.py | 5 ----- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/superset/__init__.py b/superset/__init__.py index 7d0df26e039e7..c405c01de72a1 100644 --- a/superset/__init__.py +++ b/superset/__init__.py @@ -34,8 +34,7 @@ from superset import config from superset.connectors.connector_registry import ConnectorRegistry from superset.security import SupersetSecurityManager -from superset.utils.core import ( - get_update_perms_flag, pessimistic_connection_handling, setup_cache) +from superset.utils.core import pessimistic_connection_handling, setup_cache wtforms_json.init() @@ -202,7 +201,6 @@ def index(self): base_template='superset/base.html', indexview=MyIndexView, security_manager_class=custom_sm, - update_perms=get_update_perms_flag(), ) security_manager = appbuilder.sm @@ -226,11 +224,6 @@ def is_feature_enabled(feature): return get_feature_flags().get(feature) -# Registering sources -module_datasource_map = app.config.get('DEFAULT_MODULE_DS_MAP') -module_datasource_map.update(app.config.get('ADDITIONAL_MODULE_DS_MAP')) -ConnectorRegistry.register_sources(module_datasource_map) - # Flask-Compress if conf.get('ENABLE_FLASK_COMPRESS'): Compress(app) @@ -242,3 +235,8 @@ def is_feature_enabled(feature): flask_app_mutator(app) from superset import views # noqa + +# Registering sources +module_datasource_map = app.config.get('DEFAULT_MODULE_DS_MAP') +module_datasource_map.update(app.config.get('ADDITIONAL_MODULE_DS_MAP')) +ConnectorRegistry.register_sources(module_datasource_map) diff --git a/superset/utils/core.py b/superset/utils/core.py index be8bc08b3e7b0..6230b6f39900b 100644 --- a/superset/utils/core.py +++ b/superset/utils/core.py @@ -846,11 +846,6 @@ def merge_request_params(form_data, params): form_data['url_params'] = url_params -def get_update_perms_flag(): - val = os.environ.get('SUPERSET_UPDATE_PERMS') - return val.lower() not in ('0', 'false', 'no') if val else True - - def user_label(user): """Given a user ORM FAB object, returns a label""" if user: From 19aa5e28f0b3aa070daea8299a5d8a69763eab13 Mon Sep 17 00:00:00 2001 From: Daniel Gaspar Date: Fri, 19 Apr 2019 12:58:19 +0100 Subject: [PATCH 5/8] [docs] New, FAB_UPDATE_PERMS and flask fab cli --- UPDATING.md | 5 +++++ docs/installation.rst | 8 +------- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/UPDATING.md b/UPDATING.md index bec2d4204bb43..e669d39a96ec4 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -21,6 +21,11 @@ under the License. This file documents any backwards-incompatible changes in Superset and assists people when migrating to a new version. +## Superset 0.33.0 + +* `fabmanager` command line is deprecated since Flask-AppBuilder 2.0.0, use +the new `flask fab ` integrated with *Flask cli*. + ## Superset 0.32.0 * `npm run backend-sync` is deprecated and no longer needed, will fail if called diff --git a/docs/installation.rst b/docs/installation.rst index 0c6fee1abd86b..ff8603ace5b02 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -237,17 +237,11 @@ workers this creates a lot of contention and race conditions when defining permissions and views. To alleviate this issue, the automatic updating of permissions can be disabled -by setting the environment variable -`SUPERSET_UPDATE_PERMS` environment variable to `0`. -The value `1` enables it, `0` disables it. Note if undefined the functionality -is enabled to maintain backwards compatibility. +by setting `FAB_UPDATE_PERMS = False` (defaults to True). In a production environment initialization could take on the following form: - export SUPERSET_UPDATE_PERMS=1 superset init - - export SUPERSET_UPDATE_PERMS=0 gunicorn -w 10 ... superset:app Configuration behind a load balancer From 87523c465b261c6dc42f8cb0f07e3f85edb97d05 Mon Sep 17 00:00:00 2001 From: Daniel Gaspar Date: Fri, 19 Apr 2019 13:58:32 +0100 Subject: [PATCH 6/8] [docs] Fix, db upgrade needs to come first, create-admin needs a db --- docs/installation.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/installation.rst b/docs/installation.rst index 00c3ac22759e4..1f406c956fe6b 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -170,13 +170,13 @@ Follow these few simple steps to install Superset.:: # Install superset pip install superset + # Initialize the database + superset db upgrade + # Create an admin user (you will be prompted to set a username, first and last name before setting a password) $ export FLASK_APP=superset flask fab create-admin - # Initialize the database - superset db upgrade - # Load some data to play with superset load_examples @@ -184,7 +184,7 @@ Follow these few simple steps to install Superset.:: superset init # To start a development web server on port 8088, use -p to bind to another port - superset runserver -d + flask run -p 8080 --with-threads --reload --debugger After installation, you should be able to point your browser to the right From 6b34e1234eb6eb8c72bc77db0107df95dea55759 Mon Sep 17 00:00:00 2001 From: Daniel Gaspar Date: Tue, 23 Apr 2019 10:07:56 +0100 Subject: [PATCH 7/8] [cli] New, superset init bootstraps all permissions for FAB and Superset --- UPDATING.md | 3 +++ superset/cli.py | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/UPDATING.md b/UPDATING.md index e669d39a96ec4..0b5126e1425ca 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -25,6 +25,9 @@ assists people when migrating to a new version. * `fabmanager` command line is deprecated since Flask-AppBuilder 2.0.0, use the new `flask fab ` integrated with *Flask cli*. +* `SUPERSET_UPDATE_PERMS` environment variable was replaced by +`FAB_UPDATE_PERMS` config boolean key. To disable automatic +creation of permissions set `FAB_UPDATE_PERMS = False` on config. ## Superset 0.32.0 diff --git a/superset/cli.py b/superset/cli.py index 7f5fe1773f879..c097eb338dfdf 100755 --- a/superset/cli.py +++ b/superset/cli.py @@ -28,7 +28,7 @@ import yaml from superset import ( - app, data, db, security_manager, + app, data, db, security_manager, appbuilder ) from superset.utils import ( core as utils, dashboard_import_export, dict_import_export) @@ -50,6 +50,7 @@ def make_shell_context(): def init(): """Inits the Superset application""" utils.get_or_create_main_db() + appbuilder.add_permissions(update_perms=True) security_manager.sync_role_definitions() From e0c0d3eb0f5a9a35cf1c425eb5a13e982205729f Mon Sep 17 00:00:00 2001 From: Daniel Gaspar Date: Tue, 23 Apr 2019 16:44:27 +0100 Subject: [PATCH 8/8] [style] Fix, flakes --- superset/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/superset/cli.py b/superset/cli.py index c097eb338dfdf..1aff4bd44da4d 100755 --- a/superset/cli.py +++ b/superset/cli.py @@ -28,7 +28,7 @@ import yaml from superset import ( - app, data, db, security_manager, appbuilder + app, appbuilder, data, db, security_manager, ) from superset.utils import ( core as utils, dashboard_import_export, dict_import_export)