From 7b944c7b219182c7490dd21e69601e12494de811 Mon Sep 17 00:00:00 2001 From: Jannis Leidel Date: Mon, 18 Mar 2019 10:18:12 +0100 Subject: [PATCH 1/9] Split redash/__init__.py to prevent import time side-effects. Refs #3569 and #3466. --- redash/__init__.py | 88 +++++++------------------------------ redash/app.py | 59 +++++++++++++++++++++++++ redash/cli/__init__.py | 13 +++--- redash/metrics/celery.py | 5 --- redash/settings/__init__.py | 14 +----- redash/tasks/queries.py | 2 +- redash/utils/routes.py | 15 +++++++ redash/worker.py | 13 ++++-- redash/wsgi.py | 2 +- tests/__init__.py | 4 +- 10 files changed, 113 insertions(+), 102 deletions(-) create mode 100644 redash/app.py create mode 100644 redash/utils/routes.py diff --git a/redash/__init__.py b/redash/__init__.py index cc50844c89..4856d214bb 100644 --- a/redash/__init__.py +++ b/redash/__init__.py @@ -1,27 +1,29 @@ -import sys import logging -import urlparse +import os +import sys import urllib +import urlparse import redis +<<<<<<< HEAD from flask import Flask, current_app from werkzeug.contrib.fixers import ProxyFix from werkzeug.routing import BaseConverter from statsd import StatsClient +======= +>>>>>>> Split redash/__init__.py to prevent import time side-effects. from flask_mail import Mail from flask_limiter import Limiter from flask_limiter.util import get_ipaddr from flask_migrate import Migrate +from statsd import StatsClient +from werkzeug.local import LocalProxy -from redash import settings -from redash.query_runner import import_query_runners -from redash.destinations import import_destinations - +from . import settings __version__ = '7.0.0' -import os if os.environ.get("REMOTE_DEBUG"): import ptvsd ptvsd.enable_attach(address=('0.0.0.0', 5678)) @@ -36,10 +38,8 @@ def setup_logging(): # Make noisy libraries less noisy if settings.LOG_LEVEL != "DEBUG": - logging.getLogger("passlib").setLevel("ERROR") - logging.getLogger("requests.packages.urllib3").setLevel("ERROR") - logging.getLogger("snowflake.connector").setLevel("ERROR") - logging.getLogger('apiclient').setLevel("ERROR") + for name in ["passlib", "requests.packages.urllib3", "snowflake.connector", "apiclient"]: + logging.getLogger(name).setLevel("ERROR") def create_redis_connection(): @@ -67,69 +67,13 @@ def create_redis_connection(): setup_logging() -redis_connection = create_redis_connection() - -mail = Mail() -migrate = Migrate() -mail.init_mail(settings.all_settings()) -statsd_client = StatsClient(host=settings.STATSD_HOST, port=settings.STATSD_PORT, prefix=settings.STATSD_PREFIX) -limiter = Limiter(key_func=get_ipaddr, storage_uri=settings.LIMITER_STORAGE) -import_query_runners(settings.QUERY_RUNNERS) -import_destinations(settings.DESTINATIONS) +redis_connection = LocalProxy(create_redis_connection) -from redash.version_check import reset_new_version_status -reset_new_version_status() +mail = LocalProxy(Mail) +migrate = LocalProxy(Migrate) -class SlugConverter(BaseConverter): - def to_python(self, value): - # This is ay workaround for when we enable multi-org and some files are being called by the index rule: - # for path in settings.STATIC_ASSETS_PATHS: - # full_path = safe_join(path, value) - # if os.path.isfile(full_path): - # raise ValidationError() - - return value - - def to_url(self, value): - return value - - -def create_app(): - from redash import authentication, extensions, handlers, security - from redash.handlers.webpack import configure_webpack - from redash.handlers import chrome_logger - from redash.models import db, users - from redash.metrics import request as request_metrics - from redash.utils import sentry - - sentry.init() - - app = Flask(__name__, - template_folder=settings.STATIC_ASSETS_PATH, - static_folder=settings.STATIC_ASSETS_PATH, - static_url_path='/static') - - # Make sure we get the right referral address even behind proxies like nginx. - app.wsgi_app = ProxyFix(app.wsgi_app, settings.PROXIES_COUNT) - app.url_map.converters['org_slug'] = SlugConverter - - # configure our database - app.config['SQLALCHEMY_DATABASE_URI'] = settings.SQLALCHEMY_DATABASE_URI - app.config.update(settings.all_settings()) - - security.init_app(app) - request_metrics.init_app(app) - db.init_app(app) - migrate.init_app(app, db) - mail.init_app(app) - authentication.init_app(app) - limiter.init_app(app) - handlers.init_app(app) - configure_webpack(app) - extensions.init_app(app) - chrome_logger.init_app(app) - users.init_app(app) +statsd_client = StatsClient(host=settings.STATSD_HOST, port=settings.STATSD_PORT, prefix=settings.STATSD_PREFIX) - return app +limiter = Limiter(key_func=get_ipaddr, storage_uri=settings.LIMITER_STORAGE) diff --git a/redash/app.py b/redash/app.py new file mode 100644 index 0000000000..f2456e5394 --- /dev/null +++ b/redash/app.py @@ -0,0 +1,59 @@ +from flask import Flask +from werkzeug.contrib.fixers import ProxyFix + +from . import mail, settings +from .utils import routes + + +class Redash(Flask): + """A custom Flask app for Redash""" + def __init__(self, *args, **kwargs): + kwargs.update({ + 'template_folder': settings.STATIC_ASSETS_PATH, + 'static_folder': settings.STATIC_ASSETS_PATH, + 'static_path': '/static', + }) + super(Redash, self).__init__(__name__, *args, **kwargs) + # Make sure we get the right referral address even behind proxies like nginx. + self.wsgi_app = ProxyFix(self.wsgi_app, settings.PROXIES_COUNT) + # Handle the special case of the org slug + self.url_map.converters['org_slug'] = routes.SlugConverter + # Configure Redash using our settings + self.config.from_object('redash.settings') + + +def create_app(): + from . import authentication, extensions, handlers, limiter, migrate, security + from .destinations import import_destinations + from .handlers import chrome_logger + from .handlers.webpack import configure_webpack + from .metrics import request as request_metrics + from .models import db, users + from .query_runner import import_query_runners + from .utils import sentry + from .version_check import reset_new_version_status + + sentry.init() + app = Redash() + + # Check and update the cached version for use by the client + app.before_first_request(reset_new_version_status) + + # Load query runners and destinations + import_query_runners(settings.QUERY_RUNNERS) + import_destinations(settings.DESTINATIONS) + + security.init_app(app) + request_metrics.init_app(app) + db.init_app(app) + migrate.init_app(app, db) + mail.init_app(app) + authentication.init_app(app) + limiter.init_app(app) + handlers.init_app(app) + configure_webpack(app) + extensions.init_app(app) + chrome_logger.init_app(app) + users.init_app(app) + + return app diff --git a/redash/cli/__init__.py b/redash/cli/__init__.py index 316e426d5d..78a10c35cb 100644 --- a/redash/cli/__init__.py +++ b/redash/cli/__init__.py @@ -4,7 +4,8 @@ from flask.cli import FlaskGroup, run_command from flask import current_app -from redash import create_app, settings, __version__ +from redash import settings, __version__ +from redash.app import create_app from redash.cli import users, groups, database, data_sources, organization from redash.monitor import get_status @@ -15,9 +16,11 @@ def create(group): @app.shell_context_processor def shell_context(): - from redash import models - return dict(models=models) - + from redash import models, settings + return { + 'models': models, + 'settings': settings, + } return app @@ -48,7 +51,7 @@ def status(): @manager.command() def check_settings(): """Show the settings as Redash sees them (useful for debugging).""" - for name, item in settings.all_settings().iteritems(): + for name, item in current_app.config.iteritems(): print("{} = {}".format(name, item)) diff --git a/redash/metrics/celery.py b/redash/metrics/celery.py index 6760d96b68..6d972f000e 100644 --- a/redash/metrics/celery.py +++ b/redash/metrics/celery.py @@ -1,17 +1,13 @@ -from __future__ import absolute_import - import logging import socket import time -from celery.signals import task_postrun, task_prerun from redash import settings, statsd_client from redash.utils import json_dumps tasks_start_time = {} -@task_prerun.connect def task_prerun_handler(signal, sender, task_id, task, args, kwargs, **kw): try: tasks_start_time[task_id] = time.time() @@ -29,7 +25,6 @@ def metric_name(name, tags): return "{},{}".format(name, tags_string) -@task_postrun.connect def task_postrun_handler(signal, sender, task_id, task, args, kwargs, retval, state, **kw): try: run_time = 1000 * (time.time() - tasks_start_time.pop(task_id)) diff --git a/redash/settings/__init__.py b/redash/settings/__init__.py index 2bb024d8b6..1b260177d4 100644 --- a/redash/settings/__init__.py +++ b/redash/settings/__init__.py @@ -4,19 +4,7 @@ from flask_talisman import talisman from .helpers import fix_assets_path, array_from_string, parse_boolean, int_or_none, set_from_string -from .organization import DATE_FORMAT - - -def all_settings(): - from types import ModuleType - - settings = {} - for name, item in globals().iteritems(): - if not callable(item) and not name.startswith("__") and not isinstance(item, ModuleType): - settings[name] = item - - return settings - +from .organization import DATE_FORMAT # noqa REDIS_URL = os.environ.get('REDASH_REDIS_URL', os.environ.get('REDIS_URL', "redis://localhost:6379/0")) PROXIES_COUNT = int(os.environ.get('REDASH_PROXIES_COUNT', "1")) diff --git a/redash/tasks/queries.py b/redash/tasks/queries.py index ab16209138..6002ccd275 100644 --- a/redash/tasks/queries.py +++ b/redash/tasks/queries.py @@ -11,7 +11,7 @@ from redash import models, redis_connection, settings, statsd_client from redash.query_runner import InterruptException from redash.tasks.alerts import check_alerts_for_query -from redash.utils import gen_query_hash, json_dumps, json_loads, utcnow, mustache_render +from redash.utils import gen_query_hash, json_dumps, utcnow, mustache_render from redash.worker import celery logger = get_task_logger(__name__) diff --git a/redash/utils/routes.py b/redash/utils/routes.py new file mode 100644 index 0000000000..00a7af0388 --- /dev/null +++ b/redash/utils/routes.py @@ -0,0 +1,15 @@ +from werkzeug.routing import BaseConverter + + +class SlugConverter(BaseConverter): + def to_python(self, value): + # This is ay workaround for when we enable multi-org and some files are being called by the index rule: + # for path in settings.STATIC_ASSETS_PATHS: + # full_path = safe_join(path, value) + # if os.path.isfile(full_path): + # raise ValidationError() + + return value + + def to_url(self, value): + return value diff --git a/redash/worker.py b/redash/worker.py index 23c3d7b6fe..d99f151fd0 100644 --- a/redash/worker.py +++ b/redash/worker.py @@ -7,9 +7,12 @@ from celery import Celery from celery.schedules import crontab -from celery.signals import worker_process_init -from redash import create_app, settings -from redash.metrics import celery as celery_metrics +from celery.signals import task_postrun, task_prerun, worker_process_init + +from redash import settings +from redash.app import create_app +from redash.metrics.celery import task_postrun_handler, task_prerun_handler + celery = Celery('redash', broker=settings.CELERY_BROKER, @@ -66,6 +69,10 @@ def __call__(self, *args, **kwargs): celery.Task = ContextTask +# Connect the task signal handles for Redash metrics +task_prerun.connect(task_prerun_handler) +task_postrun.connect(task_postrun_handler) + # Create Flask app after forking a new worker, to make sure no resources are shared between processes. @worker_process_init.connect diff --git a/redash/wsgi.py b/redash/wsgi.py index 8c94f55037..a8c20bbe6f 100644 --- a/redash/wsgi.py +++ b/redash/wsgi.py @@ -1,3 +1,3 @@ -from redash import create_app +from redash.app import create_app app = create_app() diff --git a/tests/__init__.py b/tests/__init__.py index 07a8fe8f85..9d0f0832b2 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -16,8 +16,8 @@ # Make sure rate limit is enabled os.environ['REDASH_RATELIMIT_ENABLED'] = "true" -from redash import create_app, limiter -from redash import redis_connection +from redash import limiter, redis_connection +from redash.app import create_app from redash.models import db from redash.utils import json_dumps, json_loads from tests.factories import Factory, user_factory From 1a02e71e5506bf63273dc10ff8c9ff5544493ddf Mon Sep 17 00:00:00 2001 From: Jannis Leidel Date: Mon, 18 Mar 2019 11:42:07 +0100 Subject: [PATCH 2/9] Review fixes. --- redash/__init__.py | 8 ++++---- redash/app.py | 4 ++-- redash/cli/__init__.py | 3 +-- redash/worker.py | 3 +-- redash/wsgi.py | 2 +- 5 files changed, 9 insertions(+), 11 deletions(-) diff --git a/redash/__init__.py b/redash/__init__.py index 4856d214bb..466af8acd1 100644 --- a/redash/__init__.py +++ b/redash/__init__.py @@ -17,9 +17,9 @@ from flask_limiter.util import get_ipaddr from flask_migrate import Migrate from statsd import StatsClient -from werkzeug.local import LocalProxy from . import settings +from .app import create_app # noqa __version__ = '7.0.0' @@ -68,11 +68,11 @@ def create_redis_connection(): setup_logging() -redis_connection = LocalProxy(create_redis_connection) +redis_connection = create_redis_connection() -mail = LocalProxy(Mail) +mail = Mail() -migrate = LocalProxy(Migrate) +migrate = Migrate() statsd_client = StatsClient(host=settings.STATSD_HOST, port=settings.STATSD_PORT, prefix=settings.STATSD_PREFIX) diff --git a/redash/app.py b/redash/app.py index f2456e5394..cea2f182d0 100644 --- a/redash/app.py +++ b/redash/app.py @@ -1,7 +1,7 @@ from flask import Flask from werkzeug.contrib.fixers import ProxyFix -from . import mail, settings +from . import settings from .utils import routes @@ -23,7 +23,7 @@ def __init__(self, *args, **kwargs): def create_app(): - from . import authentication, extensions, handlers, limiter, migrate, security + from . import authentication, extensions, handlers, limiter, mail, migrate, security from .destinations import import_destinations from .handlers import chrome_logger from .handlers.webpack import configure_webpack diff --git a/redash/cli/__init__.py b/redash/cli/__init__.py index 78a10c35cb..fdf714ad70 100644 --- a/redash/cli/__init__.py +++ b/redash/cli/__init__.py @@ -4,8 +4,7 @@ from flask.cli import FlaskGroup, run_command from flask import current_app -from redash import settings, __version__ -from redash.app import create_app +from redash import create_app, settings, __version__ from redash.cli import users, groups, database, data_sources, organization from redash.monitor import get_status diff --git a/redash/worker.py b/redash/worker.py index d99f151fd0..41467d2b34 100644 --- a/redash/worker.py +++ b/redash/worker.py @@ -9,8 +9,7 @@ from celery.schedules import crontab from celery.signals import task_postrun, task_prerun, worker_process_init -from redash import settings -from redash.app import create_app +from redash import create_app, settings from redash.metrics.celery import task_postrun_handler, task_prerun_handler diff --git a/redash/wsgi.py b/redash/wsgi.py index a8c20bbe6f..8c94f55037 100644 --- a/redash/wsgi.py +++ b/redash/wsgi.py @@ -1,3 +1,3 @@ -from redash.app import create_app +from redash import create_app app = create_app() From 6381c65aa0df7d3449987537b9e23d10cb17b671 Mon Sep 17 00:00:00 2001 From: Jannis Leidel Date: Mon, 18 Mar 2019 11:42:34 +0100 Subject: [PATCH 3/9] Remove special slug converter for org_slug. --- redash/app.py | 2 -- redash/utils/routes.py | 15 --------------- 2 files changed, 17 deletions(-) delete mode 100644 redash/utils/routes.py diff --git a/redash/app.py b/redash/app.py index cea2f182d0..995f4f23da 100644 --- a/redash/app.py +++ b/redash/app.py @@ -16,8 +16,6 @@ def __init__(self, *args, **kwargs): super(Redash, self).__init__(__name__, *args, **kwargs) # Make sure we get the right referral address even behind proxies like nginx. self.wsgi_app = ProxyFix(self.wsgi_app, settings.PROXIES_COUNT) - # Handle the special case of the org slug - self.url_map.converters['org_slug'] = routes.SlugConverter # Configure Redash using our settings self.config.from_object('redash.settings') diff --git a/redash/utils/routes.py b/redash/utils/routes.py deleted file mode 100644 index 00a7af0388..0000000000 --- a/redash/utils/routes.py +++ /dev/null @@ -1,15 +0,0 @@ -from werkzeug.routing import BaseConverter - - -class SlugConverter(BaseConverter): - def to_python(self, value): - # This is ay workaround for when we enable multi-org and some files are being called by the index rule: - # for path in settings.STATIC_ASSETS_PATHS: - # full_path = safe_join(path, value) - # if os.path.isfile(full_path): - # raise ValidationError() - - return value - - def to_url(self, value): - return value From 7b454ecaa85b3ae0a0177440ee9996ad39fd49af Mon Sep 17 00:00:00 2001 From: Jannis Leidel Date: Mon, 18 Mar 2019 13:04:42 +0100 Subject: [PATCH 4/9] Minor import fix. --- redash/app.py | 1 - 1 file changed, 1 deletion(-) diff --git a/redash/app.py b/redash/app.py index 995f4f23da..ea9e3d5536 100644 --- a/redash/app.py +++ b/redash/app.py @@ -2,7 +2,6 @@ from werkzeug.contrib.fixers import ProxyFix from . import settings -from .utils import routes class Redash(Flask): From 5e5ea9e5ba1154445e65e9263dec5b980447ca81 Mon Sep 17 00:00:00 2001 From: Jannis Leidel Date: Mon, 18 Mar 2019 13:13:33 +0100 Subject: [PATCH 5/9] Another fix. --- redash/__init__.py | 7 ------- redash/handlers/base.py | 2 +- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/redash/__init__.py b/redash/__init__.py index 466af8acd1..1abf9c3413 100644 --- a/redash/__init__.py +++ b/redash/__init__.py @@ -5,13 +5,6 @@ import urlparse import redis -<<<<<<< HEAD -from flask import Flask, current_app -from werkzeug.contrib.fixers import ProxyFix -from werkzeug.routing import BaseConverter -from statsd import StatsClient -======= ->>>>>>> Split redash/__init__.py to prevent import time side-effects. from flask_mail import Mail from flask_limiter import Limiter from flask_limiter.util import get_ipaddr diff --git a/redash/handlers/base.py b/redash/handlers/base.py index 05e82d6c2a..e4bc8e66b2 100644 --- a/redash/handlers/base.py +++ b/redash/handlers/base.py @@ -117,7 +117,7 @@ def paginate(query_set, page, page_size, serializer, **kwargs): def org_scoped_rule(rule): if settings.MULTI_ORG: - return "/{}".format(rule) + return "/{}".format(rule) return rule From a279e7acbf63b198daf2f5a7eab1029ab47b21c4 Mon Sep 17 00:00:00 2001 From: Jannis Leidel Date: Thu, 18 Apr 2019 17:57:30 +0200 Subject: [PATCH 6/9] Move Celery signal callback connections back. --- redash/metrics/celery.py | 3 +++ redash/worker.py | 5 ----- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/redash/metrics/celery.py b/redash/metrics/celery.py index 6d972f000e..98f62f8018 100644 --- a/redash/metrics/celery.py +++ b/redash/metrics/celery.py @@ -2,12 +2,14 @@ import socket import time +from celery.signals import task_postrun, task_prerun from redash import settings, statsd_client from redash.utils import json_dumps tasks_start_time = {} +@task_prerun.connect def task_prerun_handler(signal, sender, task_id, task, args, kwargs, **kw): try: tasks_start_time[task_id] = time.time() @@ -25,6 +27,7 @@ def metric_name(name, tags): return "{},{}".format(name, tags_string) +@task_postrun.connect def task_postrun_handler(signal, sender, task_id, task, args, kwargs, retval, state, **kw): try: run_time = 1000 * (time.time() - tasks_start_time.pop(task_id)) diff --git a/redash/worker.py b/redash/worker.py index 41467d2b34..20287ce704 100644 --- a/redash/worker.py +++ b/redash/worker.py @@ -10,7 +10,6 @@ from celery.signals import task_postrun, task_prerun, worker_process_init from redash import create_app, settings -from redash.metrics.celery import task_postrun_handler, task_prerun_handler celery = Celery('redash', @@ -68,10 +67,6 @@ def __call__(self, *args, **kwargs): celery.Task = ContextTask -# Connect the task signal handles for Redash metrics -task_prerun.connect(task_prerun_handler) -task_postrun.connect(task_postrun_handler) - # Create Flask app after forking a new worker, to make sure no resources are shared between processes. @worker_process_init.connect From 322fca8c53ceff3f06049e0378c7182fb54672c9 Mon Sep 17 00:00:00 2001 From: Jannis Leidel Date: Thu, 18 Apr 2019 18:00:46 +0200 Subject: [PATCH 7/9] Remove useless import. --- redash/worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redash/worker.py b/redash/worker.py index 20287ce704..300c859fc6 100644 --- a/redash/worker.py +++ b/redash/worker.py @@ -7,7 +7,7 @@ from celery import Celery from celery.schedules import crontab -from celery.signals import task_postrun, task_prerun, worker_process_init +from celery.signals import worker_process_init from redash import create_app, settings From 18d67a5bc8d89205b50bcb95c21958e9b2d81802 Mon Sep 17 00:00:00 2001 From: Jannis Leidel Date: Thu, 18 Apr 2019 18:01:52 +0200 Subject: [PATCH 8/9] More import stuff. --- redash/worker.py | 1 + 1 file changed, 1 insertion(+) diff --git a/redash/worker.py b/redash/worker.py index 300c859fc6..b46db432e2 100644 --- a/redash/worker.py +++ b/redash/worker.py @@ -10,6 +10,7 @@ from celery.signals import worker_process_init from redash import create_app, settings +from redash.metrics import celery as celery_metrics # noqa celery = Celery('redash', From 12949baa8bcb7eaae39239f40af26dd8dc10f1b9 Mon Sep 17 00:00:00 2001 From: Jannis Leidel Date: Thu, 18 Apr 2019 18:20:00 +0200 Subject: [PATCH 9/9] Sigh. --- redash/metrics/celery.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/redash/metrics/celery.py b/redash/metrics/celery.py index 98f62f8018..6760d96b68 100644 --- a/redash/metrics/celery.py +++ b/redash/metrics/celery.py @@ -1,3 +1,5 @@ +from __future__ import absolute_import + import logging import socket import time