Pull from upstream

.circleci/config.yml

@@ -39,7 +39,7 @@ jobs:
           name: Copy Test Results
           command: |
             mkdir -p /tmp/test-results/unit-tests
-            docker cp tests:/app/coverage.xml ./coverage.xml 
+            docker cp tests:/app/coverage.xml ./coverage.xml
             docker cp tests:/app/junit.xml /tmp/test-results/unit-tests/results.xml
       - store_test_results:
           path: /tmp/test-results
@@ -61,6 +61,7 @@ jobs:
     steps:
       - checkout
       - run: sudo apt install python-pip
+      - run: sudo pip install -r requirements_bundles.txt
       - run: npm install
       - run: npm run bundle
       - run: npm test
@@ -95,6 +96,7 @@ jobs:
     steps:
       - checkout
       - run: sudo apt install python-pip
+      - run: sudo pip install -r requirements_bundles.txt
       - run: npm install
       - run: .circleci/update_version
       - run: npm run bundle
@@ -105,11 +107,14 @@ jobs:
           path: /tmp/artifacts/
   build-docker-image:
     docker:
-      - image: circleci/buildpack-deps:xenial
+      - image: circleci/node:8
     steps:
       - setup_remote_docker
       - checkout
+      - run: sudo apt install python-pip
+      - run: sudo pip install -r requirements_bundles.txt
       - run: .circleci/update_version
+      - run: npm run bundle
       - run: .circleci/docker_build
 workflows:
   version: 2

.circleci/docker-compose.cypress.yml

@@ -39,6 +39,10 @@ services:
       PERCY_BRANCH: ${CIRCLE_BRANCH}
       PERCY_COMMIT: ${CIRCLE_SHA1}
       PERCY_PULL_REQUEST: ${CIRCLE_PR_NUMBER}
+      COMMIT_INFO_BRANCH: ${CIRCLE_BRANCH}
+      COMMIT_INFO_AUTHOR: ${CIRCLE_USERNAME}
+      COMMIT_INFO_SHA: ${CIRCLE_SHA1}
+      COMMIT_INFO_REMOTE: ${CIRCLE_REPOSITORY_URL}
       CYPRESS_PROJECT_ID: ${CYPRESS_PROJECT_ID}
       CYPRESS_RECORD_KEY: ${CYPRESS_RECORD_KEY}
   redis:

.editorconfig

@@ -9,6 +9,6 @@ trim_trailing_whitespace = true
 indent_style = space
 indent_size = 4
 
-[*.{js,css,html}]
+[*.{js,jsx,css,less,html}]
 indent_style = space
 indent_size = 2

Dockerfile

@@ -4,17 +4,18 @@ WORKDIR /frontend
 COPY package.json package-lock.json /frontend/
 RUN npm install
 
-COPY . /frontend
+COPY client /frontend/client
+COPY webpack.config.js /frontend/
 RUN npm run build
 
-FROM redash/base:latest
+FROM redash/base:debian
 
 # Controls whether to install extra dependencies needed for all data sources.
 ARG skip_ds_deps
 
 # We first copy only the requirements file, to avoid rebuilding on every file
 # change.
-COPY requirements.txt requirements_dev.txt requirements_all_ds.txt ./
+COPY requirements.txt requirements_bundles.txt requirements_dev.txt requirements_all_ds.txt ./
 RUN pip install -r requirements.txt -r requirements_dev.txt
 RUN if [ "x$skip_ds_deps" = "x" ] ; then pip install -r requirements_all_ds.txt ; else echo "Skipping pip install -r requirements_all_ds.txt" ; fi
 

README.md

@@ -1,11 +1,10 @@
 <p align="center">
   <img title="Redash" src='https://redash.io/assets/images/logo.png' width="200px"/>
 </p>
-<p align="center">
-    <img title="Build Status" src='https://circleci.com/gh/getredash/redash.png?circle-token=8a695aa5ec2cbfa89b48c275aea298318016f040'/>
-</p>
 
 [![Documentation](https://img.shields.io/badge/docs-redash.io/help-brightgreen.svg)](https://redash.io/help/)
+[![Datree](https://s3.amazonaws.com/catalog.static.datree.io/datree-badge-20px.svg)](https://datree.io/?src=badge)
+![Build Status](https://circleci.com/gh/getredash/redash.png?circle-token=8a695aa5ec2cbfa89b48c275aea298318016f040)
 
 **_Redash_** is our take on freeing the data within our company in a way that will better fit our culture and usage patterns.
 

SECURITY.md

@@ -0,0 +1,5 @@
+# Security Policy
+
+## Reporting a Vulnerability
+
+Please email [email protected] to report any security vulnerabilities. We will acknowledge receipt of your vulnerability and strive to send you regular updates about our progress. If you're curious about the status of your disclosure please feel free to email us again. If you want to encrypt your disclosure email, you can use [this PGP key](https://keybase.io/arikfr/key.asc).

bin/bundle-extensions

@@ -1,39 +1,118 @@
 #!/usr/bin/env python
-
+# -*- coding: utf-8 -*-
+"""Copy bundle extension files to the client/app/extension directory"""
+import logging
 import os
-from subprocess import call
-from distutils.dir_util import copy_tree
+from pathlib2 import Path
+from shutil import copy
+from collections import OrderedDict as odict
+
+from importlib_metadata import entry_points
+from importlib_resources import contents, is_resource, path
 
-from pkg_resources import iter_entry_points, resource_filename, resource_isdir
+# Name of the subdirectory
+BUNDLE_DIRECTORY = "bundle"
 
+logger = logging.getLogger(__name__)
 
 
 # Make a directory for extensions and set it as an environment variable
 # to be picked up by webpack.
-EXTENSIONS_RELATIVE_PATH = os.path.join('client', 'app', 'extensions')
-EXTENSIONS_DIRECTORY = os.path.join(
-    os.path.dirname(os.path.dirname(__file__)),
-    EXTENSIONS_RELATIVE_PATH)
-
-if not os.path.exists(EXTENSIONS_DIRECTORY):
-    os.makedirs(EXTENSIONS_DIRECTORY)
-os.environ["EXTENSIONS_DIRECTORY"] = EXTENSIONS_RELATIVE_PATH
-
-for entry_point in iter_entry_points('redash.extensions'):
-    # This is where the frontend code for an extension lives
-    # inside of its package.
-    content_folder_relative = os.path.join(
-        entry_point.name, 'bundle')
-    (root_module, _) = os.path.splitext(entry_point.module_name)
-
-    if not resource_isdir(root_module, content_folder_relative):
-        continue
+extensions_relative_path = Path('client', 'app', 'extensions')
+extensions_directory = Path(__file__).parent.parent / extensions_relative_path
+
+if not extensions_directory.exists():
+    extensions_directory.mkdir()
+os.environ["EXTENSIONS_DIRECTORY"] = str(extensions_relative_path)
+
+
+def resource_isdir(module, resource):
+    """Whether a given resource is a directory in the given module
+
+    https://importlib-resources.readthedocs.io/en/latest/migration.html#pkg-resources-resource-isdir
+    """
+    try:
+        return resource in contents(module) and not is_resource(module, resource)
+    except (ImportError, TypeError):
+        # module isn't a package, so can't have a subdirectory/-package
+        return False
+
+
+def entry_point_module(entry_point):
+    """Returns the dotted module path for the given entry point"""
+    return entry_point.pattern.match(entry_point.value).group("module")
+
+
+def load_bundles():
+    """"Load bundles as defined in Redash extensions.
 
-    content_folder = resource_filename(root_module, content_folder_relative)
+    The bundle entry point can be defined as a dotted path to a module
+    or a callable, but it won't be called but just used as a means
+    to find the files under its file system path.
+
+    The name of the directory it looks for files in is "bundle".
+
+    So a Python package with an extension bundle could look like this::
+
+        my_extensions/
+        ├── __init__.py
+        └── wide_footer
+            ├── __init__.py
+            └── bundle
+                ├── extension.js
+                └── styles.css
+
+    and would then need to register the bundle with an entry point
+    under the "redash.bundles" group, e.g. in your setup.py::
+
+        setup(
+            # ...
+            entry_points={
+                "redash.bundles": [
+                    "wide_footer = my_extensions.wide_footer",
+                ]
+                # ...
+            },
+            # ...
+        )
+
+    """
+    bundles = odict()
+    for entry_point in entry_points().get("redash.bundles", []):
+        logger.info('Loading Redash bundle "%s".', entry_point.name)
+        module = entry_point_module(entry_point)
+        # Try to get a list of bundle files
+        if not resource_isdir(module, BUNDLE_DIRECTORY):
+            logger.error(
+                'Redash bundle directory "%s" could not be found.', entry_point.name
+            )
+            continue
+        with path(module, BUNDLE_DIRECTORY) as bundle_dir:
+            bundles[entry_point.name] = list(bundle_dir.rglob("*"))
+
+    return bundles
+
+
+bundles = load_bundles().items()
+if bundles:
+    print('Number of extension bundles found: {}'.format(len(bundles)))
+else:
+    print('No extension bundles found.')
+
+for bundle_name, paths in bundles:
+    # Shortcut in case not paths were found for the bundle
+    if not paths:
+        print('No paths found for bundle "{}".'.format(bundle_name))
+        continue
 
-    # This is where we place our extensions folder.
-    destination = os.path.join(
-        EXTENSIONS_DIRECTORY,
-        entry_point.name)
+    # The destination for the bundle files with the entry point name as the subdirectory
+    destination = Path(extensions_directory, bundle_name)
+    if not destination.exists():
+        destination.mkdir()
 
-    copy_tree(content_folder, destination)
+    # Copy the bundle directory from the module to its destination.
+    print('Copying "{}" bundle to {}:'.format(bundle_name, destination.resolve()))
+    for src_path in paths:
+        dest_path = destination / src_path.name
+        print(" - {} -> {}".format(src_path, dest_path))
+        copy(str(src_path), str(dest_path))

bin/docker-entrypoint

@@ -20,8 +20,21 @@ scheduler() {
   exec /usr/local/bin/celery worker --app=redash.worker --beat -s$SCHEDULE_DB -c$WORKERS_COUNT -Q$QUEUES -linfo --max-tasks-per-child=10 -Ofair
 }
 
+dev_worker() {
+  WORKERS_COUNT=${WORKERS_COUNT:-2}
+  QUEUES=${QUEUES:-queries,scheduled_queries,celery,schemas}
+  SCHEDULE_DB=${SCHEDULE_DB:-celerybeat-schedule}
+
+  echo "Starting dev scheduler and $WORKERS_COUNT workers for queues: $QUEUES..."
+
+  exec watchmedo auto-restart --directory=./redash/ --pattern=*.py --recursive -- /usr/local/bin/celery worker --app=redash.worker --beat -s$SCHEDULE_DB -c$WORKERS_COUNT -Q$QUEUES -linfo --max-tasks-per-child=10 -Ofair
+}
+
 server() {
-  exec /usr/local/bin/gunicorn -b 0.0.0.0:5000 --name redash -w${REDASH_WEB_WORKERS:-4} redash.wsgi:app
+  # Recycle gunicorn workers every n-th request. See http://docs.gunicorn.org/en/stable/settings.html#max-requests for more details.
+  MAX_REQUESTS=${MAX_REQUESTS:-1000}
+  MAX_REQUESTS_JITTER=${MAX_REQUESTS_JITTER:-100}
+  exec /usr/local/bin/gunicorn -b 0.0.0.0:5000 --name redash -w${REDASH_WEB_WORKERS:-4} redash.wsgi:app --max-requests $MAX_REQUESTS --max-requests-jitter $MAX_REQUESTS_JITTER
 }
 
 create_db() {
@@ -41,6 +54,7 @@ help() {
   echo "server -- start Redash server (with gunicorn)"
   echo "worker -- start Celery worker"
   echo "scheduler -- start Celery worker with a beat (scheduler) process"
+  echo "dev_worker  -- start Celery worker with a beat (scheduler) process which picks up code changes and reloads"
   echo "celery_healthcheck -- runs a Celery healthcheck. Useful for Docker's HEALTHCHECK mechanism."
   echo ""
   echo "shell -- open shell"
@@ -75,6 +89,10 @@ case "$1" in
     shift
     scheduler
     ;;
+  dev_worker)
+    shift
+    dev_worker
+    ;;
   dev_server)
     export FLASK_DEBUG=1
     exec /app/manage.py runserver --debugger --reload -h 0.0.0.0

client/.eslintrc.js

@@ -1,7 +1,7 @@
 module.exports = {
   root: true,
   extends: ["airbnb", "plugin:compat/recommended"],
-  plugins: ["jest", "compat"],
+  plugins: ["jest", "compat", "no-only-tests"],
   settings: {
     "import/resolver": "webpack"
   },
@@ -26,7 +26,7 @@ module.exports = {
     "consistent-return": "off",
     "no-control-regex": "off",
     "no-multiple-empty-lines": "warn",
-    "no-script-url": "off", // some <a> tags should have href="javascript:void(0)"
+    "no-only-tests/no-only-tests": "error",
     "operator-linebreak": "off",
     "react/destructuring-assignment": "off",
     "react/jsx-filename-extension": "off",

client/app/.eslintrc.js

@@ -4,4 +4,7 @@ module.exports = {
   env: {
     "jest/globals": true,
   },
+  rules: {
+    "jest/no-focused-tests": "off",
+  },
 };