diff --git a/.travis.yml b/.travis.yml index 34b83e360992b..0fc30037e9648 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,30 +1,30 @@ language: python services: + - mysql + - postgres - redis-server cache: - directories: - - $HOME/.wheelhouse/ + - pip env: - global: - - TRAVIS_CACHE=$HOME/.travis_cache/ - matrix: - - TOX_ENV=flake8 - - TOX_ENV=javascript - - TOX_ENV=pylint - - TOX_ENV=py34-postgres - - TOX_ENV=py34-sqlite - - TOX_ENV=py27-mysql - - TOX_ENV=py27-sqlite + - TOXENV=javascript + - TOXENV=py27-flake8 + - TOXENV=py27-mysql + - TOXENV=py27-sqlite + - TOXENV=py27-pylint + - TOXENV=py34-flake8 + - TOXENV=py34-postgres + - TOXENV=py34-pylint + - TOXENV=py34-sqlite before_script: - mysql -u root -e "DROP DATABASE IF EXISTS superset; CREATE DATABASE superset DEFAULT CHARACTER SET utf8 COLLATE utf8_unicode_ci" - mysql -u root -e "CREATE USER 'mysqluser'@'localhost' IDENTIFIED BY 'mysqluserpassword';" - mysql -u root -e "GRANT ALL ON superset.* TO 'mysqluser'@'localhost';" - psql -U postgres -c "CREATE DATABASE superset;" - psql -U postgres -c "CREATE USER postgresuser WITH PASSWORD 'pguserpassword';" - - export PATH=${PATH}:/tmp/hive/bin install: - pip install --upgrade pip - - pip install codecov tox tox-travis -script: tox -e $TOX_ENV + - pip install codecov tox +script: + - tox after_success: - codecov diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6034b571c83bb..a47aefbc93e2e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -61,20 +61,18 @@ meets these guidelines: 1. The pull request should include tests, either as doctests, unit tests, or both. -2. Run `npm run lint` and resolve all errors. Run `npm run test` and - resolve all test failures. -3. Check code coverage by running the following commands in the `assets` - directory. Run `npm run cover` to check code coverage on `.js` work, and - run `nosetests --with-coverage` to check code coverage on `.py` work. You - may have to first run `pip install nose coverage`. -4. If the pull request adds functionality, the docs should be updated +2. Run `tox` and resolve all errors and test failures. +3. If the pull request adds functionality, the docs should be updated as part of the same PR. Doc string are often sufficient, make sure to follow the sphinx compatible standards. -5. The pull request should work for Python 2.7, and ideally python 3.4+. +4. The pull request should work for Python 2.7, and ideally Python 3.4+. ``from __future__ import`` will be required in every `.py` file soon. -6. Code will be reviewed by re running the unittests, flake8 and syntax - should be as rigorous as the core Python project. -7. Please rebase and resolve all conflicts before submitting. +5. If the pull request adds a Python dependency include it in `setup.py` + denoting any specific restrictions and in `requirements.txt` pinned to a + specific version which ensures that the application build is deterministic. +6. Please rebase and resolve all conflicts before submitting. +7. Please ensure the necessary checks pass and that code coverage does not + decrease. 8. If you are asked to update your pull request with some changes there's no need to create a new one. Push your changes to the same branch. @@ -108,7 +106,7 @@ Finally, to make changes to the rst files and build the docs using Sphinx, you'll need to install a handful of dependencies from the repo you cloned: cd incubator-superset - pip install -r dev-reqs-for-docs.txt + pip install -r docs/requirements-docs.txt To get the feel for how to edit and build the docs, let's edit a file, build the docs and see our changes in action. First, you'll want to @@ -183,6 +181,7 @@ Check the [OS dependencies](https://superset.incubator.apache.org/installation.h source env/bin/activate # install for development + pip install -r requirements.txt pip install -e . # Create an admin user @@ -270,21 +269,28 @@ npm run dev Should you add or upgrade a npm package, which involves changing `package.json`, you'll need to re-run `yarn install` and push the newly generated `yarn.lock` file so we get the reproducible build. More information at (https://yarnpkg.com/blog/2016/11/24/lockfiles-for-all/) ## Testing +All tests are carried out in [tox](http://tox.readthedocs.io/en/latest/index.html) +a standardized testing framework mostly for Python (though we also used it for Javascript). +All python tests can be run with any of the tox [environments](http://tox.readthedocs.io/en/latest/example/basic.html#a-simple-tox-ini-default-environments), via, -Before running python unit tests, please setup local testing environment: -``` -pip install -r dev-reqs.txt -``` + tox -e + +i.e., -All python tests can be run with: + tox -e py27 + tox -e py34 - ./run_tests.sh +Alternatively, you can run all tests in a single file via, -Alternatively, you can run a specific test with: + tox -e -- tests/test_file.py - ./run_specific_test.sh tests.core_tests:CoreTests.test_function_name +or for a specific test via, -Note that before running specific tests, you have to both setup the local testing environment and run all tests. + tox -e -- tests/test_file.py:TestClassName.test_method_name + +Note that the test environment uses a temporary directory for defining the +SQLite databases which will be cleared each time before the group of test +commands are invoked. We use [Mocha](https://mochajs.org/), [Chai](http://chaijs.com/) and [Enzyme](http://airbnb.io/enzyme/) to test Javascript. Tests can be run with: @@ -297,16 +303,17 @@ We use [Mocha](https://mochajs.org/), [Chai](http://chaijs.com/) and [Enzyme](ht Lint the project with: # for python - flake8 + tox -e flake8 # for javascript - npm run lint + tox -e eslint ## API documentation Generate the documentation with: - cd docs && ./build.sh + pip install -r docs/requirements.txt + python setup.py build_sphinx ## CSS Themes As part of the npm build process, CSS for Superset is compiled from `Less`, a dynamic stylesheet language. @@ -400,7 +407,7 @@ https://github.com/apache/incubator-superset/pull/3013 .. code:: # install doc dependencies - pip install -r dev-reqs-for-docs.txt + pip install -r docs/requirements.txt # build the docs python setup.py build_sphinx diff --git a/dev-reqs-for-docs.txt b/dev-reqs-for-docs.txt deleted file mode 100644 index a3885b5d2e0e8..0000000000000 --- a/dev-reqs-for-docs.txt +++ /dev/null @@ -1,3 +0,0 @@ -sphinx -sphinx-rtd-theme -sphinxcontrib.youtube diff --git a/dev-reqs.txt b/dev-reqs.txt deleted file mode 100644 index b565fbdf23254..0000000000000 --- a/dev-reqs.txt +++ /dev/null @@ -1,14 +0,0 @@ -codeclimate-test-reporter -flake8 -flask_cors -ipdb -mock -mysqlclient -nose -psycopg2 -pylint -pyyaml -redis -statsd -# Also install everything we need to build Sphinx docs --r dev-reqs-for-docs.txt diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000000000..748df23c7ce5d --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,3 @@ +sphinx==1.7.1 +sphinx-rtd-theme==0.2.4 +sphinxcontrib.youtube==0.1.2 diff --git a/pylint-errors.sh b/pylint-errors.sh deleted file mode 100755 index 5c2d022887c61..0000000000000 --- a/pylint-errors.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -pylint superset --errors-only diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000000000..3ca7ffbe545c7 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,7 @@ +flask-cors==3.0.3 +ipdb==0.11 +mysqlclient==1.3.12 +psycopg2==2.7.4 +redis==2.10.6 +statsd==3.2.2 +tox==2.9.1 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000..fb2fcd8cd475a --- /dev/null +++ b/requirements.txt @@ -0,0 +1,40 @@ +bleach==2.1.2 +boto3==1.4.7 +celery==4.1.0 +colorama==0.3.9 +cryptography==1.9 +flask==0.12.2 +flask-appbuilder==1.10.0 +flask-cache==0.13.1 +flask-compress==1.4.0 +flask-migrate==2.1.1 +flask-script==2.0.6 +flask-sqlalchemy==2.1 +flask-testing==0.7.1 +flask-wtf==0.14.2 +flower==0.9.2 +future==0.16.0 +geopy==1.11.0 +gunicorn==19.7.1 +humanize==0.5.1 +idna==2.6 +markdown==2.6.11 +pandas==0.22.0 +parsedatetime==2.0.0 +pathlib2==2.3.0 +polyline==1.3.2 +pydruid==0.4.1 +pyhive==0.5.0 +python-dateutil==2.6.1 +python-geohash==0.8.5 +pyyaml==3.12 +requests==2.18.4 +simplejson==3.13.2 +six==1.11.0 +sqlalchemy==1.2.2 +sqlalchemy-utils==0.32.21 +sqlparse==0.2.4 +thrift==0.11.0 +thrift-sasl==0.3.0 +unicodecsv==0.14.1 +unidecode==1.0.22 diff --git a/run_tests.sh b/run_tests.sh deleted file mode 100755 index 0648b5703c053..0000000000000 --- a/run_tests.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash -echo $DB -rm ~/.superset/unittests.db -rm ~/.superset/celerydb.sqlite -rm ~/.superset/celery_results.sqlite -rm -f .coverage -export PYTHONPATH=./ -export SUPERSET_CONFIG=tests.superset_test_config -set -e -superset/bin/superset db upgrade -superset/bin/superset version -v -python setup.py nosetests diff --git a/setup.cfg b/setup.cfg index 1e4e4f4e1bc49..cdf1ed547aef7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -10,16 +10,16 @@ license = Apache License, Version 2.0 packages = superset [build_sphinx] -source-dir = docs/ -build-dir = docs/_build -all_files = 1 +source-dir = docs +build-dir = docs/_build +all_files = 1 [upload_sphinx] upload-dir = docs/_build/html [nosetests] -verbosity=3 -detailed-errors=1 -with-coverage=1 -nocapture=1 -cover-package=superset +verbosity = 3 +detailed-errors = 1 +with-coverage = 1 +nocapture = 1 +cover-package = superset diff --git a/setup.py b/setup.py index 3bf44fb6a797b..43f0c4abaa629 100644 --- a/setup.py +++ b/setup.py @@ -50,56 +50,50 @@ def get_git_sha(): zip_safe=False, scripts=['superset/bin/superset'], install_requires=[ + 'bleach', 'boto3>=1.4.6', - 'celery==4.1.0', - 'colorama==0.3.9', - 'cryptography==1.9', - 'flask==0.12.2', - 'flask-appbuilder==1.10.0', - 'flask-cache==0.13.1', - 'flask-migrate==2.1.1', - 'flask-script==2.0.6', - 'flask-compress==1.4.0', - 'flask-sqlalchemy==2.1', - 'flask-testing==0.7.1', - 'flask-wtf==0.14.2', - 'flower==0.9.2', # deprecated + 'celery', + 'colorama', + 'cryptography', + 'flask', + 'flask-appbuilder', + 'flask-cache', + 'flask-compress', + 'flask-migrate', + 'flask-script', + 'flask-sqlalchemy', + 'flask-testing', + 'flask-wtf', + 'flower', # deprecated 'future>=0.16.0, <0.17', - 'geopy==1.11.0', - 'python-geohash==0.8.5', - 'humanize==0.5.1', - 'gunicorn==19.7.1', # deprecated - 'idna==2.6', - 'markdown==2.6.11', - 'pandas==0.22.0', - 'parsedatetime==2.0.0', - 'pathlib2==2.3.0', - 'polyline==1.3.2', - 'pydruid==0.4.1', - 'PyHive>=0.4.0', - 'python-dateutil==2.6.1', + 'geopy', + 'gunicorn', # deprecated + 'humanize', + 'idna', + 'markdown', + 'pandas', + 'parsedatetime', + 'pathlib2', + 'polyline', + 'pydruid', + 'pyhive>=0.4.0', + 'python-dateutil', + 'python-geohash', 'pyyaml>=3.11', - 'requests==2.18.4', - 'simplejson==3.13.2', - 'six==1.11.0', - 'sqlalchemy==1.2.2', - 'sqlalchemy-utils==0.32.21', - 'sqlparse==0.2.4', + 'requests', + 'simplejson', + 'six', + 'sqlalchemy', + 'sqlalchemy-utils', + 'sqlparse', 'thrift>=0.9.3', 'thrift-sasl>=0.2.1', + 'unicodecsv', 'unidecode>=0.04.21', - 'unicodecsv==0.14.1', - 'bleach==2.1.2', ], extras_require={ - 'cors': ['Flask-Cors>=2.0.0'], + 'cors': ['flask-cors>=2.0.0'], }, - tests_require=[ - 'coverage', - 'mock', - 'nose', - 'redis', - ], author='Maxime Beauchemin', author_email='maximebeauchemin@gmail.com', url='https://github.com/apache/incubator-superset', diff --git a/tests/base_tests.py b/tests/base_tests.py index c491637f22a58..3cec2157e4159 100644 --- a/tests/base_tests.py +++ b/tests/base_tests.py @@ -29,7 +29,6 @@ class SupersetTestCase(unittest.TestCase): def __init__(self, *args, **kwargs): if ( self.requires_examples and - not os.environ.get('SOLO_TEST') and not os.environ.get('examples_loaded') ): logging.info('Loading examples') diff --git a/tox.ini b/tox.ini index 8f226c67c45f4..fba53956ac384 100644 --- a/tox.ini +++ b/tox.ini @@ -1,12 +1,3 @@ -[tox] -envlist = - py27-mysql - py27-sqlite - py27-postgres - py34-mysql - py35-mysql -skipsdist=True - [flake8] accept-encodings = utf-8 application-import-names = superset @@ -28,36 +19,37 @@ ignore = FI54 import-order-style = google max-line-length = 90 -require-code = True - -[global] -wheel_dir = {homedir}/.wheelhouse -find_links = - {homedir}/.wheelhouse - {homedir}/.pip-cache +require-code = true [testenv] +commands = + {toxinidir}/superset/bin/superset db upgrade + nosetests {posargs} deps = - wheel - coveralls + -rrequirements.txt + -rrequirements-dev.txt + coverage + mock + nose +setenv = + PYTHONPATH = {toxinidir} + SUPERSET_CONFIG = tests.superset_test_config + SUPERSET_HOME = {envtmpdir} + py27-mysql: SUPERSET__SQLALCHEMY_DATABASE_URI = mysql://mysqluser:mysqluserpassword@localhost/superset?charset=utf8 + py34-mysql: SUPERSET__SQLALCHEMY_DATABASE_URI = mysql://mysqluser:mysqluserpassword@localhost/superset + {py27,py34}-postgres: SUPERSET__SQLALCHEMY_DATABASE_URI = postgresql+psycopg2://postgresuser:pguserpassword@localhost/superset + {py27,py34}-sqlite: SUPERSET__SQLALCHEMY_DATABASE_URI = sqlite:////{envtmpdir}/superset.db whitelist_externals = - pip npm -passenv = - HOME - USER - TRAVIS_CACHE - PATH + +[testenv:eslint] +changedir = {toxinidir}/superset/assets commands = - python --version - pip wheel -w {homedir}/.wheelhouse -f {homedir}/.wheelhouse . - pip install --find-links={homedir}/.wheelhouse --no-index . - pip install -r dev-reqs.txt - {toxinidir}/run_tests.sh + npm run lint +deps = [testenv:flake8] commands = - pip install --upgrade flake8 flake8 deps = flake8 @@ -71,45 +63,22 @@ deps = commands = npm install -g npm@'>=5.6.0' {toxinidir}/superset/assets/js_build.sh +deps = [testenv:pylint] commands = - pip wheel -w {homedir}/.wheelhouse -f {homedir}/.wheelhouse . - pip install --find-links={homedir}/.wheelhouse --no-index . - pip install -r dev-reqs.txt - {toxinidir}/pylint-errors.sh - -[testenv:py27-mysql] -basepython = python2.7 -setenv = - SUPERSET__SQLALCHEMY_DATABASE_URI = mysql://mysqluser:mysqluserpassword@localhost/superset?charset=utf8 - -[testenv:py34-mysql] -basepython = python3.4 -setenv = - SUPERSET__SQLALCHEMY_DATABASE_URI = mysql://mysqluser:mysqluserpassword@localhost/superset - -[testenv:py35-mysql] -basepython = python3.5 -setenv = - SUPERSET__SQLALCHEMY_DATABASE_URI = mysql://mysqluser:mysqluserpassword@localhost/superset - -[testenv:py27-sqlite] -basepython = python2.7 -setenv = - SUPERSET__SQLALCHEMY_DATABASE_URI = sqlite:////tmp/superset.db - -[testenv:py34-sqlite] -basepython = python3.4 -setenv = - SUPERSET__SQLALCHEMY_DATABASE_URI = sqlite:////tmp/superset.db - -[testenv:py27-postgres] -basepython = python2.7 -setenv = - SUPERSET__SQLALCHEMY_DATABASE_URI = postgresql+psycopg2://postgresuser:pguserpassword@localhost/superset + pylint superset --errors-only +deps = + -rrequirements.txt + -rrequirements-dev.txt + pylint -[testenv:py34-postgres] -basepython = python3.4 -setenv = - SUPERSET__SQLALCHEMY_DATABASE_URI = postgresql+psycopg2://postgresuser:pguserpassword@localhost/superset +[tox] +envlist = + eslint + flake8 + javascript + py27 + py34 + pylint +skipsdist = true