diff --git a/.gitignore b/.gitignore index d9afb0c99d..12e67119b0 100644 --- a/.gitignore +++ b/.gitignore @@ -24,9 +24,6 @@ dist-packages-temp/ dist-packages-downloads/ dist-packages/ -# Generated by kalite manage setup -secretkey.txt - # User cache data generated at runtime cache/ @@ -48,8 +45,15 @@ static-updates # Python compiling *.pyc *.pyo + +# Backup files *~ + +# Sublime editor *.sublime-* + +# Ignore all .zip files!? No comment?? Needs fixing, I guess it's to avoid +# committing local assessment items. *.zip # Documentation diff --git a/MANIFEST.in.dist b/MANIFEST.in.dist index c87008e666..811c9b900c 100644 --- a/MANIFEST.in.dist +++ b/MANIFEST.in.dist @@ -27,3 +27,6 @@ recursive-include data * # Necessary because it's a data directory so they # do not get automatically excluded recursive-exclude python-packages *pyc + +# Docs +recursive-include docs/_build/html * diff --git a/Makefile b/Makefile new file mode 100644 index 0000000000..66f7bbccde --- /dev/null +++ b/Makefile @@ -0,0 +1,96 @@ +.PHONY: clean-pyc clean-build docs clean + +help: + @echo "clean - remove all build, test, coverage and Python artifacts" + @echo "clean-build - remove build artifacts" + @echo "clean-pyc - remove Python file artifacts" + @echo "clean-test - remove test and coverage artifacts" + @echo "lint - check style with pep8" + @echo "test - run tests the default Python" + @echo "test-bdd - run BDD tests only" + @echo "test-nobdd - run non-BDD tests only" + @echo "assets - build all JS/CSS assets" + @echo "coverage - check code coverage quickly with the default Python" + @echo "docs - generate Sphinx HTML documentation, including API docs" + @echo "release - package and upload a release" + @echo "dist - package locally" + @echo "install - install the package to the active Python's site-packages" + +clean: clean-build clean-pyc clean-test + +clean-build: + rm -fr build/ + rm -fr dist/ + rm -fr .eggs/ + rm -fr dist-packages/ + rm -fr dist-packages-temp/ + find . -name '*.egg-info' -exec rm -fr {} + + find . -name '*.egg' -exec rm -f {} + + +clean-pyc: + find . -name '*.pyc' -exec rm -f {} + + find . -name '*.pyo' -exec rm -f {} + + find . -name '*~' -exec rm -f {} + + find . -name '__pycache__' -exec rm -fr {} + + +clean-test: + rm -fr .tox/ + rm -f .coverage + rm -fr htmlcov/ + +lint: + pep8 kalite + jshint kalite/*/static/js/*/ + +test: + bin/kalite manage test --bdd-only + +test-bdd: + bin/kalite manage test --bdd-only + +test-nobdd: + bin/kalite manage test --no-bdd + +test-all: + @echo "Not supported yet" + # tox + +coverage: + coverage run --source kalite kalitectl.py test + coverage report -m + +coverage-bdd: + coverage run --source kalite kalitectl.py test --bdd-only + coverage report -m + +coverage-nobdd: + coverage run --source kalite kalitectl.py test --no-bdd + coverage report -m + +docs: + # rm -f docs/ka-lite.rst + # rm -f docs/modules.rst + # sphinx-apidoc -o docs/ ka-lite-gtk + $(MAKE) -C docs clean + $(MAKE) -C docs html + # open docs/_build/html/index.html + +assets: + # Necessary because NPM may have wrong versions in the cache + npm cache clean + npm install --production + node build.js + bin/kalite manage compileymltojson + +release: clean docs assets + python setup.py sdist --formats=gztar,zip upload --sign + python setup.py sdist --formats=gztar,zip upload --sign --static + ls -l dist + +dist: clean docs assets + python setup.py sdist + python setup.py sdist --static + ls -l dist + +install: clean + python setup.py install diff --git a/circle.yml b/circle.yml index 09bdbebfa5..aeb73234ec 100644 --- a/circle.yml +++ b/circle.yml @@ -9,22 +9,25 @@ dependencies: override: - pip install -r requirements_test.txt - pip install -e . - - npm install - - npm install -g jshint + # This cannot be done because pip on Circle doesn't understand our sdist + # - make sdist + # - pip install dist/ka-lite-"$(python setup.py --version)".tar.gz post: - if [[ ! -e sc-latest-linux/bin/sc ]]; then wget https://saucelabs.com/downloads/sc-latest-linux.tar.gz && tar -xzf sc-latest-linux.tar.gz && mv sc-*-linux sc-latest-linux; fi test: override: - - node build.js + - make assets - kalite start --settings=kalite.project.settings.disk_based_cache --traceback -v2 - kalite status - kalite stop --traceback -v2 - cd sc-*-linux && ./bin/sc -u $SAUCE_USERNAME -k $SAUCE_ACCESS_KEY --tunnel-identifier $CIRCLE_BUILD_NUM-$CIRCLE_NODE_INDEX --readyfile ~/sauce_is_ready > sc_output.txt 2>&1: background: true - while [ ! -e ~/sauce_is_ready ]; do sleep 1; done - - case $CIRCLE_NODE_INDEX in 0) kalite manage test --bdd-only ;; 1) kalite manage test --no-bdd;; esac: + - case $CIRCLE_NODE_INDEX in 0) make test-bdd ;; 1) make test-nobdd;; esac: parallel: true + # TODO: replace below with "make link" when we're pep8 + - npm install -g jshint - jshint kalite/*/static/js/*/ post: - killall --wait sc # wait for Sauce Connect to close the tunnel diff --git a/kalite/updates/videos.py b/kalite/updates/videos.py index b9240d9691..b336b4543b 100644 --- a/kalite/updates/videos.py +++ b/kalite/updates/videos.py @@ -6,9 +6,6 @@ from fle_utils import videos # keep access to all functions from fle_utils.general import softload_json - -# I will have nightmares from this :) -# Fixing all this in 0.16 from fle_utils.videos import * # get all into the current namespace, override some. from kalite.topic_tools import settings as topic_tools_settings diff --git a/kalitectl.py b/kalitectl.py index 98904f9d9e..932f077ce4 100644 --- a/kalitectl.py +++ b/kalitectl.py @@ -97,7 +97,7 @@ # Match all patterns of "--option value" and fail if they exist __validate_cmd_options = re.compile(r"--?[^\s]+\s+(?:(?!--|-[\w]))") if __validate_cmd_options.search(" ".join(sys.argv[1:])): - sys.stderr.write("Please only use --option=value or -x123 patterns. No spaces allowed between option and value. The option parser gets confused if you do otherwise.\n\nWill be fixed for next version 0.15") + sys.stderr.write("Please only use --option=value or -x123 patterns. No spaces allowed between option and value. The option parser gets confused if you do otherwise.\n\nWill be fixed in a future release.") sys.exit(1) from threading import Thread diff --git a/requirements_dev.txt b/requirements_dev.txt index e90b466b01..1a1851d6d4 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,4 +1,5 @@ -r requirements_test.txt werkzeug # Not sure about version django-debug-toolbar # Version was not described in previously bundled files, used by kalite.testing app -pyyaml==3.11 +pep8 +sphinx diff --git a/requirements_test.txt b/requirements_test.txt index d24bc668ec..79cc3fe319 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -8,3 +8,7 @@ hachoir-core==1.3.3 hachoir-parser==1.3.4 hachoir-metadata==1.3.3 sauceclient==0.2.1 + +# Used for building assets, which is necessary for testing +pyyaml==3.11 + diff --git a/setup.cfg b/setup.cfg index 396c126565..c0049d6ef0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -6,8 +6,16 @@ [metadata] description-file = README.md +# This is the one we use officially +[pep8] +ignore = E226,E302,E41,E402 +max-line-length = 160 +exclude = kalite/*/migrations/* + +# Using flake8 is also supported.. [flake8] # E124: closing bracket does not match indentation # E702: semicolon at end of line; ignored because we have `from django.conf import settings; logging = settings.LOG` lines and others ignore = E702,E124 -max-line-length = 130 +max-line-length = 160 + diff --git a/setup.py b/setup.py index 04c8824c9c..18ebfccd7a 100644 --- a/setup.py +++ b/setup.py @@ -123,17 +123,26 @@ def get_installed_packages(): # site-packages directory. -def gen_data_files(*dirs): +def gen_data_files(*dirs, **kwargs): """ We can only link files, not directories. Therefore, we use an approach that scans all files to pass them to the data_files kwarg for setup(). Thanks: http://stackoverflow.com/a/7288382/405682 """ results = [] - - filter_illegal_extensions = lambda f: os.path.splitext(f)[1] != ".pyc" - + + optional = kwargs.pop('optional', False) + + def filter_illegal_extensions(f): + return os.path.splitext(f)[1] != ".pyc" + for src_dir in dirs: + if not os.path.isdir(src_dir): + if optional: + continue + else: + raise RuntimeError("{dir:s} does not exist, cannot continue".format(dir=src_dir)) + for root, dirs, files in os.walk(src_dir): results.append( ( @@ -162,6 +171,11 @@ def gen_data_files(*dirs): gen_data_files('static-libraries') ) +data_files += map( + lambda x: (os.path.join(kalite.ROOT_DATA_PATH, x[0]), x[1]), + gen_data_files(os.path.join('docs', '_build', 'html'), optional=True) +) + # For now, just disguise the kalitectl.py script here as it's only to be accessed # with the bin/kalite proxy. data_files += [( @@ -228,37 +242,37 @@ def run(self): # If it's a static build, we invoke pip to bundle dependencies in python-packages # This would be the case for commands "bdist" and "sdist" if STATIC_BUILD: - + manifest_content = file(os.path.join(where_am_i, 'MANIFEST.in.dist'), 'r').read() manifest_content += "\n" + "recursive-include dist-packages *\nrecursive-exclude dist-packages *pyc" file(os.path.join(where_am_i, 'MANIFEST.in'), "w").write(manifest_content) - + sys.stderr.write( "This is a static build... invoking pip to put static dependencies in " "dist-packages/\n" ) - + STATIC_DIST_PACKAGES_DOWNLOAD_CACHE = os.path.join(where_am_i, 'dist-packages-downloads') STATIC_DIST_PACKAGES_TEMP = os.path.join(where_am_i, 'dist-packages-temp') - + # Create directory where dynamically created dependencies are put if not os.path.exists(STATIC_DIST_PACKAGES_DOWNLOAD_CACHE): os.mkdir(STATIC_DIST_PACKAGES_DOWNLOAD_CACHE) - + # Should remove the temporary directory always if os.path.exists(STATIC_DIST_PACKAGES_TEMP): print("Removing previous temporary sources for pip {}".format(STATIC_DIST_PACKAGES_TEMP)) shutil.rmtree(STATIC_DIST_PACKAGES_TEMP) - + # Install from pip - + # Code modified from this example: # http://threebean.org/blog/2011/06/06/installing-from-pip-inside-python-or-a-simple-pip-api/ import pip.commands.install - + # Ensure we get output from pip enable_log_to_stdout('pip.commands.install') - + def install_distributions(distributions): command = pip.commands.install.InstallCommand() opts, ___ = command.parser.parse_args([]) @@ -273,17 +287,17 @@ def install_distributions(distributions): command.run(opts, distributions) # requirement_set.source_dir = STATIC_DIST_PACKAGES_TEMP # requirement_set.install(opts) - + # Install requirements into dist-packages if DIST_BUILDING_COMMAND: install_distributions(STATIC_REQUIREMENTS) - + # Empty the requirements.txt file # It's not a build command with --static or it's not a build command at all else: - + # If the dist-packages directory is non-empty if os.listdir(STATIC_DIST_PACKAGES): # If we are building something or running from the source @@ -302,7 +316,7 @@ def install_distributions(distributions): # everything in the requirements.txt file DIST_REQUIREMENTS = [] DIST_NAME = 'ka-lite-static' - + if "ka-lite" in get_installed_packages(): raise RuntimeError( "Already installed ka-lite so cannot install ka-lite-static. " @@ -314,7 +328,7 @@ def install_distributions(distributions): "...or other possible installation mechanisms you may have " "been using." ) - + # No dist-packages/ and not building, so must be installing the dynamic # version elif not DIST_BUILDING_COMMAND: @@ -335,7 +349,7 @@ def install_distributions(distributions): manifest_content = file(os.path.join(where_am_i, 'MANIFEST.in.dist'), 'r').read() manifest_content += "\n" + "recursive-include dist-packages *" file(os.path.join(where_am_i, 'MANIFEST.in'), "w").write(manifest_content) - + # All files from dist-packages are included if the directory exists if os.listdir(STATIC_DIST_PACKAGES):