Include docs in sdist and add Makefile for common build, dev, test

.gitignore

@@ -24,9 +24,6 @@ dist-packages-temp/
 dist-packages-downloads/
 dist-packages/
 
-# Generated by kalite manage setup
-secretkey.txt
-
 # User cache data generated at runtime
 cache/
 
@@ -48,8 +45,15 @@ static-updates
 # Python compiling
 *.pyc
 *.pyo
+
+# Backup files
 *~
+
+# Sublime editor
 *.sublime-*
+
+# Ignore all .zip files!? No comment?? Needs fixing, I guess it's to avoid
+# committing local assessment items.
 *.zip
 
 # Documentation

MANIFEST.in.dist

@@ -27,3 +27,6 @@ recursive-include data *
 # Necessary because it's a data directory so they
 # do not get automatically excluded
 recursive-exclude python-packages *pyc
+
+# Docs
+recursive-include docs/_build/html *

Makefile

@@ -0,0 +1,96 @@
+.PHONY: clean-pyc clean-build docs clean
+
+help:
+	@echo "clean - remove all build, test, coverage and Python artifacts"
+	@echo "clean-build - remove build artifacts"
+	@echo "clean-pyc - remove Python file artifacts"
+	@echo "clean-test - remove test and coverage artifacts"
+	@echo "lint - check style with pep8"
+	@echo "test - run tests the default Python"
+	@echo "test-bdd - run BDD tests only"
+	@echo "test-nobdd - run non-BDD tests only"
+	@echo "assets - build all JS/CSS assets"
+	@echo "coverage - check code coverage quickly with the default Python"
+	@echo "docs - generate Sphinx HTML documentation, including API docs"
+	@echo "release - package and upload a release"
+	@echo "dist - package locally"
+	@echo "install - install the package to the active Python's site-packages"
+
+clean: clean-build clean-pyc clean-test
+
+clean-build:
+	rm -fr build/
+	rm -fr dist/
+	rm -fr .eggs/
+	rm -fr dist-packages/
+	rm -fr dist-packages-temp/
+	find . -name '*.egg-info' -exec rm -fr {} +
+	find . -name '*.egg' -exec rm -f {} +
+
+clean-pyc:
+	find . -name '*.pyc' -exec rm -f {} +
+	find . -name '*.pyo' -exec rm -f {} +
+	find . -name '*~' -exec rm -f {} +
+	find . -name '__pycache__' -exec rm -fr {} +
+
+clean-test:
+	rm -fr .tox/
+	rm -f .coverage
+	rm -fr htmlcov/
+
+lint:
+	pep8 kalite
+	jshint kalite/*/static/js/*/
+
+test:
+	bin/kalite manage test --bdd-only
+
+test-bdd:
+	bin/kalite manage test --bdd-only
+
+test-nobdd:
+	bin/kalite manage test --no-bdd
+
+test-all:
+	@echo "Not supported yet"
+	# tox
+
+coverage:
+	coverage run --source kalite kalitectl.py test
+	coverage report -m
+
+coverage-bdd:
+	coverage run --source kalite kalitectl.py test --bdd-only
+	coverage report -m
+
+coverage-nobdd:
+	coverage run --source kalite kalitectl.py test --no-bdd
+	coverage report -m
+
+docs:
+	# rm -f docs/ka-lite.rst
+	# rm -f docs/modules.rst
+	# sphinx-apidoc -o docs/ ka-lite-gtk
+	$(MAKE) -C docs clean
+	$(MAKE) -C docs html
+	# open docs/_build/html/index.html
+
+assets:
+	# Necessary because NPM may have wrong versions in the cache
+	npm cache clean
+	npm install --production
+	node build.js
+	bin/kalite manage compileymltojson
+
+release: clean docs assets
+	python setup.py sdist --formats=gztar,zip upload --sign
+	python setup.py sdist --formats=gztar,zip upload --sign --static
+	ls -l dist
+
+dist: clean docs assets
+	python setup.py sdist
+	python setup.py sdist --static
+	ls -l dist
+
+install: clean
+	python setup.py install

circle.yml

@@ -9,22 +9,25 @@ dependencies:
   override:
     - pip install -r requirements_test.txt
     - pip install -e .
-    - npm install
-    - npm install -g jshint
+    # This cannot be done because pip on Circle doesn't understand our sdist
+    # - make sdist
+    # - pip install dist/ka-lite-"$(python setup.py --version)".tar.gz
   post:
     - if [[ ! -e sc-latest-linux/bin/sc ]]; then wget https://saucelabs.com/downloads/sc-latest-linux.tar.gz && tar -xzf sc-latest-linux.tar.gz &&  mv sc-*-linux sc-latest-linux; fi
 
 test:
   override:
-    - node build.js
+    - make assets
     - kalite start --settings=kalite.project.settings.disk_based_cache --traceback -v2
     - kalite status
     - kalite stop --traceback -v2
     - cd sc-*-linux && ./bin/sc -u $SAUCE_USERNAME -k $SAUCE_ACCESS_KEY --tunnel-identifier $CIRCLE_BUILD_NUM-$CIRCLE_NODE_INDEX --readyfile ~/sauce_is_ready > sc_output.txt 2>&1:
         background: true
     - while [ ! -e ~/sauce_is_ready ]; do sleep 1; done
-    - case $CIRCLE_NODE_INDEX in 0) kalite manage test --bdd-only ;; 1) kalite manage test --no-bdd;; esac:
+    - case $CIRCLE_NODE_INDEX in 0) make test-bdd ;; 1) make test-nobdd;; esac:
         parallel: true
+    # TODO: replace below with "make link" when we're pep8
+    - npm install -g jshint
     - jshint kalite/*/static/js/*/
   post:
     - killall --wait sc  # wait for Sauce Connect to close the tunnel

kalite/updates/videos.py

@@ -6,9 +6,6 @@
 
 from fle_utils import videos  # keep access to all functions
 from fle_utils.general import softload_json
-
-# I will have nightmares from this :)
-# Fixing all this in 0.16
 from fle_utils.videos import *  # get all into the current namespace, override some.
 
 from kalite.topic_tools import settings as topic_tools_settings

kalitectl.py

@@ -97,7 +97,7 @@
 # Match all patterns of "--option value" and fail if they exist
 __validate_cmd_options = re.compile(r"--?[^\s]+\s+(?:(?!--|-[\w]))")
 if __validate_cmd_options.search(" ".join(sys.argv[1:])):
-    sys.stderr.write("Please only use --option=value or -x123 patterns. No spaces allowed between option and value. The option parser gets confused if you do otherwise.\n\nWill be fixed for next version 0.15")
+    sys.stderr.write("Please only use --option=value or -x123 patterns. No spaces allowed between option and value. The option parser gets confused if you do otherwise.\n\nWill be fixed in a future release.")
     sys.exit(1)
 
 from threading import Thread

requirements_dev.txt

@@ -1,4 +1,5 @@
 -r requirements_test.txt
 werkzeug # Not sure about version
 django-debug-toolbar # Version was not described in previously bundled files, used by kalite.testing app
-pyyaml==3.11
+pep8
+sphinx

requirements_test.txt

@@ -8,3 +8,7 @@ hachoir-core==1.3.3
 hachoir-parser==1.3.4
 hachoir-metadata==1.3.3
 sauceclient==0.2.1
+
+# Used for building assets, which is necessary for testing
+pyyaml==3.11
+

setup.cfg

@@ -6,8 +6,16 @@
 [metadata]
 description-file = README.md
 
+# This is the one we use officially
+[pep8]
+ignore = E226,E302,E41,E402
+max-line-length = 160
+exclude = kalite/*/migrations/*
+
+# Using flake8 is also supported..
 [flake8]
 # E124: closing bracket does not match indentation
 # E702: semicolon at end of line; ignored because we have `from django.conf import settings; logging = settings.LOG` lines and others
 ignore = E702,E124
-max-line-length = 130
+max-line-length = 160
+

setup.py

@@ -123,17 +123,26 @@ def get_installed_packages():
 # site-packages directory.
 
 
-def gen_data_files(*dirs):
+def gen_data_files(*dirs, **kwargs):
     """
     We can only link files, not directories. Therefore, we use an approach
     that scans all files to pass them to the data_files kwarg for setup().
     Thanks: http://stackoverflow.com/a/7288382/405682
     """
     results = []
-
-    filter_illegal_extensions = lambda f: os.path.splitext(f)[1] != ".pyc"
-
+
+    optional = kwargs.pop('optional', False)
+
+    def filter_illegal_extensions(f):
+        return os.path.splitext(f)[1] != ".pyc"
+
     for src_dir in dirs:
+        if not os.path.isdir(src_dir):
+            if optional:
+                continue
+            else:
+                raise RuntimeError("{dir:s} does not exist, cannot continue".format(dir=src_dir))
+
         for root, dirs, files in os.walk(src_dir):
             results.append(
                 (
@@ -162,6 +171,11 @@ def gen_data_files(*dirs):
     gen_data_files('static-libraries')
 )
 
+data_files += map(
+    lambda x: (os.path.join(kalite.ROOT_DATA_PATH, x[0]), x[1]),
+    gen_data_files(os.path.join('docs', '_build', 'html'), optional=True)
+)
+
 # For now, just disguise the kalitectl.py script here as it's only to be accessed
 # with the bin/kalite proxy.
 data_files += [(
@@ -228,37 +242,37 @@ def run(self):
 # If it's a static build, we invoke pip to bundle dependencies in python-packages
 # This would be the case for commands "bdist" and "sdist"
 if STATIC_BUILD:
-    
+
     manifest_content = file(os.path.join(where_am_i, 'MANIFEST.in.dist'), 'r').read()
     manifest_content += "\n" + "recursive-include dist-packages *\nrecursive-exclude dist-packages *pyc"
     file(os.path.join(where_am_i, 'MANIFEST.in'), "w").write(manifest_content)
-    
+
     sys.stderr.write(
         "This is a static build... invoking pip to put static dependencies in "
         "dist-packages/\n"
     )
-    
+
     STATIC_DIST_PACKAGES_DOWNLOAD_CACHE = os.path.join(where_am_i, 'dist-packages-downloads')
     STATIC_DIST_PACKAGES_TEMP = os.path.join(where_am_i, 'dist-packages-temp')
-    
+
     # Create directory where dynamically created dependencies are put
     if not os.path.exists(STATIC_DIST_PACKAGES_DOWNLOAD_CACHE):
         os.mkdir(STATIC_DIST_PACKAGES_DOWNLOAD_CACHE)
-    
+
     # Should remove the temporary directory always
     if os.path.exists(STATIC_DIST_PACKAGES_TEMP):
         print("Removing previous temporary sources for pip {}".format(STATIC_DIST_PACKAGES_TEMP))
         shutil.rmtree(STATIC_DIST_PACKAGES_TEMP)
-    
+
     # Install from pip
-    
+
     # Code modified from this example:
     # http://threebean.org/blog/2011/06/06/installing-from-pip-inside-python-or-a-simple-pip-api/
     import pip.commands.install
-    
+
     # Ensure we get output from pip
     enable_log_to_stdout('pip.commands.install')
-    
+
     def install_distributions(distributions):
         command = pip.commands.install.InstallCommand()
         opts, ___ = command.parser.parse_args([])
@@ -273,17 +287,17 @@ def install_distributions(distributions):
         command.run(opts, distributions)
         # requirement_set.source_dir = STATIC_DIST_PACKAGES_TEMP
         # requirement_set.install(opts)
-    
+
     # Install requirements into dist-packages
     if DIST_BUILDING_COMMAND:
         install_distributions(STATIC_REQUIREMENTS)
-    
+
     # Empty the requirements.txt file
 
 
 # It's not a build command with --static or it's not a build command at all
 else:
-    
+
     # If the dist-packages directory is non-empty
     if os.listdir(STATIC_DIST_PACKAGES):
         # If we are building something or running from the source
@@ -302,7 +316,7 @@ def install_distributions(distributions):
             # everything in the requirements.txt file
             DIST_REQUIREMENTS = []
             DIST_NAME = 'ka-lite-static'
-            
+
             if "ka-lite" in get_installed_packages():
                 raise RuntimeError(
                     "Already installed ka-lite so cannot install ka-lite-static. "
@@ -314,7 +328,7 @@ def install_distributions(distributions):
                     "...or other possible installation mechanisms you may have "
                     "been using."
                 )
-    
+
     # No dist-packages/ and not building, so must be installing the dynamic
     # version
     elif not DIST_BUILDING_COMMAND:
@@ -335,7 +349,7 @@ def install_distributions(distributions):
         manifest_content = file(os.path.join(where_am_i, 'MANIFEST.in.dist'), 'r').read()
         manifest_content += "\n" + "recursive-include dist-packages *"
         file(os.path.join(where_am_i, 'MANIFEST.in'), "w").write(manifest_content)
-    
+
 
 # All files from dist-packages are included if the directory exists
 if os.listdir(STATIC_DIST_PACKAGES):