ci

#!/usr/bin/python
# -*- mode: python -*-

"""\
%prog [options]

This program is designed for the continuous build, test and deployment
process.  A secondary purpose is to run on a development workstation
for the purposes of recreating and deugging any continuous build, test
and deployment failures.

As such it expects to execute with the current working directory
pointing to some "home directory" or "workspace".  The contents of
this workspace are:

   ./git/jwallib      

                     - The source tree. Maintained by the CI framework
                       so that it can report on the changes since last
                       build and poll for SCM changes.

   ./virtualenv
   
                     - A python virtualenv prefix into which this
                       script will install the couchapp utility and
                       dependencies.  Also includes the nose testrunner.

If the program detects that it is being run from a git repository (by
the presence of a ./.git directory) then it assumes that it is running
on a development workstation.  In this case it makes a fresh,
temporary home directory and cleans it up after completion.  The
git/jwallib sub-directory is built from the detected git repository,
including a fresh commit of any local changes.  Note that local
changes are only committed to a temporary repository that is then
discarded.

As a third mode of operation, the script has no import dependencies
outside of the standard library so it can be called directly from
Github like this:

python -c "$( wget -O - https://github.com/jwal/jwallib/ci )"

In this mode of operation, detected by the python interpreter having
no __file__ variable on the __main__ module, the script will allocate
a temporary working directory and clone the jwallib git repository
from github.

"""

from __future__ import with_statement

try:
    from jwalutil import add_user_to_url
except:
    def add_user_to_url(base_url, username, password):
        scheme, rest = base_url.split("://", 1)
        return "%s://%s:%s@%s" % (
            scheme,
            urllib.quote(username, safe=""),
            urllib.quote(password, safe=""),
            rest)

import json
import optparse
import os
import posixpath
import subprocess
import sys
import contextlib
import tempfile
import shutil

@contextlib.contextmanager
def maybe_mkdtemp(candidate):
    if candidate is None:
        temp_dir = tempfile.mkdtemp()
    try:
        if candidate is None:
            yield temp_dir
        else:
            yield candidate
    finally:
        if candidate is None:
            shutil.rmtree(temp_dir)

try:
    from gitdevcommit import git_clone
except:
    def git_clone(git_url, dest_path):
        if os.path.exists(dest_path):
            return
        subprocess.check_call(["git", "clone", git_url, dest_path])


try:
    from jwalutil import on_error_return_none
except:
    def on_error_return_none(message):
        return None

try:
    from jwalutil import on_error_raise
except:
    def on_error_raise(message):
        raise Exception(message)





try:
    from gitdevcommit import find_git_dir
except:
    def find_git_dir(start_dir=".", on_missing=on_error_raise):
        start_dir = os.path.abspath(start_dir)
        candidate = start_dir
        while True:
            candidate_git = os.path.join(candidate, ".git")
            if os.path.exists(candidate_git):
                return candidate
            if os.path.dirname(candidate) == candidate:
                return on_missing("Unable to find a git repository: %s"
                                  % (start_dir,))
            candidate = os.path.dirname(candidate)

try:
    from gitdevcommit import git_status
except:
    def git_status(git_path):
        child = subprocess.Popen(["git", "status", "--porcelain"],
                                 cwd=git_path, stdout=subprocess.PIPE)
        stdout, stderr = child.communicate()
        assert child.returncode == 0, git_path
        return [x for x in stdout.rstrip("\n").split("\n") if x != ""]

try:
    from gitdevcommit import git_generate_patch
except:
    def git_generate_patch(git_path):
        child = subprocess.Popen(["git", "diff", "HEAD"],
                                 cwd=git_path, stdout=subprocess.PIPE)
        stdout, stderr = child.communicate()
        assert child.returncode == 0, git_path
        return stdout

def fixup_url(url, config):
    for candidate in config["urlrewrites"]:
        if url.startswith(candidate["from"]):
            url = candidate["to"] + url[len(candidate["from"]):]
    return url

try:
    from gitdevcommit import git_apply_patch
except:
    def git_apply_patch(patch, git_path):
        child = subprocess.Popen(["git", "apply", "-"],
                                 cwd=git_path, stdin=subprocess.PIPE)
        stdout, stderr = child.communicate(patch)
        assert child.returncode == 0, git_path

try:
    from gitdevcommit import git_commit
except:
    def git_commit(git_path, message="Autocommit", username="root",
                   email="fail@example.com"):
        subprocess.check_call(["git", "-c", "user.name=%s" % (username,),
                               "-c", "user.email=%s" % (email,),
                               "commit", "-am", message], cwd=git_path)

try:
    from gitdevcommit import git_dev_commit
except:
    def git_dev_commit(dev_repo, dest_repo):
        if dest_repo.startswith(dev_repo):
            raise Exception("Dev repo %r within dest repo %r"
                            % (dev_repo, dest_repo))
        if dev_repo.startswith(dest_repo):
            raise Exception("Dest repo %r within dev repo %r"
                            % (dest_repo, dev_repo))
        git_clone(dev_repo, git_path)
        # TODO: Also commit untracked files?
        if len(git_status(dev_repo)) > 0:
            print git_status(dev_repo)
            patch = git_generate_patch(dev_repo)
            print patch
            # TODO: Reset to the revision in the dev_repo first
            git_apply_patch(patch, git_path)
            git_commit(git_path)


def main(argv):
    parser = optparse.OptionParser(__doc__)
    parser.add_option("--config", dest="config_path", default=None)
    parser.add_option("--saucelabs-url", dest="saucelabs_url",
                      default="http://ondemand.saucelabs.com:80/wd/hub")
    parser.add_option("--home-dir", dest="home_dir", default=None,
                      help=("The workspace used by the script for source code"
                            "and other dependencies.  Defaults to a temporary "
                            "directory in dev and download modes.  Defaults "
                            "to the current directory in ci mode."))
    parser.add_option("--ci-mode", dest="mode", default="auto",
                      action="store_const", const="ci")
    parser.add_option("--dev-mode", dest="mode", default="auto",
                      action="store_const", const="dev")
    parser.add_option("--auto-mode", dest="mode", default="auto",
                      action="store_const", const="auto")
    parser.add_option("--download-mode", dest="mode", default="auto",
                      action="store_const", const="download")
    parser.add_option("--development-repository", dest="dev_repo",
                      help=("A git repository, normally a folder on the local "
                            "machine, to run the code from.  Only used in dev "
                            "mode.  The repository is cloned and any local "
                            "changes are committed to the temporary clone."))
    parser.add_option("--git-working-copy", dest="git_working_copy_path",
                      default=".")
    parser.add_option("--git-url", dest="git_url",
                      default="https://github.com/jwal/jwallib")
    parser.add_option("--couchdb-url", dest="couchdb_url",
                      default="https://jwal.cloudant.com/")
    parser.add_option("--couchdb-deploy-url", dest="couchdb_deploy_url")
    parser.add_option("--couchdb-test-url", dest="couchdb_test_url")
    parser.add_option("--deploy", dest="do_deploy", default=False,
                      action="store_const", const=True, 
                      help=("Deploy if the tests pass"))
    options, args = parser.parse_args(argv)
    if len(args) > 0:
        parser.error("Unexpected: %r" % (args,))
    config_path = options.config_path
    config = {}
    if config_path is None:
        config_path = "file://" + os.path.join(os.path.expanduser("~"),
                                               ".config", "jwallib", 
                                               "config.json")
    config_json = None
    if config_path.startswith("{"):
        config_json = config_path
    elif config_path.startswith("json:"):
        config_json = config_path[len("json:"):]
    elif config_path.startswith("file:"):
        config_path = config_path[len("file:"):]
    if config_json is None:
        if os.path.exists(config_path):
            with open(config_path, "rb") as fh:
                config_json = fh.read()
        else:
            config_json = "{}"
    config.update(json.loads(config_json))
    mode = options.mode
    if mode == "auto":
        if "__file__" not in globals().keys():
            mode = "download"
        elif find_git_dir(on_missing=on_error_return_none) is not None:
            mode = "dev"
        else:
            mode = "ci"
    if mode == "ci" and options.home_dir is None:
        home_dir = os.path.abspath(".")
    else:
        home_dir = options.home_dir
    with maybe_mkdtemp(home_dir) as home_dir:
        git_path = os.path.join(home_dir, "git", "jwallib")
        if mode == "download":
            git_clone(options.git_url, git_path)
        elif mode == "dev":
            if not os.path.exists(git_path):
                dev_repo = options.dev_repo or find_git_dir()
                git_dev_commit(dev_repo, git_path)
        elif mode == "ci":
            pass
        else:
            raise NotImplementedError(mode)
        virtualenv_path = os.path.join(home_dir, "virtualenv")
        virtualenv_activate = os.path.join(virtualenv_path, "bin", "activate")
        env_script = ["bash", "-c",
                      'source "$1" && shift && exec "$@"', "-",
                      virtualenv_activate]
        cwd_script = ["bash", "-c", 'cd "$1" && shift && exec "$@"', "-", 
                      git_path]
        if not os.path.exists(virtualenv_path):
            subprocess.check_call(["virtualenv", virtualenv_path])
            subprocess.check_call(env_script + ["pip", "install", "couchapp"])
            subprocess.check_call(env_script + ["pip", "install", "nose"])
            subprocess.check_call(env_script + ["pip", "install", "selenium"])
            subprocess.check_call(env_script + ["pip", "install", "pycurl"])
        subprocess.check_call(env_script + ["couchapp", "--version"])
        #subprocess.check_call(cwd_script + ["git", "checkout", "master"])
        #subprocess.check_call(cwd_script + ["git", "reset", "--hard", 
        #                                    "remotes/origin/master"])
        subprocess.check_call(cwd_script + ["git", "log", "-1"])
        subprocess.check_call(cwd_script + ["git", "branch", "-a"])
#        subprocess.check_call(env_script + cwd_script + ["nosetests"])
        subprocess.check_call(env_script + cwd_script
                              + ["python", "test_aptconfig.py", "-v"])
        ## Needs python 2.7
        #subprocess.check_call(env_script + cwd_script
        #                      + ["python", "test_shellescape.py", "-v"])
        couchdb_url = fixup_url(options.couchdb_url, config)
        if options.couchdb_test_url is None:
            couchdb_test_url = posixpath.join(
                couchdb_url, "gitbrowser-testing")
        else:
            couchdb_test_url = fixup_url(options.couchdb_test_url, config)
        argv = ["python", "-m", "test_gitbrowser"]
        argv.extend(["--couchdb-url", couchdb_test_url])
        argv.extend(["--saucelabs-url", fixup_url(options.saucelabs_url, 
                                                  config)])
        argv.extend(["--", "--verbose"])
        subprocess.check_call(
            env_script
            + ["bash", "-c", 
               'export PYTHONPATH="$1:$PYTHONPATH" && shift && exec "$@"', "-",
               git_path] + argv)
        if options.do_deploy:
            if options.couchdb_deploy_url is None:
                couchdb_deploy_url = posixpath.join(couchdb_url, "jwallib")
            else:
                couchdb_deploy_url = fixup_url(options.couchdb_deploy_url, 
                                               config)
            subprocess.check_call(cwd_script + ["python", "gitcouchdbsync.py", 
                                                couchdb_deploy_url])
            subprocess.check_call(env_script + cwd_script 
                                  + ["python", "selfcouchapp.py",
                                     "--app-subdir", "gitbrowser",
                                     couchdb_deploy_url])

if __name__ == "__main__":
    sys.exit(main(sys.argv[1:]))