buildtool

#!/usr/bin/env python
# Works with either Python 2 or Python 3.

from __future__ import print_function

import argparse
import contextlib
import logging
import os
import platform
import re
import shutil
import subprocess
import sys

__version__ = '0.1.2'

prog_name = os.path.basename(sys.argv[0])

extra_environ_vars = {}

download_dirs = [
    '~/download/programming/vim',
    '/tools/software/vim'
]

configure_options = """
    --quiet
    --with-features=huge
    --enable-gui=gtk3
    --enable-perlinterp
    --enable-pythoninterp
    --enable-python3interp
    --enable-tclinterp
    --enable-rubyinterp
    --enable-cscope
    --with-compiledby=vim@drmikehenry.com
    """.split()

ruby_command = '/usr/bin/ruby'
if os.path.exists(ruby_command):
    configure_options.extend(
        ['--with-ruby-command=' + ruby_command]
    )

tmp_dir = 'build'

usage = """%(prog)s [-h] [--version] [-v] [-q] CMD
For more detailed help: %(prog)s --help
"""

help_description = """
buildtool aids in the compilation of Vim.
"""

help_epilog = """
For help on specific CMD:    %%(prog)s CMD --help

Typical CMD invocations:
  %%(prog)s build [SOURCE]   - unpack, packagesrc, configure, build, package
  %%(prog)s update           - "git pull" in newest Git repo in download_dirs

Less common CMD invocations:
  %%(prog)s unpack [SOURCE]  - unpack source to vim-x.y.z/
  %%(prog)s packagesrc       - make vim-x.y.z.arch.tar.bz2 (source) in PWD
                               (Note: runs "make clean" before packaging!)
  %%(prog)s configure        - run "./configure [options]" in PWD
  %%(prog)s make             - run "make" in PWD
  %%(prog)s package          - make vim-x.y.z.arch.tar.bz2 (binary) in PWD

SOURCE is a Vim source tarball or a Git repo.  If not provided, %%(prog)s
will search through download_dirs (below) for the newest tarball or Git repo to
use:
  %(download_dirs)s

Configuration is done with these options:
  %(configure_options)s
""" % dict(
    download_dirs='\n  '.join(download_dirs),
    configure_options=' '.join(configure_options))


# Add a default handler for logging.
logging.getLogger().addHandler(logging.StreamHandler(sys.stdout))

logger = logging.getLogger('buildtool')


class BuildError(Exception):
    pass


# Hold parsed arguments in a namespace that may have member functions and
# that allows for more control over storage of argument values.
# Argument values are stored here as regular attributes that match the
# name given in the add_argument() call.
class Args(argparse.Namespace):
    def __init__(self):
        argparse.Namespace.__init__(self)


@contextlib.contextmanager
def preserve_cwd(new_dir=None):
    old_cwd = os.getcwd()
    try:
        if new_dir:
            os.chdir(new_dir)
        yield
    finally:
        os.chdir(old_cwd)


def rm_rf(path):
    if os.path.isfile(path):
        os.remove(path)
    elif os.path.exists(path):
        shutil.rmtree(path)


def mkdir_p(path):
    if not os.path.exists(path):
        os.makedirs(path)


def path_join(*parts):
    return os.path.join(*(p for p in parts if p))


def get_run_env():
    env = os.environ.copy()
    env.update(extra_environ_vars)
    return env


def run(args):
    logger.debug('  ' + ' '.join(args))
    output_bytes = subprocess.check_output(
        args,
        env=get_run_env(),
        stderr=subprocess.STDOUT)
    return output_bytes.decode(errors='ignore')


def run_pipe(args):
    logger.debug('  ' + ' '.join(args))
    process = subprocess.Popen(
        args,
        env=get_run_env(),
        stdout=subprocess.PIPE,
        stderr=subprocess.STDOUT)
    return process


def vim_version_from_version_ch(version_ch):
    """version_ch is text of Vim source tree's version.c + version.h.
    """
    major = -1
    minor = -1
    patch = -1
    in_patches = False
    for line in version_ch.splitlines():
        if in_patches:
            m = re.match(r'^\s*(\d+)\s*,?\s*$', line)
            if m:
                patch = max(patch, int(m.group(1)))
            elif re.match(r'^\s*\}', line):
                in_patches = False
        elif re.search(r'^\s*static\s+int\s+included_patches\[\]\s*=', line):
            in_patches = True

        else:
            m = re.match(
                r'\s*#define\s+VIM_VERSION_(MAJOR|MINOR)\s+(\d+)\s*$',
                line)
            if m:
                value = int(m.group(2))
                if m.group(1) == 'MAJOR':
                    major = value
                else:
                    minor = value

    if in_patches or major < 0 or minor < 0 or patch < 0:
        raise BuildError('Could not determine version from Vim source')

    version = '%d.%d.%04d' % (major, minor, patch)
    return version


def vim_version_from_source_tree(tree='.'):
    c = path_join(tree, 'src/version.c')
    h = path_join(tree, 'src/version.h')
    try:
        source = open(c).read() + open(h).read()
    except IOError:
        raise BuildError('not a valid Vim source tree')
    return vim_version_from_version_ch(source)


def could_be_git_repo(path):
    return (os.path.exists(path_join(path, '.git')) or
            os.path.exists(path_join(path, 'refs')))


def vim_version_ch_from_git(git_source):
    if not could_be_git_repo(git_source):
        return ''
    with preserve_cwd(git_source):
        try:
            version_ch = run(
                'git show HEAD:src/version.c HEAD:src/version.h'.split())
        except subprocess.CalledProcessError:
            version_ch = ''
    return version_ch


def vim_version_from_git(git_source):
    version_ch = vim_version_ch_from_git(git_source)
    if not version_ch:
        raise BuildError('%s is not a Vim git repository' % git_source)
    return vim_version_from_version_ch(version_ch)


def download_dir():
    for d in download_dirs:
        d = os.path.expanduser(d)
        if os.path.isdir(d):
            return d
    raise BuildError('cannot find download directory')


def versioned_tarballs(root):
    for root, dirs, files in os.walk(download_dir()):
        for f in files:
            m = re.match(r'vim-(\d+\.\d+(\.\d+)?)\.tar\.(gz|bz2)$', f)
            if m:
                yield m.group(1), path_join(root, f)


def versioned_git_repos(root):
    for root, dirs, files in os.walk(download_dir()):
        version_ch = vim_version_ch_from_git(root)
        if version_ch:
            version = vim_version_from_version_ch(version_ch)
            yield version, root
            del dirs[:]


def versioned_sources(root):
    return list(versioned_tarballs(root)) + list(versioned_git_repos(root))


def sorted_versioned_sources(ver_sources):
    return sorted(
        ver_sources,
        key=lambda version_path: list(map(int, version_path[0].split('.'))),
        reverse=True)


def guess_source():
    candidates = sorted_versioned_sources(versioned_sources(download_dir()))
    if candidates:
        for version, source in candidates:
            logger.debug('Found Vim version %s at %s' % (version, source))
        version, source = candidates[0]
        print('Using Vim %s from %s' % (version, source))
        return source
    return ''


def resolve_source(source):
    """Resolve source into file (tarball) or directory (git repo)."""
    if not source:
        source = guess_source()
        if not source:
            raise BuildError('could not guess source')

    # Files must be tarballs.
    if os.path.isfile(source):
        return source

    if not os.path.isdir(source):
        raise BuildError('source must be tarball or directory')

    # If source is a Vim git repository, return it.
    if vim_version_ch_from_git(source):
        return source

    raise BuildError('bad source %s' % source)


def verify_empty_unpack_dest(dest):
    if os.path.exists(dest):
        raise BuildError(
            'must remove %s before unpacking' % dest)


def start_phase(phase):
    print()
    print('### ' + phase)


def update():
    candidates = sorted_versioned_sources(versioned_git_repos(download_dir()))
    if not candidates:
        raise BuildError('cannot find a Git repo to update')
    version, repo = candidates[0]
    print('Updating Git repo %s' % repo)
    with preserve_cwd(repo):
        run('git pull'.split())
    print('Old version = %s' % version)
    print('New version = %s' % vim_version_from_git(repo))


def unpack_tar(source):
    tmp_extract_dir = '__TMP_EXTRACT__'
    if os.path.exists(tmp_extract_dir):
        raise BuildError(
            'must remove %s for tarball extraction' % tmp_extract_dir)
    try:
        print('Extracting %s into temp area %s' %
              (source, tmp_extract_dir))
        os.mkdir(tmp_extract_dir)
        run(['tar', '-C', tmp_extract_dir, '-xf', source])

        extracted = os.listdir(tmp_extract_dir)
        if len(extracted) != 1:
            raise BuildError('invalid tarball %s' % source)
        tar_root = path_join(tmp_extract_dir, extracted[0])
        version = vim_version_from_source_tree(tar_root)
        print('Found version %s' % version)
        dest_dir = 'vim-' + version
        verify_empty_unpack_dest(dest_dir)
        os.rename(tar_root, dest_dir)
        return dest_dir

    finally:
        rm_rf(tmp_extract_dir)


def unpack_git(source):
    version = vim_version_from_git(source)
    dest_dir = 'vim-' + version
    verify_empty_unpack_dest(dest_dir)
    print('Cloning Git source %s into %s' % (source, dest_dir))
    run(['git', 'clone', source, dest_dir])
    return dest_dir


def package_src(dest_dir=None):
    start_phase('packagesrc')
    with preserve_cwd(dest_dir):
        version = vim_version_from_source_tree()
        logger.debug('Packaging source for version %s' % version)

        src_tar_name = 'vim-%s.tar.bz2' % version
        rm_rf(src_tar_name)

        # To ensure the proper directory structure, first tar everything into
        # a temporary tarball, extract into a temporary directory with the
        # proper parent name, then create the final tarball.

        print('Performing "make distclean"')
        make_dist_clean()
        mkdir_p(tmp_dir)
        src_dir_name = 'vim-%s' % version
        src_dir = path_join(tmp_dir, src_dir_name)
        mkdir_p(src_dir)
        tmp_tar = path_join(tmp_dir, 'tmp.tar')
        # Create temporary tarfile with no enclosing parent directory, e.g.:
        #   tar -cf build/tmp.tar \
        #   --exclude .git --exclude *.gz --exclude *.bz2 --exclude build .
        run([
            'tar',
            '-cf',
            tmp_tar,
            '--exclude', '.git',
            '--exclude', '*.gz',
            '--exclude', '*.bz2',
            '--exclude', tmp_dir,
            '.'])

        # Extract back into src_dir, e.g.::
        #   tar -C build/vim-8.1.0216 -xf build/tmp.tar
        run(['tar', '-C', src_dir, '-xf', tmp_tar])

        # Create final tarfile with proper parent directory, e.g.::
        #   tar -C build -cf vim-8.1.0216.tar.bz2 vim-8.1.0216
        run(['tar', '-C', tmp_dir, '-cf', src_tar_name, src_dir_name])
        print('Created source archive %s' % path_join(dest_dir, src_tar_name))


def unpack(source):
    start_phase('unpack')
    if os.path.isfile(source):
        dest_dir = unpack_tar(source)
    elif os.path.isdir(source):
        dest_dir = unpack_git(source)
    else:
        raise BuildError('source is neither file nor directory')
    print('Unpacked to %s' % dest_dir)
    return dest_dir


def configure(dest_dir=None):
    start_phase('configure')
    with preserve_cwd(dest_dir):
        print('If configure fails, look at %s' %
              path_join(dest_dir, 'src/auto/config.log'))
        run(['./configure'] + configure_options)


def make(dest_dir=None):
    start_phase('make')
    with preserve_cwd(dest_dir):
        log_path = path_join(tmp_dir, 'build.log')
        print('On build failure, examine %s for errors' %
              path_join(dest_dir, log_path))
        mkdir_p(tmp_dir)

        p = run_pipe(['make'])

        i = 0
        width = 40
        with open(log_path, 'wb') as log:
            for line_bytes in iter(p.stdout.readline, b''):
                log.write(line_bytes)

                line = line_bytes.decode(errors='ignore')
                if line.startswith('gcc'):
                    msg = '[compile] %s' % line.split()[-1]
                else:
                    msg = 'Working %s' % ('-\|/'[i])
                    i = (i + 1) % 4
                msg = msg.ljust(width)
                sys.stdout.write('\r' + msg)
                sys.stdout.flush()
        print('\r' + ' ' * width)
        sys.stdout.flush()
        exit_status = p.wait()
        if exit_status != 0:
            raise BuildError('error running make')


def make_dist_clean(dest_dir=None):
    with preserve_cwd(dest_dir):
        run('make distclean'.split())


def package(dest_dir=None):
    start_phase('package')
    with preserve_cwd(dest_dir):
        version = vim_version_from_source_tree()
        arch = platform.uname()[-1]
        logger.debug('Packaging for version %s on %s' % (version, arch))

        bin_tar_name = 'vim-%s.%s.tar.gz' % (version, arch)

        tar_opts = []
        try:
            if 'GNU tar' in run('tar --version'.split()):
                tar_opts = '--owner=0 --group=0'.split()
        except subprocess.CalledProcessError:
            pass

        mkdir_p(tmp_dir)
        dest = path_join(tmp_dir, 'DESTDIR')
        rm_rf(dest)
        run(['make', 'install', 'DESTDIR=' + os.path.abspath(dest)])
        run('chmod -R u=rwX,go=rX'.split() + [dest])
        run([
            'tar',
            '-C', dest,
            '--numeric-owner',
            ] + tar_opts + [
            '-zcf',
            bin_tar_name,
            '.'])
        tar_path = path_join(dest_dir, bin_tar_name)
        print('Created binary archive %s' % tar_path)
        print('To install:')
        print('sudo tar -C / -xf %s' % tar_path)


def setup_fedora_environment():
    """
    Setup environment variables needed on Fedora 23+.

    On Fedora 23+, libraries are built with "hardened" settings.  They are
    automatically applied when building an RPM, but not when compiling by hand.
    These variables are needed (at least through vim-8.0.0133) when compiling
    with --enable-rubyinterp and using Fedora-supplied ruby libraries.  Vim's
    configure script asks the ruby interpreter what LDFLAGS to use without also
    asking about CFLAGS; this leads to mismatched values of certain "hardened"
    settings that are now the default in all RPM builds.  The %configure macro
    in RPM SPEC files sets up both CFLAGS and LDFLAGS to their %hardened
    values, which is why Vim's configure script works when building an %RPM,
    but these settings must be applied manually when not building an RPM.

    The variables may be extracted from ``rpmbuild`` via::

      eval $(rpmbuild --eval '%{configure}' | egrep '^\s*[A-Z]+=')

    Example variables are::

      CFLAGS="${CFLAGS:--O2 -g -pipe -Wall -Werror=format-security
        -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fexceptions
        -fstack-protector-strong -grecord-gcc-switches
        -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1
        -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 -m64 -mtune=generic
        -fasynchronous-unwind-tables -fstack-clash-protection -fcf-protection}"

      CXXFLAGS="${CXXFLAGS:--O2 -g -pipe -Wall -Werror=format-security
        -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fexceptions
        -fstack-protector-strong -grecord-gcc-switches
        -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1
        -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 -m64 -mtune=generic
        -fasynchronous-unwind-tables -fstack-clash-protection -fcf-protection}"

      FFLAGS="${FFLAGS:--O2 -g -pipe -Wall -Werror=format-security
        -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fexceptions
        -fstack-protector-strong -grecord-gcc-switches
        -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1
        -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 -m64 -mtune=generic
        -fasynchronous-unwind-tables -fstack-clash-protection -fcf-protection
        -I/usr/lib64/gfortran/modules}"

      FCFLAGS="${FCFLAGS:--O2 -g -pipe -Wall -Werror=format-security
        -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fexceptions
        -fstack-protector-strong -grecord-gcc-switches
        -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1
        -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 -m64 -mtune=generic
        -fasynchronous-unwind-tables -fstack-clash-protection -fcf-protection
        -I/usr/lib64/gfortran/modules}"

      LDFLAGS="${LDFLAGS:--Wl,-z,relro  -Wl,-z,now
        -specs=/usr/lib/rpm/redhat/redhat-hardened-ld}"

    """
    if not os.path.exists('/etc/redhat-release'):
        return
    try:
        output = run('rpmbuild --eval %{configure}'.split())
    except OSError:
        print('## Warning: could not run rpmbuild; '
              'not setting environment variables')
        return

    env_vars = re.findall(
        r"""
            ^\s*
            ([A-Z]+)
            =
            " \$ \{ \w+ :-
            ([^}"]+)
            }"
        """,
        output,
        re.MULTILINE | re.VERBOSE)
    print('## Detected rpmbuild; adding environment variables')
    for var, value in env_vars:
        logger.debug('%s="%s"' % (var, value))
        extra_environ_vars[var] = value


def inner_main():
    parser = argparse.ArgumentParser(
        usage=usage,
        description=help_description,
        epilog=help_epilog,
        formatter_class=argparse.RawDescriptionHelpFormatter)

    parser.add_argument(
        '--version',
        action='version',
        version=__version__)
    parser.add_argument(
        '-v', '--verbose',
        action='store_const',
        dest='verbose',
        const=logging.DEBUG,
        help="""verbose output for debugging""")
    parser.add_argument(
        '-q', '--quiet',
        action='store_const',
        dest='verbose',
        const=logging.WARNING,
        help="""suppress informational output""")

    subparsers = parser.add_subparsers(
        dest='cmd',
        help=argparse.SUPPRESS)

    # 'update' command:
    subparsers.add_parser(
        'update',
        help='"git pull" in newest Git repo in download_dirs')

    # 'build' command:
    parser_build = subparsers.add_parser(
        'build',
        help='unpack, packagesrc, configure, build, and package')
    parser_build.add_argument(
        'source',
        nargs='?',
        help='Vim source tarball or Git repo')

    # 'unpack' command:
    parser_unpack = subparsers.add_parser(
        'unpack',
        help='unpack source to vim-x.y.z/')
    parser_unpack.add_argument(
        'source',
        nargs='?',
        help='Vim source tarball or Git repo')

    # 'packagesrc' command:
    subparsers.add_parser(
        'packagesrc',
        help='create source package vim-x.y.z.tar.bz2 in PWD (Note: '
             'runs "make clean" before packaging!)')

    # 'configure' command:
    subparsers.add_parser(
        'configure',
        help='run "./configure [options]" in PWD')

    # 'make' command:
    subparsers.add_parser(
        'make',
        help='run "make" in PWD')

    # 'package' command:
    subparsers.add_parser(
        'package',
        help='create binary package vim-x.y.z.arch.tar.bz2 in PWD')

    global args
    args = parser.parse_args(namespace=Args())

    if args.verbose is None:
        logging.getLogger().setLevel(logging.INFO)
    else:
        logging.getLogger().setLevel(args.verbose)

    setup_fedora_environment()

    if args.cmd == 'update':
        update()

    elif args.cmd == 'build':
        dest_dir = unpack(resolve_source(args.source))
        package_src(dest_dir)
        configure(dest_dir)
        make(dest_dir)
        package(dest_dir)

    elif args.cmd == 'unpack':
        unpack(resolve_source(args.source))

    elif args.cmd == 'configure':
        configure()

    elif args.cmd == 'make':
        make()

    elif args.cmd == 'package':
        package()

    elif args.cmd == 'packagesrc':
        package_src()


def main():
    try:
        inner_main()
    except BuildError as e:
        print('%s: %s' % (prog_name, e))
        sys.exit(2)


if __name__ == '__main__':
    main()