diff --git a/Makefile b/Makefile index 0704fe03..2f1cf0bc 100644 --- a/Makefile +++ b/Makefile @@ -12,3 +12,11 @@ release-pypi: $(PYTHON) setup.py sdist python setup.py bdist_wheel --universal twine upload dist/* + +update-buildsupport: + git subtree pull \ + -m "Update DataLad build helper" \ + --squash \ + --prefix _datalad_build_support \ + https://github.com/datalad/datalad-buildsupport.git \ + master diff --git a/_datalad_build_support/__init__.py b/_datalad_build_support/__init__.py new file mode 100644 index 00000000..c91633ab --- /dev/null +++ b/_datalad_build_support/__init__.py @@ -0,0 +1,13 @@ +# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## +# +# See COPYING file distributed along with the DataLad package for the +# copyright and license terms. +# +# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## +"""Python package for functionality needed at package 'build' time by DataLad and its extensions + +__init__ here should be really minimalistic, not import submodules by default +and submodules should also not require heavy dependencies. +""" + +__version__ = '0.1' diff --git a/_datalad_build_support/formatters.py b/_datalad_build_support/formatters.py new file mode 100644 index 00000000..015f62a4 --- /dev/null +++ b/_datalad_build_support/formatters.py @@ -0,0 +1,314 @@ +# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## +# +# See COPYING file distributed along with the DataLad package for the +# copyright and license terms. +# +# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## + +import argparse +import datetime +import re + + +class ManPageFormatter(argparse.HelpFormatter): + # This code was originally distributed + # under the same License of Python + # Copyright (c) 2014 Oz Nahum Tiram + def __init__(self, + prog, + indent_increment=2, + max_help_position=4, + width=1000000, + section=1, + ext_sections=None, + authors=None, + version=None + ): + + super(ManPageFormatter, self).__init__( + prog, + indent_increment=indent_increment, + max_help_position=max_help_position, + width=width) + + self._prog = prog + self._section = 1 + self._today = datetime.date.today().strftime('%Y\\-%m\\-%d') + self._ext_sections = ext_sections + self._version = version + + def _get_formatter(self, **kwargs): + return self.formatter_class(prog=self.prog, **kwargs) + + def _markup(self, txt): + return txt.replace('-', '\\-') + + def _underline(self, string): + return "\\fI\\s-1" + string + "\\s0\\fR" + + def _bold(self, string): + if not string.strip().startswith('\\fB'): + string = '\\fB' + string + if not string.strip().endswith('\\fR'): + string = string + '\\fR' + return string + + def _mk_synopsis(self, parser): + self.add_usage(parser.usage, parser._actions, + parser._mutually_exclusive_groups, prefix='') + usage = self._format_usage(None, parser._actions, + parser._mutually_exclusive_groups, '') + # replace too long list of commands with a single placeholder + usage = re.sub(r'{[^]]*?create,.*?}', ' COMMAND ', usage, flags=re.MULTILINE) + # take care of proper wrapping + usage = re.sub(r'\[([-a-zA-Z0-9]*)\s([a-zA-Z0-9{}|_]*)\]', r'[\1\~\2]', usage) + + usage = usage.replace('%s ' % self._prog, '') + usage = '.SH SYNOPSIS\n.nh\n.HP\n\\fB%s\\fR %s\n.hy\n' % (self._markup(self._prog), + usage) + return usage + + def _mk_title(self, prog): + name_version = "{0} {1}".format(prog, self._version) + return '.TH "{0}" "{1}" "{2}" "{3}"\n'.format( + prog, self._section, self._today, name_version) + + def _mk_name(self, prog, desc): + """ + this method is in consitent with others ... it relies on + distribution + """ + desc = desc.splitlines()[0] if desc else 'it is in the name' + # ensure starting lower case + desc = desc[0].lower() + desc[1:] + return '.SH NAME\n%s \\- %s\n' % (self._bold(prog), desc) + + def _mk_description(self, parser): + desc = parser.description + desc = '\n'.join(desc.splitlines()[1:]) + if not desc: + return '' + desc = desc.replace('\n\n', '\n.PP\n') + # sub-section headings + desc = re.sub(r'^\*(.*)\*$', r'.SS \1', desc, flags=re.MULTILINE) + # italic commands + desc = re.sub(r'^ ([-a-z]*)$', r'.TP\n\\fI\1\\fR', desc, flags=re.MULTILINE) + # deindent body text, leave to troff viewer + desc = re.sub(r'^ (\S.*)\n', '\\1\n', desc, flags=re.MULTILINE) + # format NOTEs as indented paragraphs + desc = re.sub(r'^NOTE\n', '.TP\nNOTE\n', desc, flags=re.MULTILINE) + # deindent indented paragraphs after heading setup + desc = re.sub(r'^ (.*)$', '\\1', desc, flags=re.MULTILINE) + + return '.SH DESCRIPTION\n%s\n' % self._markup(desc) + + def _mk_footer(self, sections): + if not hasattr(sections, '__iter__'): + return '' + + footer = [] + for section, value in sections.items(): + part = ".SH {}\n {}".format(section.upper(), value) + footer.append(part) + + return '\n'.join(footer) + + def format_man_page(self, parser): + page = [] + page.append(self._mk_title(self._prog)) + page.append(self._mk_name(self._prog, parser.description)) + page.append(self._mk_synopsis(parser)) + page.append(self._mk_description(parser)) + page.append(self._mk_options(parser)) + page.append(self._mk_footer(self._ext_sections)) + + return ''.join(page) + + def _mk_options(self, parser): + + formatter = parser._get_formatter() + + # positionals, optionals and user-defined groups + for action_group in parser._action_groups: + formatter.start_section(None) + formatter.add_text(None) + formatter.add_arguments(action_group._group_actions) + formatter.end_section() + + # epilog + formatter.add_text(parser.epilog) + + # determine help from format above + help = formatter.format_help() + # add spaces after comma delimiters for easier reformatting + help = re.sub(r'([a-z]),([a-z])', '\\1, \\2', help) + # get proper indentation for argument items + help = re.sub(r'^ (\S.*)\n', '.TP\n\\1\n', help, flags=re.MULTILINE) + # deindent body text, leave to troff viewer + help = re.sub(r'^ (\S.*)\n', '\\1\n', help, flags=re.MULTILINE) + return '.SH OPTIONS\n' + help + + def _format_action_invocation(self, action, doubledash='--'): + if not action.option_strings: + metavar, = self._metavar_formatter(action, action.dest)(1) + return metavar + + else: + parts = [] + + # if the Optional doesn't take a value, format is: + # -s, --long + if action.nargs == 0: + parts.extend([self._bold(action_str) for action_str in + action.option_strings]) + + # if the Optional takes a value, format is: + # -s ARGS, --long ARGS + else: + default = self._underline(action.dest.upper()) + args_string = self._format_args(action, default) + for option_string in action.option_strings: + parts.append('%s %s' % (self._bold(option_string), + args_string)) + + return ', '.join(p.replace('--', doubledash) for p in parts) + + +class RSTManPageFormatter(ManPageFormatter): + def _get_formatter(self, **kwargs): + return self.formatter_class(prog=self.prog, **kwargs) + + def _markup(self, txt): + # put general tune-ups here + return txt + + def _underline(self, string): + return "*{0}*".format(string) + + def _bold(self, string): + return "**{0}**".format(string) + + def _mk_synopsis(self, parser): + self.add_usage(parser.usage, parser._actions, + parser._mutually_exclusive_groups, prefix='') + usage = self._format_usage(None, parser._actions, + parser._mutually_exclusive_groups, '') + + usage = usage.replace('%s ' % self._prog, '') + usage = 'Synopsis\n--------\n::\n\n %s %s\n' \ + % (self._markup(self._prog), usage) + return usage + + def _mk_title(self, prog): + # and an easy to use reference point + title = ".. _man_%s:\n\n" % prog.replace(' ', '-') + title += "{0}".format(prog) + title += '\n{0}\n\n'.format('=' * len(prog)) + return title + + def _mk_name(self, prog, desc): + return '' + + def _mk_description(self, parser): + desc = parser.description + if not desc: + return '' + return 'Description\n-----------\n%s\n' % self._markup(desc) + + def _mk_footer(self, sections): + if not hasattr(sections, '__iter__'): + return '' + + footer = [] + for section, value in sections.items(): + part = "\n{0}\n{1}\n{2}\n".format( + section, + '-' * len(section), + value) + footer.append(part) + + return '\n'.join(footer) + + def _mk_options(self, parser): + + # this non-obvious maneuver is really necessary! + formatter = self.__class__(self._prog) + + # positionals, optionals and user-defined groups + for action_group in parser._action_groups: + formatter.start_section(None) + formatter.add_text(None) + formatter.add_arguments(action_group._group_actions) + formatter.end_section() + + # epilog + formatter.add_text(parser.epilog) + + # determine help from format above + option_sec = formatter.format_help() + + return '\n\nOptions\n-------\n{0}'.format(option_sec) + + def _format_action(self, action): + # determine the required width and the entry label + action_header = self._format_action_invocation(action, doubledash='-\\\\-') + + if action.help: + help_text = self._expand_help(action) + help_lines = self._split_lines(help_text, 80) + help = ' '.join(help_lines) + else: + help = '' + + # return a single string + return '{0}\n{1}\n{2}\n\n'.format( + action_header, + + '~' * len(action_header), + help) + + +def cmdline_example_to_rst(src, out=None, ref=None): + if out is None: + from io import StringIO + out = StringIO() + + # place header + out.write('.. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n') + if ref: + # place cross-ref target + out.write('.. {0}:\n\n'.format(ref)) + + # parser status vars + inexample = False + incodeblock = False + + for line in src: + if line.startswith('#% EXAMPLE START'): + inexample = True + incodeblock = False + continue + if not inexample: + continue + if line.startswith('#% EXAMPLE END'): + break + if not inexample: + continue + if line.startswith('#%'): + incodeblock = not incodeblock + if incodeblock: + out.write('\n.. code-block:: sh\n\n') + continue + if not incodeblock and line.startswith('#'): + out.write(line[(min(2, len(line) - 1)):]) + continue + if incodeblock: + if not line.rstrip().endswith('#% SKIP'): + out.write(' %s' % line) + continue + if not len(line.strip()): + continue + else: + raise RuntimeError("this should not happen") + + return out diff --git a/_datalad_build_support/setup.py b/_datalad_build_support/setup.py new file mode 100644 index 00000000..b83ccfa8 --- /dev/null +++ b/_datalad_build_support/setup.py @@ -0,0 +1,255 @@ +# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## +# +# See COPYING file distributed along with the DataLad package for the +# copyright and license terms. +# +# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## + + +import datetime +import os + +from distutils.core import Command +from distutils.errors import DistutilsOptionError +from os.path import ( + dirname, + join as opj, +) +from setuptools.config import read_configuration + +import versioneer + +from . import formatters as fmt + + +class BuildManPage(Command): + # The BuildManPage code was originally distributed + # under the same License of Python + # Copyright (c) 2014 Oz Nahum Tiram + + description = 'Generate man page from an ArgumentParser instance.' + + user_options = [ + ('manpath=', None, + 'output path for manpages (relative paths are relative to the ' + 'datalad package)'), + ('rstpath=', None, + 'output path for RST files (relative paths are relative to the ' + 'datalad package)'), + ('parser=', None, 'module path to an ArgumentParser instance' + '(e.g. mymod:func, where func is a method or function which return' + 'a dict with one or more arparse.ArgumentParser instances.'), + ('cmdsuite=', None, 'module path to an extension command suite ' + '(e.g. mymod:command_suite) to limit the build to the contained ' + 'commands.'), + ] + + def initialize_options(self): + self.manpath = opj('build', 'man') + self.rstpath = opj('docs', 'source', 'generated', 'man') + self.parser = 'datalad.cmdline.main:setup_parser' + self.cmdsuite = None + + def finalize_options(self): + if self.manpath is None: + raise DistutilsOptionError('\'manpath\' option is required') + if self.rstpath is None: + raise DistutilsOptionError('\'rstpath\' option is required') + if self.parser is None: + raise DistutilsOptionError('\'parser\' option is required') + mod_name, func_name = self.parser.split(':') + fromlist = mod_name.split('.') + try: + mod = __import__(mod_name, fromlist=fromlist) + self._parser = getattr(mod, func_name)( + ['datalad'], + formatter_class=fmt.ManPageFormatter, + return_subparsers=True, + # ignore extensions only for the main package to avoid pollution + # with all extension commands that happen to be installed + help_ignore_extensions=self.distribution.get_name() == 'datalad') + + except ImportError as err: + raise err + if self.cmdsuite: + mod_name, suite_name = self.cmdsuite.split(':') + mod = __import__(mod_name, fromlist=mod_name.split('.')) + suite = getattr(mod, suite_name) + self.cmdlist = [c[2] if len(c) > 2 else c[1].replace('_', '-') + for c in suite[1]] + + self.announce('Writing man page(s) to %s' % self.manpath) + self._today = datetime.date.today() + + @classmethod + def handle_module(cls, mod_name, **kwargs): + """Module specific handling. + + This particular one does + 1. Memorize (at class level) the module name of interest here + 2. Check if 'datalad.extensions' are specified for the module, + and then analyzes them to obtain command names it provides + + If cmdline commands are found, its entries are to be used instead of + the ones in datalad's _parser. + + Parameters + ---------- + **kwargs: + all the kwargs which might be provided to setuptools.setup + """ + cls.mod_name = mod_name + + exts = kwargs.get('entry_points', {}).get('datalad.extensions', []) + for ext in exts: + assert '=' in ext # should be label=module:obj + ext_label, mod_obj = ext.split('=', 1) + assert ':' in mod_obj # should be module:obj + mod, obj = mod_obj.split(':', 1) + assert mod_name == mod # AFAIK should be identical + + mod = __import__(mod_name) + if hasattr(mod, obj): + command_suite = getattr(mod, obj) + assert len(command_suite) == 2 # as far as I see it + if not hasattr(cls, 'cmdline_names'): + cls.cmdline_names = [] + cls.cmdline_names += [ + cmd + for _, _, cmd, _ in command_suite[1] + ] + + def run(self): + + dist = self.distribution + #homepage = dist.get_url() + #appname = self._parser.prog + appname = 'datalad' + + cfg = read_configuration( + opj(dirname(dirname(__file__)), 'setup.cfg'))['metadata'] + + sections = { + 'Authors': """{0} is developed by {1} <{2}>.""".format( + appname, cfg['author'], cfg['author_email']), + } + + for cls, opath, ext in ((fmt.ManPageFormatter, self.manpath, '1'), + (fmt.RSTManPageFormatter, self.rstpath, 'rst')): + if not os.path.exists(opath): + os.makedirs(opath) + for cmdname in getattr(self, 'cmdline_names', list(self._parser)): + if hasattr(self, 'cmdlist') and cmdname not in self.cmdlist: + continue + p = self._parser[cmdname] + cmdname = "{0}{1}".format( + 'datalad ' if cmdname != 'datalad' else '', + cmdname) + format = cls( + cmdname, + ext_sections=sections, + version=versioneer.get_version()) + formatted = format.format_man_page(p) + with open(opj(opath, '{0}.{1}'.format( + cmdname.replace(' ', '-'), + ext)), + 'w') as f: + f.write(formatted) + + +class BuildRSTExamplesFromScripts(Command): + description = 'Generate RST variants of example shell scripts.' + + user_options = [ + ('expath=', None, 'path to look for example scripts'), + ('rstpath=', None, 'output path for RST files'), + ] + + def initialize_options(self): + self.expath = opj('docs', 'examples') + self.rstpath = opj('docs', 'source', 'generated', 'examples') + + def finalize_options(self): + if self.expath is None: + raise DistutilsOptionError('\'expath\' option is required') + if self.rstpath is None: + raise DistutilsOptionError('\'rstpath\' option is required') + self.announce('Converting example scripts') + + def run(self): + opath = self.rstpath + if not os.path.exists(opath): + os.makedirs(opath) + + from glob import glob + for example in glob(opj(self.expath, '*.sh')): + exname = os.path.basename(example)[:-3] + with open(opj(opath, '{0}.rst'.format(exname)), 'w') as out: + fmt.cmdline_example_to_rst( + open(example), + out=out, + ref='_example_{0}'.format(exname)) + + +class BuildConfigInfo(Command): + description = 'Generate RST documentation for all config items.' + + user_options = [ + ('rstpath=', None, 'output path for RST file'), + ] + + def initialize_options(self): + self.rstpath = opj('docs', 'source', 'generated', 'cfginfo') + + def finalize_options(self): + if self.rstpath is None: + raise DistutilsOptionError('\'rstpath\' option is required') + self.announce('Generating configuration documentation') + + def run(self): + opath = self.rstpath + if not os.path.exists(opath): + os.makedirs(opath) + + from datalad.interface.common_cfg import definitions as cfgdefs + from datalad.dochelpers import _indent + + categories = { + 'global': {}, + 'local': {}, + 'dataset': {}, + 'misc': {} + } + for term, v in cfgdefs.items(): + categories[v.get('destination', 'misc')][term] = v + + for cat in categories: + with open(opj(opath, '{}.rst.in'.format(cat)), 'w') as rst: + rst.write('.. glossary::\n') + for term, v in sorted(categories[cat].items(), key=lambda x: x[0]): + rst.write(_indent(term, '\n ')) + qtype, docs = v.get('ui', (None, {})) + desc_tmpl = '\n' + if 'title' in docs: + desc_tmpl += '{title}:\n' + if 'text' in docs: + desc_tmpl += '{text}\n' + if 'default' in v: + default = v['default'] + if hasattr(default, 'replace'): + # protect against leaking specific home dirs + v['default'] = default.replace(os.path.expanduser('~'), '~') + desc_tmpl += 'Default: {default}\n' + if 'type' in v: + type_ = v['type'] + if hasattr(type_, 'long_description'): + type_ = type_.long_description() + else: + type_ = type_.__name__ + desc_tmpl += '\n[{type}]\n' + v['type'] = type_ + if desc_tmpl == '\n': + # we need something to avoid joining terms + desc_tmpl += 'undocumented\n' + v.update(docs) + rst.write(_indent(desc_tmpl.format(**v), ' ')) diff --git a/docs/source/conf.py b/docs/source/conf.py index 69a21ffa..1e674363 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -12,9 +12,16 @@ # All configuration values have a default; values that are commented out # serve to show the default. +import sys +import os + import datetime -from os.path import join as opj, exists -from os.path import dirname +from os.path import ( + abspath, + dirname, + exists, + join as opj, +) from os import pardir import datalad_helloworld @@ -24,17 +31,23 @@ # documentation root, use os.path.abspath to make it absolute, like shown here. #sys.path.insert(0, os.path.abspath('.')) -## generate missing pieces -#for setup_py_path in (opj(pardir, 'setup.py'), # travis -# opj(pardir, pardir, 'setup.py')): # RTD -# if exists(setup_py_path): -# sys.path.insert(0, os.path.abspath(dirname(setup_py_path))) -# try: -# for cmd in 'manpage',: #'examples': -# os.system('{} build_{}'.format(setup_py_path, cmd)) -# except: -# # shut up and do your best -# pass +# generate missing pieces +for setup_py_path in (opj(pardir, 'setup.py'), # travis + opj(pardir, pardir, 'setup.py')): # RTD + if exists(setup_py_path): + sys.path.insert(0, os.path.abspath(dirname(setup_py_path))) + try: + for cmd in 'manpage',: #'examples': + os.system( + '{} build_{} --cmdsuite {} --manpath {} --rstpath {}'.format( + setup_py_path, + cmd, + 'datalad_helloworld:command_suite', + abspath(opj(dirname(setup_py_path), 'build', 'man')), + opj(dirname(__file__), 'generated', 'man'))) + except: + # shut up and do your best + pass # -- General configuration ------------------------------------------------ diff --git a/docs/source/index.rst b/docs/source/index.rst index 4f3decca..ce2fdb3d 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -18,6 +18,15 @@ High-level API commands hello_cmd +Command line reference +---------------------- + +.. toctree:: + :maxdepth: 1 + + generated/man/datalad-hello-cmd + + Indices and tables ================== diff --git a/setup.py b/setup.py index 0b24fc2c..6f165b5c 100755 --- a/setup.py +++ b/setup.py @@ -4,6 +4,13 @@ from setuptools import setup import versioneer +from _datalad_build_support.setup import ( + BuildManPage, +) + +cmdclass = versioneer.get_cmdclass() +cmdclass.update(build_manpage=BuildManPage) + # Give setuptools a hint to complain if it's too old a version # 30.3.0 allows us to put most metadata in setup.cfg # Should match pyproject.toml @@ -14,7 +21,7 @@ if __name__ == '__main__': setup(name='datalad_helloworld', version=versioneer.get_version(), - cmdclass=versioneer.get_cmdclass(), + cmdclass=cmdclass, setup_requires=SETUP_REQUIRES, entry_points={ # 'datalad.extensions' is THE entrypoint inspected by the datalad API builders