feat(mkdocs): cli postprocessing support

That way, a single huge markdown file containing documentation for
commands and methods can be split up into multiple files for
individual inclusion in mkdocs.

It's done by a post-processor which is loaded by mako-render, providing
access to the entire context. Said processor may also drop results
altogether and thus prevent files to be written that have been split up
by it.

.gitignore

@@ -6,6 +6,7 @@ gen/*-cli/
 *.go
 *.pyc
 **target/
+**docs/
 **build_html/
 .*.deps
 **Cargo.lock

etc/api/type-cli.yaml

@@ -2,6 +2,10 @@ mkdocs:
   ## A directory to bring us from the mkdocs invocation directory to the gen-root
   gen_root_dir: ..
   site_dir: build_html
+  # if docs_dir changes, remember to update the sources as well.
+  docs_dir: docs
+mako:
+  post_processor_module: cli
 make:
   id: cli
   target_name: CLIs
@@ -12,6 +16,7 @@ make:
   templates:
     - source: ../LICENSE.md
     - source: ../Cargo.toml
+    - source: docs/commands.yml
     - source: mkdocs.yml
     - source: README.md
     - source: main.rs

etc/bin/mako-render

@@ -271,6 +271,11 @@ def cmdline(argv=None):
                     "parent directory of the file provided.")
     parser.add_argument('-io', nargs="+",
                         help="input and ouptut pairs. can be used multiple times, use TEMPLATE_FILE_IN=[OUTPUT_FILE])")
+    parser.add_argument('--post-process-python-module', default="",
+                        help="Specify a python module with a `module.process_template_result(r, output_file|None) -> None|r'."
+                             "If it returns None, no output file will be written. Use it to perform any operation on "
+                             "the template's result. The module, like 'foo.handler' will be imported and thus "
+                             " needs to be in the PYTHONPATH.")
 
     options = parser.parse_args(argv)
     if len(options.io) == 0:
@@ -284,6 +289,16 @@ def cmdline(argv=None):
     data_converted.update(dict([varsplit(var) for var in options.var]))
     del data
 
+    post_processor = lambda r, of: r
+    if options.post_process_python_module:
+        fn_name = 'process_template_result'
+        pm = __import__(options.post_process_python_module, globals(), locals(), [])
+        post_processor = getattr(pm, fn_name, None)
+        if post_processor is None:
+            raise AssertionError("python module '%s' must have a function called '%s'" 
+                                 % (options.post_process_python_module, fn_name))
+    # end handle post processor
+
     seen_stdin = False
     for input_file, output_file in options.io:
         if input_file == '-':
@@ -306,7 +321,9 @@ def cmdline(argv=None):
                 _exit()
 
         try:
-            result = template.render(**data_converted)
+            result = post_processor(template.render(**data_converted), output_file)
+            if result is None:
+                continue
             if output_file:
                 dir = dirname(output_file)
                 if dir and not os.path.isdir(dir):

src/mako/cli/main.rs.mako

@@ -1,10 +1,14 @@
 <%namespace name="docopt" file="lib/docopt.mako"/>\
+<%namespace name="util" file="../lib/util.mako"/>\
 <%  
-    from util import new_context
+    from util import (new_context, rust_comment)
 
     c = new_context(schemas, resources, context.get('methods'))
     default_user_agent = "google-cli-rust-client/" + cargo.build_version
 %>\
+<%block filter="rust_comment">\
+<%util:gen_info source="${self.uri}" />\
+</%block>
 #![feature(plugin)]
 #![plugin(docopt_macros)]
 

src/mako/cli/mkdocs.yml.mako

@@ -1,17 +1,24 @@
-<%! from util import put_and %>\
+<%
+    from util import (put_and, new_context)
+    from cli import (subcommand_md_filename, mangle_subcommand)
+
+    c = new_context(schemas, resources, context.get('methods'))
+%>\
 <%namespace name="util" file="../lib/util.mako"/>\
 site_name: ${util.canonical_name()} v${util.crate_version()}
 site_url: ${cargo.doc_base_url}/${util.crate_name()}
 site_description: Write integrating applications with bcore
 
 repo_url: ${util.github_source_root_url()}
 
-docs_dir: docs
+docs_dir: ${mkdocs.docs_dir}
 site_dir: ${mkdocs.site_dir}
 
 pages:
 - ['index.md', 'Home']
-## - ['be.md', 'Features', 'BE - universal commandline tool']
+% for resource in sorted(c.rta_map.keys()):
+- ['${subcommand_md_filename(resource)}', 'Commands', '${' '.join(s.capitalize() for s in mangle_subcommand(resource).split('-'))}']
+% endfor
 
 theme: readthedocs
 

src/mako/deps.mako

@@ -28,6 +28,10 @@
 	agsuffix = make.aggregated_target_suffix
 	global_targets = make.get('global_targets', False)
 
+	post_processor_arg = ''
+	if mako is not UNDEFINED:
+		post_processor_arg = '--post-process-python-module=%s' % mako.post_processor_module
+
 	try:
 		root = directories.mako_src + '/' + make.id + '/lib'
 		lib_files = [os.path.join(root, file_name) for file_name in os.listdir(root)]
@@ -85,7 +89,7 @@ ${api_common}: $(RUST_SRC)/${make.id}/cmn.rs $(lastword $(MAKEFILE_LIST)) ${gen_
 
 ${gen_root_stamp}: ${' '.join(i[0] for i in sds)} ${' '.join(lib_files)} ${api_json_inputs} $(MAKO_STANDARD_DEPENDENCIES) ${depends_on_target}
 	@echo Generating ${api_target}
-	@$(MAKO) -io ${' '.join("%s=%s" % (s, d) for s, d in sds)} --data-files ${api_json_inputs}
+	@$(MAKO) -io ${' '.join("%s=%s" % (s, d) for s, d in sds)} ${post_processor_arg} --data-files ${api_json_inputs}
 	@touch $@
 
 ${api_target}: ${api_common}
@@ -105,7 +109,7 @@ ${api_doc_index}: ${api_common}
 	% else:
 	@echo mkdocs ${api_doc_index}
 	## Our README is the landing page, and thus will serve multiple roles at once !
-	@cd ${gen_root} && (mkdir -p docs && cd docs && ln -s ../README.md index.md &>/dev/null) || : && $(MKDOCS) build --clean
+	@cd ${gen_root} && (mkdir -p ${mkdocs.docs_dir} && cd ${mkdocs.docs_dir} && ln -s ../README.md index.md &>/dev/null) || : && $(MKDOCS) build --clean
 	% endif
 
 ${api_doc}: ${api_doc_index}

src/mako/lib/cli.py

@@ -1,5 +1,41 @@
 import util
 
+import os
+import re
+
+SPLIT_START = '>>>>>>>'
+SPLIT_END = '<<<<<<<'
+
+re_splitters = re.compile(r"%s ([\w\-\.]+)\n(.*?)\n%s" % (SPLIT_START, SPLIT_END), re.MULTILINE|re.DOTALL)
+
 # transform name to be a suitable subcommand
 def mangle_subcommand(name):
     return util.camel_to_under(name).replace('_', '-').replace('.', '-')
+
+
+# transform the resource name into a suitable filename to contain the markdown documentation for it
+def subcommand_md_filename(resource):
+    return mangle_subcommand(resource) + '.md'
+
+
+# split the result along split segments
+def process_template_result(r, output_file):
+    found = False
+    dir = None
+    if output_file:
+        dir = os.path.dirname(output_file)
+        if not os.path.isdir(dir):
+            os.makedirs(dir)
+    # end handle output directory
+
+    for m in re_splitters.finditer(r):
+        found = True
+        fh = open(os.path.join(dir, m.group(1)), 'wb')
+        fh.write(m.group(2))
+        fh.close()
+    # end for each match
+
+    if found:
+        r = None
+
+    return r