diff --git a/.gitignore b/.gitignore index 491116c163fd..7149b667bf29 100644 --- a/.gitignore +++ b/.gitignore @@ -242,3 +242,8 @@ conda/pkg .tvm_test_data .dgl .caffe2 + +# Local docs build +_docs/ +jvm/target +.config/configstore/ diff --git a/Makefile b/Makefile index 8ebbea610f57..eba5d5710fdd 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ .PHONY: all \ runtime vta cpptest crttest \ lint pylint cpplint scalalint \ - doc \ + cppdoc docs \ web webclean \ cython cython3 cyclean \ clean @@ -114,7 +114,7 @@ scalalint: mypy: tests/scripts/task_mypy.sh -doc: +cppdoc: doxygen docs/Doxyfile @@ -175,3 +175,6 @@ jvminstall: # Final cleanup rules, delegate to more specific rules. clean: cmake_clean cyclean webclean + +docs: + python3 tests/scripts/ci.py docs diff --git a/docker/bash.sh b/docker/bash.sh index cbd71870747c..6f31aa7a5180 100755 --- a/docker/bash.sh +++ b/docker/bash.sh @@ -85,6 +85,10 @@ Usage: docker/bash.sh [-i|--interactive] [--net=host] [-t|--tty] Print the docker command to be run, but do not execute it. +--env + + Pass an environment variable through to the container. + --name Set the name of the docker container, and the hostname that will @@ -149,6 +153,7 @@ function parse_error() { # to overwrite the parent scope's behavior. break_joined_flag='if (( ${#1} == 2 )); then shift; else set -- -"${1#-i}" "${@:2}"; fi' +DOCKER_ENV=( ) while (( $# )); do case "$1" in @@ -195,6 +200,11 @@ while (( $# )); do fi ;; + --env) + DOCKER_ENV+=( --env "$2" ) + shift 2 + ;; + --dry-run) DRY_RUN=true shift @@ -263,7 +273,6 @@ fi source "$(dirname $0)/dev_common.sh" || exit 2 DOCKER_FLAGS=( ) -DOCKER_ENV=( ) DOCKER_MOUNT=( ) DOCKER_DEVICES=( ) diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 000000000000..0573e9595aa3 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,101 @@ + + + + + + + + + + + + + + + + + +# TVM Documentation +This folder contains the source of TVM's documentation, hosted at https://tvm.apache.org/docs + +## Build Locally + +See also the instructions below to run a specific tutorial. Note that some of the tutorials need GPU support. Once build, either of these can be served using Python's build in HTTP server: + +```bash +# Run this and then visit http://localhost:8000 in your browser +cd docs/_build/html && python3 -m http.server +``` + +### With Docker (recommended) + +1. Build TVM and the docs inside the [tlcpack/ci-gpu image](https://hub.docker.com/r/tlcpack/ci-gpu) + + ```bash + # If this runs into errors, try cleaning your 'build' directory + make docs + ``` + + +### Native + +1. [Build TVM](https://tvm.apache.org/docs/install/from_source.html) first in the repo root folder +2. Install dependencies + + ```bash + # Pillow on Ubuntu may require libjpeg-dev from apt + docker run tlcpack/ci-gpu:v0.78 bash -c \ + 'python3 -m pip install --quiet tlcpack-sphinx-addon==0.2.1 synr==0.5.0 && python3 -m pip freeze' > frozen-requirements.txt + + pip install -r frozen-requirements.txt + ``` + +3. Generate the docs + + ```bash + # TVM_TUTORIAL_EXEC_PATTERN=none skips the tutorial execution to the build + # work on most environments (e.g. MacOS). + export TVM_TUTORIAL_EXEC_PATTERN=none + + make html + ``` + + +## Only Execute Specified Tutorials +The document build process will execute all the tutorials in the sphinx gallery. +This will cause failure in some cases when certain machines do not have necessary +environment. You can set `TVM_TUTORIAL_EXEC_PATTERN` to only execute +the path that matches the regular expression pattern. + +For example, to only build tutorials under `/vta/tutorials`, run + +```bash +TVM_TUTORIAL_EXEC_PATTERN=/vta/tutorials make html +``` + +To only build one specific file, do + +```bash +# The slash \ is used to get . in regular expression +TVM_TUTORIAL_EXEC_PATTERN=file_name\.py make html +``` + +## Helper Scripts + +You can run the following script to reproduce the CI sphinx pre-check stage. +This script skips the tutorial executions and is useful to quickly check the content. + +```bash +./tests/scripts/task_sphinx_precheck.sh +``` + +The following script runs the full build which includes tutorial executions. +You will need a GPU CI environment. + +```bash +python tests/scripts/ci.py --precheck --full +``` + +## Define the Order of Tutorials +You can define the order of tutorials with `conf.py::subsection_order` and `conf.py::within_subsection_order`. +By default, the tutorials within one subsection is sorted by filename. diff --git a/docs/README.txt b/docs/README.txt deleted file mode 100644 index 06d7a0f6e444..000000000000 --- a/docs/README.txt +++ /dev/null @@ -1,59 +0,0 @@ -TVM Documentations -================== -This folder contains the source of TVM documents - -- A hosted version of doc is at https://tvm.apache.org/docs -- pip install "sphinx>=1.5.5" sphinx-gallery sphinx_rtd_theme matplotlib Image recommonmark "Pillow<7" "autodocsumm<0.2.0" tlcpack-sphinx-addon "docutils<0.17" -- (Versions 0.2.0 to 0.2.2 of autodocsumm are incompatible with sphinx>=3.4, https://github.com/Chilipp/autodocsumm/pull/42 ) -- Build tvm first in the root folder. -- Run the following command -```bash -TVM_TUTORIAL_EXEC_PATTERN=none make html -``` - -```TVM_TUTORIAL_EXEC_PATTERN=none``` skips the tutorial execution to make it work on most environment(e.g. Mac book). - -See also the instructions below to run a specific tutorial. Note that some of the tutorials need GPU support. - - -Only Execute Specified Tutorials --------------------------------- -The document build process will execute all the tutorials in the sphinx gallery. -This will cause failure in some cases when certain machines do not have necessary -environment. You can set ```TVM_TUTORIAL_EXEC_PATTERN``` to only execute -the path that matches the regular expression pattern. - -For example, to only build tutorials under /vta/tutorials, run - -```bash -TVM_TUTORIAL_EXEC_PATTERN=/vta/tutorials make html -``` - -To only build one specific file, do - -```bash -# The slash \ is used to get . in regular expression -TVM_TUTORIAL_EXEC_PATTERN=file_name\.py make html -``` - -Helper Scripts --------------- - -You can run the following script to reproduce the CI sphinx pre-check stage. -This script skips the tutorial executions and is useful for quickly check the content. - -```bash -./tests/scripts/task_sphinx_precheck.sh -``` - -The following script runs the full build which includes tutorial executions. -You will need a gpu CI environment. - -```bash -./tests/scripts/task_python_docs.sh -``` - -Define the Order of Tutorials ------------------------------ -You can define the order of tutorials with `conf.py::subsection_order` and `conf.py::within_subsection_order`. -By default, the tutorials within one subsection is sorted by filename. diff --git a/python/tvm/script/parser.py b/python/tvm/script/parser.py index 0132025024b2..917a16d478c0 100644 --- a/python/tvm/script/parser.py +++ b/python/tvm/script/parser.py @@ -155,7 +155,9 @@ class TVMScriptParser(Transformer): ast.BuiltinOp.Not: tvm.tir.Not, } - def __init__(self, base_lineno, tir_namespace): + # pylint gets confused here with synr.Transformer which doesn't have a + # custom init, so just disable it + def __init__(self, base_lineno, tir_namespace): # pylint: disable=super-init-not-called self.context = None self.base_lineno = base_lineno diff --git a/tests/lint/cppdocs.sh b/tests/lint/cppdocs.sh index e453b7bd2f90..d29b6cdd236b 100755 --- a/tests/lint/cppdocs.sh +++ b/tests/lint/cppdocs.sh @@ -16,13 +16,14 @@ # specific language governing permissions and limitations # under the License. +set -euxo pipefail function cleanup() { rm -f /tmp/$$.log.txt /tmp/$$.logclean.txt } trap cleanup EXIT -make doc 2>/tmp/$$.log.txt +make cppdoc 2>/tmp/$$.log.txt grep -v -E "ENABLE_PREPROCESSING|unsupported tag" < /tmp/$$.log.txt > /tmp/$$.logclean.txt || true echo "---------Error Log----------" diff --git a/tests/lint/pylint.sh b/tests/lint/pylint.sh index e47d576ced2f..2cb38802bf90 100755 --- a/tests/lint/pylint.sh +++ b/tests/lint/pylint.sh @@ -15,8 +15,8 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +set -euxo pipefail - -python3 -m pylint python/tvm --rcfile=$(dirname "$0")/pylintrc -python3 -m pylint vta/python/vta --rcfile=$(dirname "$0")/pylintrc -python3 -m pylint tests/python/unittest/test_tvmscript_type.py --rcfile=$(dirname "$0")/pylintrc +python3 -m pylint python/tvm --rcfile="$(dirname "$0")"/pylintrc +python3 -m pylint vta/python/vta --rcfile="$(dirname "$0")"/pylintrc +python3 -m pylint tests/python/unittest/test_tvmscript_type.py --rcfile="$(dirname "$0")"/pylintrc diff --git a/tests/scripts/ci.py b/tests/scripts/ci.py new file mode 100644 index 000000000000..c9d0ee4ccc2b --- /dev/null +++ b/tests/scripts/ci.py @@ -0,0 +1,232 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import sys +import multiprocessing +import os +import getpass +import inspect +import argparse +import grp +import subprocess +from pathlib import Path +from typing import List, Dict, Any, Optional + +REPO_ROOT = Path(__file__).resolve().parent.parent.parent +NPROC = multiprocessing.cpu_count() + + +class col: + BLUE = "\033[94m" + CYAN = "\033[96m" + GREEN = "\033[92m" + YELLOW = "\033[93m" + RED = "\033[91m" + RESET = "\033[0m" + BOLD = "\033[1m" + UNDERLINE = "\033[4m" + + +def print_color(color: str, msg: str, **kwargs: Any) -> None: + if hasattr(sys.stdout, "isatty") and sys.stdout.isatty(): + print(col.BOLD + color + msg + col.RESET, **kwargs) + else: + print(msg, **kwargs) + + +def clean_exit(msg: str) -> None: + print_color(col.RED, msg, file=sys.stderr) + exit(1) + + +def cmd(commands: List[Any], **kwargs: Any): + commands = [str(s) for s in commands] + command_str = " ".join(commands) + print_color(col.BLUE, command_str) + proc = subprocess.run(commands, **kwargs) + if proc.returncode != 0: + raise RuntimeError(f"Command failed: '{command_str}'") + + +def docker(name: str, image: str, scripts: List[str], env: Dict[str, str]): + """ + Invoke a set of bash scripts through docker/bash.sh + """ + if sys.platform == "linux": + # Check that the user is in the docker group before running + try: + group = grp.getgrnam("docker") + if getpass.getuser() not in group.gr_mem: + print_color( + col.YELLOW, f"Note: User '{getpass.getuser()}' is not in the 'docker' group" + ) + except KeyError: + print_color(col.YELLOW, f"Note: 'docker' group does not exist") + + docker_bash = REPO_ROOT / "docker" / "bash.sh" + command = [docker_bash, "--name", name] + for key, value in env.items(): + command.append("--env") + command.append(f"{key}={value}") + command += [image, "bash", "-c", " && ".join(scripts)] + + try: + cmd(command) + except RuntimeError as e: + clean_exit(f"Error invoking Docker: {e}") + except KeyboardInterrupt: + cmd(["docker", "stop", "--time", "1", name]) + + +def docs( + tutorial_pattern: Optional[str] = None, + full: bool = False, + precheck: bool = False, + cpu: bool = False, +) -> None: + """ + Build the documentation from gallery/ and docs/. By default this builds only + the Python docs. If you are on a CPU machine, you can skip the tutorials + and build the docs with the '--precheck --cpu' options. + + arguments: + full -- Build all language docs, not just Python + precheck -- Run Sphinx precheck script + tutorial-pattern -- Regex for which tutorials to execute when building docs (can also be set via TVM_TUTORIAL_EXEC_PATTERN) + cpu -- Use CMake defaults for building TVM (useful for building docs on a CPU machine.) + """ + config = "./tests/scripts/task_config_build_gpu.sh" + if cpu and full: + clean_exit("--full cannot be used with --cpu") + + if cpu: + # The docs import tvm.micro, so it has to be enabled in the build + config = "cd build && cp ../cmake/config.cmake . && echo set\(USE_MICRO ON\) >> config.cmake && cd .." + + scripts = [ + config, + f"./tests/scripts/task_build.sh build -j{NPROC}", + "./tests/scripts/task_ci_setup.sh", + "./tests/scripts/task_sphinx_precheck.sh" + if precheck + else "./tests/scripts/task_python_docs.sh", + ] + + if tutorial_pattern is None: + tutorial_pattern = os.getenv("TVM_TUTORIAL_EXEC_PATTERN", ".py" if full else "none") + + env = { + "TVM_TUTORIAL_EXEC_PATTERN": tutorial_pattern, + "PYTHON_DOCS_ONLY": "0" if full else "1", + "IS_LOCAL": "1", + } + docker(name="ci-docs", image="ci_gpu", scripts=scripts, env=env) + + +def serve_docs(directory: str = "_docs") -> None: + """ + Serve the docs using Python's http server + + arguments: + directory -- Directory to serve from + """ + directory = Path(directory) + if not directory.exists(): + clean_exit("Docs have not been build, run 'ci.py docs' first") + cmd([sys.executable, "-m", "http.server"], cwd=directory) + + +def lint() -> None: + """ + Run CI's Sanity Check step + """ + docker( + name="ci-lint", + image="ci_lint", + scripts=["./tests/scripts/task_lint.sh"], + env={}, + ) + + +def cli_name(s: str) -> str: + return s.replace("_", "-") + + +def add_subparser(func, subparsers) -> Any: + """ + Utility function to make it so subparser commands can be defined locally + as a function rather than directly via argparse and manually dispatched + out. + """ + + # Each function is intended follow the example for arguments in PEP257, so + # split apart the function documentation from the arguments + split = [s.strip() for s in func.__doc__.split("arguments:\n")] + if len(split) == 1: + args_help = None + command_help = split[0] + else: + command_help, args_help = split + + # Parse out the help text for each argument if present + arg_help_texts = {} + if args_help is not None: + for line in args_help.split("\n"): + line = line.strip() + name, help_text = [t.strip() for t in line.split(" -- ")] + arg_help_texts[name] = help_text + + subparser = subparsers.add_parser(cli_name(func.__name__), help=command_help) + + # Add each parameter to the subparser + signature = inspect.signature(func) + for name, value in signature.parameters.items(): + kwargs = {"help": arg_help_texts[cli_name(name)]} + + # Grab the default value if present + if value.default is not value.empty: + kwargs["default"] = value.default + + # Check if it should be a flag + if value.annotation is bool: + kwargs["action"] = "store_true" + subparser.add_argument(f"--{cli_name(name)}", **kwargs) + + return subparser + + +def main(): + parser = argparse.ArgumentParser(description="Run CI scripts locally via Docker") + subparsers = parser.add_subparsers(dest="command") + + subparser_functions = {cli_name(func.__name__): func for func in [docs, serve_docs, lint]} + for func in subparser_functions.values(): + add_subparser(func, subparsers) + + args = parser.parse_args() + func = subparser_functions[args.command] + + # Extract out the parsed args and invoke the relevant function + kwargs = {k: getattr(args, k) for k in dir(args) if not k.startswith("_") and k != "command"} + func(**kwargs) + + +if __name__ == "__main__": + main() diff --git a/tests/scripts/task_ci_setup.sh b/tests/scripts/task_ci_setup.sh index 323bc0752801..33ea484a5268 100755 --- a/tests/scripts/task_ci_setup.sh +++ b/tests/scripts/task_ci_setup.sh @@ -28,7 +28,7 @@ set -o pipefail # # command: python3 -m pip install --user == # -echo "Addtiional setup in" ${CI_IMAGE_NAME} +echo "Additional setup in ${CI_IMAGE_NAME}" python3 -m pip install --user tlcpack-sphinx-addon==0.2.1 synr==0.6.0 diff --git a/tests/scripts/task_lint.sh b/tests/scripts/task_lint.sh index 2889c3a94f11..3ebee3219260 100755 --- a/tests/scripts/task_lint.sh +++ b/tests/scripts/task_lint.sh @@ -16,9 +16,7 @@ # specific language governing permissions and limitations # under the License. -set -e -set -u -set -o pipefail +set -euxo pipefail cleanup() { @@ -49,7 +47,7 @@ echo "Linting the Python code..." tests/lint/pylint.sh tests/lint/flake8.sh -echo "Lintinf the JNI code..." +echo "Linting the JNI code..." tests/lint/jnilint.sh echo "Checking C++ documentation..." diff --git a/tests/scripts/task_python_docs.sh b/tests/scripts/task_python_docs.sh index 765c84137730..312dd575c9ea 100755 --- a/tests/scripts/task_python_docs.sh +++ b/tests/scripts/task_python_docs.sh @@ -25,6 +25,8 @@ source tests/scripts/setup-pytest-env.sh # to avoid CI CPU thread throttling. export TVM_BIND_THREADS=0 export OMP_NUM_THREADS=4 +IS_LOCAL=${IS_LOCAL:-0} +PYTHON_DOCS_ONLY=${PYTHON_DOCS_ONLY:-0} cleanup() { @@ -53,15 +55,22 @@ find . -type f -path "*.pyc" | xargs rm -f make cython3 cd docs -PYTHONPATH=`pwd`/../python make html |& tee /tmp/$$.log.txt +PYTHONPATH=`pwd`/../python make html SPHINXOPTS='-j auto' |& tee /tmp/$$.log.txt if grep -E "failed to execute|Segmentation fault" < /tmp/$$.log.txt; then echo "Some of sphinx-gallery item example failed to execute." exit 1 fi cd .. +if [ "$IS_LOCAL" == "1" ] && [ "$PYTHON_DOCS_ONLY" == "1" ]; then + echo "PYTHON_DOCS_ONLY was set, skipping other doc builds" + rm -rf _docs + mv docs/_build/html _docs + exit 0 +fi + # C++ doc -make doc +make cppdoc rm -f docs/doxygen/html/*.map docs/doxygen/html/*.md5 # Java doc @@ -89,10 +98,12 @@ mv jvm/core/target/site/apidocs _docs/reference/api/javadoc # mv rust/target/doc _docs/api/rust mv web/dist/docs _docs/reference/api/typedoc -echo "Start creating the docs tarball.." -# make the tarball -tar -C _docs -czf docs.tgz . -echo "Finish creating the docs tarball" -du -h docs.tgz +if [ "$IS_LOCAL" != "1" ]; then + echo "Start creating the docs tarball.." + # make the tarball + tar -C _docs -czf docs.tgz . + echo "Finish creating the docs tarball" + du -h docs.tgz -echo "Finish everything" + echo "Finish everything" +fi