Set up tox environments for testing, linting, and building docs (#1667

)

* Set up `tox` environments for testing, linting, and building docs

* Move `pybind11` to the dev requirements for the `tox` linting env

* Move `pybind11` to the cpu-ci action

* Add `sklearn` to test dependencies

* Update the CPU CI action to use `tox` to build docs

* Update docs scheduled rebuild CI action to build with `tox`

* Add `clean` to docs Makefile

.github/workflows/cpu-ci.yml

@@ -29,39 +29,17 @@ jobs:
       run: |
         sudo apt-get update -y
         sudo apt-get install -y protobuf-compiler pandoc
-    - name: Install dependencies
+    - name: Install and upgrade python packages
       run: |
-        python -m pip install --upgrade pip setuptools==59.4.0 wheel 
-        python -m pip install nvidia-pyindex
-        python -m pip install -r requirements.txt pybind11
-        python -m pip install -r requirements-dev.txt
+        python -m pip install --upgrade pip setuptools==59.4.0 wheel tox pybind11
         python -m pip uninstall protobuf -y
         python -m pip install --no-binary=protobuf protobuf
-        python -m pip install tritonclient[all]
-        python -m pip install torch tensorflow-cpu
-        python -m pip install numpy==1.20.3
-        python -m pip install Markdown==3.3.7
-    - name: Lint with flake8
+    - name: Lint with flake8, black, isort, interrogate, codespell
       run: |
-        flake8 .
-    - name: Lint with black
+        tox -e lint
+    - name: Run tests
       run: |
-        black --check .
-    - name: Lint with isort
-      run: |
-        isort -c .
-    - name: Lint with interrogate
-      run: |
-        interrogate --config=pyproject.toml
-    - name: Lint with codespell
-      run: |
-        codespell
-    - name: Build
-      run: |
-        python setup.py develop --user
-    - name: Run unittests
-      run: |
-        python -m pytest -rxs tests/unit/
+        tox -e test-cpu
     - name: Generate package for pypi
       run: |
         python setup.py sdist
@@ -89,7 +67,7 @@ jobs:
     # Build docs, treat warnings as errors
     - name: Building docs
       run: |
-        make -C docs html SPHINXOPTS="-W -q"
+        tox -e docs
     - name: Upload HTML
       uses: actions/upload-artifact@v2
       with:

.github/workflows/docs-sched-rebuild.yaml

@@ -25,27 +25,13 @@ jobs:
       - name: Install Ubuntu packages
         run: |
           sudo apt-get update -y
-          sudo apt-get install -y protobuf-compiler pandoc
+          sudo apt-get install -y protobuf-compiler
       - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip setuptools==59.4.0 wheel
-          python -m pip install nvidia-pyindex
-          python -m pip install -r requirements.txt pybind11
-          python -m pip install -r requirements-dev.txt
-          python -m pip uninstall protobuf -y
-          python -m pip install --no-binary=protobuf protobuf
-          python -m pip install tritonclient[all]
-          python -m pip install torch tensorflow-cpu
-          python -m pip install numpy==1.20.3
-      - name: Build
-        run: |
-          python setup.py develop --user
-      - name: Report the versions to build
-        run: |
-          sphinx-multiversion --dump-metadata docs/source docs/build/html | jq "keys"
+          python -m pip install --upgrade pip setuptools==59.4.0 wheel tox
       - name: Building docs (multiversion)
         run: |
-          sphinx-multiversion docs/source docs/build/html
+          tox -e docs-multi
       - name: Upload HTML
         uses: actions/upload-artifact@v2
         with:

MANIFEST.in

@@ -1,4 +1,4 @@
 include versioneer.py
 recursive-include nvtabular *.py *.proto
 recursive-include cpp *.cc *.h *.cu *.cuh 
-include requirements*.txt
+include requirements/*.txt

docs/Makefile

@@ -12,8 +12,13 @@ BUILDDIR      = build
 help:
 	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
 
+clean:
+	rm -rf build source/generated source/README.md source/examples
+
 .PHONY: help Makefile
 
+
+
 # Catch-all target: route all unknown targets to Sphinx using the new
 # "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
 %: Makefile

requirements/dev.txt

@@ -0,0 +1,13 @@
+# packages necessary to run tests and push PRs
+# assumes requirements for nvtabular logic are already installed
+
+# linting
+cpplint>=1.5
+codespell
+interrogate==1.5.0
+black==22.6.0
+flake8==3.9.2
+isort==5.10.1
+pylint==2.7.4
+bandit==1.7.4
+flake8-nb==0.5.2

requirements/docs.txt

@@ -0,0 +1,10 @@
+Sphinx<3.6
+jinja2<3.1
+markupsafe==2.0.1
+sphinx-multiversion@git+https://github.com/mikemckiernan/sphinx-multiversion.git
+sphinxcontrib-copydirs@git+https://github.com/mikemckiernan/sphinxcontrib-copydirs.git
+sphinx-external-toc<0.4
+sphinx_rtd_theme
+natsort
+myst-nb
+linkify-it-py

requirements-gpu.txt → requirements/gpu.txt

@@ -1,4 +1,4 @@
--r requirements.txt
+-r base.txt
 cudf>=21.12
 dask-cudf>=21.12
 dask-cuda>=21.12

requirements/test.txt

@@ -0,0 +1,32 @@
+nvidia-pyindex
+pybind11
+
+-r base.txt
+-r dev.txt
+
+tritonclient[all]
+torch
+tensorflow-cpu
+numpy==1.20.3
+Markdown==3.3.7
+
+pytest>=5
+pytest-cov>=2
+asvdb@git+https://github.com/rapidsai/asvdb.git
+
+# needed to make test_s3 work
+pytest-xdist
+s3fs>=2021.4
+aiobotocore>=1.3.3
+flask
+flask-cors
+moto>=2
+boto3==1.17
+
+
+# Assume these are required for notebook tests, but want to doublecheck that
+scikit-learn>=0.20
+# graphviz>=0.16
+# ipython_genutils
+
+

setup.py

@@ -61,27 +61,29 @@ def run(self):
 cmdclass["develop"] = develop
 
 
-def read_requirements(filename):
+def read_requirements(req_path, filename):
     base = os.path.abspath(os.path.dirname(__file__))
-    with codecs.open(os.path.join(base, filename), "rb", "utf-8") as f:
+    with codecs.open(os.path.join(base, req_path, filename), "rb", "utf-8") as f:
         lineiter = (line.strip() for line in f)
         packages = []
         for line in lineiter:
             if line:
                 if line.startswith("-r"):
                     filename = line.replace("-r", "").strip()
-                    packages.extend(read_requirements(filename))
+                    packages.extend(read_requirements(req_path, filename))
                 elif not line.startswith("#"):
                     packages.append(line)
         return packages
 
 
 requirements = {
-    "cpu": read_requirements("requirements.txt"),
-    "gpu": read_requirements("requirements-gpu.txt"),
+    "cpu": read_requirements("requirements", "base.txt"),
+    "gpu": read_requirements("requirements", "gpu.txt"),
 }
 dev_requirements = {
-    "dev": read_requirements("requirements-dev.txt"),
+    "dev": read_requirements("requirements", "dev.txt"),
+    "test": read_requirements("requirements", "test.txt"),
+    "docs": read_requirements("requirements", "docs.txt"),
 }
 
 with open("README.md", encoding="utf8") as readme_file:

tox.ini

@@ -0,0 +1,84 @@
+; For all tests that run in Github Actions, see
+; .github/workflows/cpu-ci.yml for the workflow definition.
+
+[tox]
+envlist = python3.8,test-gpu,test-cpu
+
+[testenv]
+commands =
+    pip install --upgrade pip
+    pip install .
+
+[testenv:test-cpu]
+; Runs in: Github Actions
+; Runs all CPU-based tests. NOTE: if you are using an M1 mac, this will fail. You need to
+; change the tensorflow dependency to `tensorflow-macos` in requirements/test-cpu.txt.
+deps = -rrequirements/test.txt
+commands = python -m pytest --cov-report term --cov=merlin -rxs tests/unit
+
+[testenv:test-gpu]
+sitepackages=true
+; Runs in: Internal Jenkins
+; Runs GPU-based tests.
+; The jenkins jobs run on an image based on merlin-hugectr. This will include all cudf configuration
+; and other gpu-specific libraries that we can enxpect will always exist. Thus, we don't need
+; to install requirements.txt yet. As we get better at python environment isolation, we will
+; need to add some back.
+deps =
+    pytest
+    pytest-cov
+commands =
+    python -m pytest --cov-report term --cov merlin -rxs tests/unit
+
+[testenv:test-merlin]
+; Runs in: Internal Jenkins
+; This runs the end-to-end tests from the NVIDIA-Merlin/Merlin repo on the jenkins machine.
+; We will check out `Merlin` from github and execute the notebooks using the current PR of systems.
+passenv=GIT_COMMIT
+sitepackages=true
+allowlist_externals = git
+deps =
+    -rrequirements/gpu.txt
+commands =
+    ; the GIT_COMMIT env is the current commit of the systems repo
+    ; NOTE!!!! We must clean this up in the jenkins configuration with `rm -rf "Merlin-$GIT_COMMIT"`
+    python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/core.git
+    python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/models.git
+    python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/systems.git
+    python -m pip install .
+
+    git clone https://github.com/NVIDIA-Merlin/Merlin.git Merlin-{env:GIT_COMMIT}
+
+    ; this runs the tests then removes the Merlin repo directory whether the tests work or fail
+    python -m pytest --cov merlin --cov-report term Merlin-{env:GIT_COMMIT}/tests/unit
+
+[testenv:lint]
+; Runs in: Github Actions
+; Runs all lint/code checks and fails the PR if there are errors.
+; Install pre-commit-hooks to run these tests during development.
+deps = -rrequirements/dev.txt
+commands =
+    flake8 setup.py merlin/ tests/
+    black --check --diff merlin tests
+    pylint merlin
+    isort -c . --skip .tox
+    interrogate --config=pyproject.toml
+    codespell --skip .tox
+
+[testenv:docs]
+; Runs in: Github Actions
+; Generates documentation with sphinx. There are other steps in the Github Actions workflow
+; to publish the documentation on release.
+changedir = {toxinidir}
+deps = -rrequirements/docs.txt
+commands =
+    python -m sphinx.cmd.build -P -b html docs/source docs/build/html
+
+[testenv:docs-multi]
+; Run the multi-version build that is shown on GitHub Pages.
+changedir = {toxinidir}
+deps = -rrequirements/docs.txt
+commands =
+    sphinx-multiversion --dump-metadata docs/source docs/build/html | jq "keys"
+    sphinx-multiversion docs/source docs/build/html
+