Add optional requirement specifiers for GPU and dev requirements (#1664)

* Add optional requirement specifiers for GPU and dev requirements

* Give `asvdb` requirement a proper package name (in addition to URL)

* Include all requirements files in the `MANIFEST.in` file

MANIFEST.in

@@ -1,4 +1,4 @@
 include versioneer.py
 recursive-include nvtabular *.py *.proto
 recursive-include cpp *.cc *.h *.cu *.cuh 
-include requirements.txt
+include requirements*.txt

requirements-dev.txt

@@ -13,7 +13,7 @@ pytest-cov>=2
 scikit-learn>=0.20
 moto>=2
 boto3==1.17
-git+https://github.com/rapidsai/asvdb.git
+asvdb@git+https://github.com/rapidsai/asvdb.git
 graphviz>=0.16
 cpplint>=1.5
 codespell

setup.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+import codecs
 import os
 import sys
 
@@ -60,13 +61,31 @@ def run(self):
 cmdclass["develop"] = develop
 
 
-def parse_requirements(filename):
-    """load requirements from a pip requirements file"""
-    lineiter = (line.strip() for line in open(filename))
-    return [line for line in lineiter if line and not line.startswith("#")]
+def read_requirements(filename):
+    base = os.path.abspath(os.path.dirname(__file__))
+    with codecs.open(os.path.join(base, filename), "rb", "utf-8") as f:
+        lineiter = (line.strip() for line in f)
+        packages = []
+        for line in lineiter:
+            if line:
+                if line.startswith("-r"):
+                    filename = line.replace("-r", "").strip()
+                    packages.extend(read_requirements(filename))
+                elif not line.startswith("#"):
+                    packages.append(line)
+        return packages
 
 
-install_reqs = parse_requirements("./requirements.txt")
+requirements = {
+    "cpu": read_requirements("requirements.txt"),
+    "gpu": read_requirements("requirements-gpu.txt"),
+}
+dev_requirements = {
+    "dev": read_requirements("requirements-dev.txt"),
+}
+
+with open("README.md", encoding="utf8") as readme_file:
+    long_description = readme_file.read()
 
 setup(
     name="nvtabular",
@@ -75,7 +94,7 @@ def parse_requirements(filename):
     url="https://github.com/NVIDIA-Merlin/NVTabular",
     author="NVIDIA Corporation",
     license="Apache 2.0",
-    long_description=open("README.md", encoding="utf8").read(),
+    long_description=long_description,
     long_description_content_type="text/markdown",
     classifiers=[
         "Development Status :: 4 - Beta",
@@ -88,5 +107,9 @@ def parse_requirements(filename):
     cmdclass=cmdclass,
     ext_modules=ext_modules,
     zip_safe=False,
-    install_requires=install_reqs,
+    install_requires=requirements["cpu"],
+    extras_require={
+        **requirements,
+        **dev_requirements,
+    },
 )