Support downloading universal and pure Python wheels (#4981)

* tell the root layout about the desired python_tag

* new major version of downloader that can override --python-tag

also hardcode the version of root layout used by this major version
also some refactoring

* refactoring to make code easier to read

* add friendlier error handling for in-toto verification

* check whether wheel is compatible w/ sys Python

* replace some asserts

* manually downgrade version; will bump later

* update libs

* update to 5.root.json

sha2-256: 36d3bd0863f700e2b9b70a68cd5fc8ef97d101d74b37b4ab395db92e5ce3ce8d

* update requirements.in

* appease the style nanny

* appease black

* update in-toto in ddev

datadog_checks_base/datadog_checks/base/data/agent_requirements.in

@@ -15,7 +15,7 @@ flup==1.0.3.dev-20110405; python_version < '3.0'
 flup-py3==1.0.3; python_version > '3.0'
 gearman==2.0.2; sys_platform != 'win32' and python_version < '3.0'
 httplib2==0.10.3
-in-toto==0.3.0
+in-toto==0.4.1
 ipaddress==1.0.22; python_version < '3.0'
 jaydebeapi==1.1.1
 jpype1==0.7.0
@@ -56,15 +56,15 @@ requests==2.22.0
 requests-kerberos==0.12.0
 requests_ntlm==1.1.0
 scandir==1.8
-securesystemslib[crypto,pynacl]==0.11.3
+securesystemslib[crypto,pynacl]==0.12.2
 selectors34==1.2.0; sys_platform == 'win32' and python_version < '3.4'
 semver==2.9.0
 serpent==1.27; sys_platform == 'win32'
 service_identity[idna]==18.1.0
 simplejson==3.6.5
 six==1.12.0
 supervisor==4.0.1
-tuf==0.11.2.dev3
+tuf==0.12.1
 uptime==3.0.1
 vertica-python==0.9.4
 win-inet-pton==1.1.0; sys_platform == 'win32' and python_version < '3.0'

datadog_checks_dev/setup.py

@@ -70,7 +70,7 @@
             'click',
             'colorama',
             'docker-compose>=1.23.1,<1.24.0',
-            'in-toto==0.3.0',
+            'in-toto>=0.4.1',
             'pip-tools',
             'pylint',
             'Pillow',

datadog_checks_downloader/datadog_checks/downloader/cli.py

@@ -7,13 +7,9 @@
 import argparse
 import re
 
-# 3rd party.
-from tuf.exceptions import UnknownTargetError
-
-# 2nd party.
 # 2nd party.
 from .download import REPOSITORY_URL_PREFIX, TUFDownloader
-from .exceptions import NonCanonicalVersion, NonDatadogPackage, NoSuchDatadogPackageOrVersion
+from .exceptions import NonCanonicalVersion, NonDatadogPackage
 
 # Private module functions.
 
@@ -27,11 +23,6 @@ def __is_canonical(version):
     return re.match(P, version) is not None
 
 
-def __get_wheel_distribution_name(standard_distribution_name):
-    # https://www.python.org/dev/peps/pep-0491/#escaping-and-unicode
-    return re.sub('[^\\w\\d.]+', '_', standard_distribution_name, re.UNICODE)
-
-
 # Public module functions.
 
 
@@ -60,23 +51,11 @@ def download():
 
     if not standard_distribution_name.startswith('datadog-'):
         raise NonDatadogPackage(standard_distribution_name)
-    else:
-        wheel_distribution_name = __get_wheel_distribution_name(standard_distribution_name)
-        tuf_downloader = TUFDownloader(repository_url_prefix=repository_url_prefix, verbose=verbose)
-
-        if not version:
-            version = tuf_downloader.get_latest_version(standard_distribution_name, wheel_distribution_name)
-        else:
-            if not __is_canonical(version):
-                raise NonCanonicalVersion(version)
-
-        target_relpath = 'simple/{}/{}-{}-py2.py3-none-any.whl'.format(
-            standard_distribution_name, wheel_distribution_name, version
-        )
-
-        try:
-            target_abspath = tuf_downloader.download(target_relpath)
-        except UnknownTargetError:
-            raise NoSuchDatadogPackageOrVersion(standard_distribution_name, version)
-
-        print(target_abspath)  # pylint: disable=print-statement
+
+    if version and not __is_canonical(version):
+        raise NonCanonicalVersion(version)
+
+    tuf_downloader = TUFDownloader(repository_url_prefix=repository_url_prefix, verbose=verbose)
+    wheel_relpath = tuf_downloader.get_wheel_relpath(standard_distribution_name, version=version)
+    wheel_abspath = tuf_downloader.download(wheel_relpath)
+    print(wheel_abspath)  # pylint: disable=print-statement