Latest/Minimum filter retrieved packages by pyVersion compatibility (#…

…16510)

* filter packages by python_requires release info retrieved by pypi. now we only select package versions that are compatible with current platform

eng/tox/install_depend_packages.py

@@ -84,7 +84,7 @@ def process_requirement(req, dependency_type):
 
     # get available versions on PyPI
     client = PyPIClient()
-    versions = [str(v) for v in client.get_ordered_versions(pkg_name)]
+    versions = [str(v) for v in client.get_ordered_versions(pkg_name, True)]
     logging.info("Versions available on PyPI for %s: %s", pkg_name, versions)
 
     if pkg_name in MINIMUM_VERSION_SUPPORTED_OVERRIDE:

scripts/devops_tasks/common_tasks.py

@@ -43,6 +43,7 @@
     "azure",
     "azure-mgmt",
     "azure-storage",
+    "azure-mgmt-regionmove"
 ]
 MANAGEMENT_PACKAGE_IDENTIFIERS = [
     "mgmt",

tools/azure-sdk-tools/pypi_tools/pypi.py

@@ -1,4 +1,5 @@
 from packaging.version import parse as Version
+import sys
 
 import requests
 
@@ -34,14 +35,35 @@ def project_release(self, package_name, version):
         response.raise_for_status()
         return response.json()
 
-    def get_ordered_versions(self, package_name):
+    def filter_packages_for_compatibility(self, package_name, version_set):
+        # only need the packaging.specifiers import if we're actually executing this filter.
+        from packaging.specifiers import SpecifierSet
+
+        results = []
+
+        for version in version_set:
+            requires_python = self.project_release(package_name, version)["info"]["requires_python"]
+            if requires_python:
+                if Version('.'.join(map(str, sys.version_info[:3]))) in SpecifierSet(requires_python):
+                    results.append(version)
+            else:
+                results.append(version)
+
+        return results
+
+    def get_ordered_versions(self, package_name, filter_by_compatibility = False):
         project = self.project(package_name)
+
         versions = [
             Version(package_version)
             for package_version
             in project["releases"].keys()
         ]
         versions.sort()
+
+        if filter_by_compatibility:
+            return self.filter_packages_for_compatibility(package_name, versions)
+
         return versions
 
     def get_relevant_versions(self, package_name):