Merge pull request #24127: Update staging of Python wheels (#24114)

apache · Nov 11, 2022 · f9a86e5 · f9a86e5
2 parents 9a94381 + 7f24715
commit f9a86e5
Show file tree

Hide file tree

Showing 4 changed files with 20 additions and 10 deletions.
diff --git a/sdks/python/apache_beam/runners/portability/stager.py b/sdks/python/apache_beam/runners/portability/stager.py
@@ -699,6 +699,8 @@ def _get_platform_for_default_sdk_container():
     # Base image
     pip_version = pkg_resources.get_distribution('pip').version
     if parse_version(pip_version) >= parse_version('19.3'):
+      # pip can only recognize manylinux2014_x86_64 wheels
+      # from version 19.3.
       return 'manylinux2014_x86_64'
     else:
       return 'manylinux2010_x86_64'
@@ -830,13 +832,15 @@ def _create_beam_sdk(sdk_remote_location, temp_dir):
       try:
         abi_suffix = 'm' if sys.version_info < (3, 8) else ''
         # Stage binary distribution of the SDK, for now on a best-effort basis.
+        platform_tag = Stager._get_platform_for_default_sdk_container()
         sdk_local_file = Stager._download_pypi_sdk_package(
             temp_dir,
             fetch_binary=True,
             language_version_tag='%d%d' %
             (sys.version_info[0], sys.version_info[1]),
             abi_tag='cp%d%d%s' %
-            (sys.version_info[0], sys.version_info[1], abi_suffix))
+            (sys.version_info[0], sys.version_info[1], abi_suffix),
+            platform_tag=platform_tag)
         sdk_binary_staged_name = Stager.\
             _desired_sdk_filename_in_staging_location(sdk_local_file)
         _LOGGER.info(
@@ -873,10 +877,10 @@ def _create_beam_sdk(sdk_remote_location, temp_dir):
   def _download_pypi_sdk_package(
       temp_dir,
       fetch_binary=False,
-      language_version_tag='27',
+      language_version_tag='39',
       language_implementation_tag='cp',
-      abi_tag='cp27mu',
-      platform_tag='manylinux1_x86_64'):
+      abi_tag='cp39',
+      platform_tag='manylinux2014_x86_64'):
     """Downloads SDK package from PyPI and returns path to local path."""
     package_name = Stager.get_sdk_package_name()
     try:
@@ -911,7 +915,10 @@ def _download_pypi_sdk_package(
           '--platform',
           platform_tag
       ])
-      # Example wheel: apache_beam-2.4.0-cp27-cp27mu-manylinux1_x86_64.whl
+      # Example wheel: with manylinux14 tag.
+      # apache_beam-2.43.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl # pylint: disable=line-too-long
+      if platform_tag == 'manylinux2014_x86_64':
+        platform_tag = 'manylinux_2_17_x86_64.' + platform_tag
       expected_files = [
           os.path.join(
               temp_dir,
@@ -921,8 +928,9 @@ def _download_pypi_sdk_package(
                   language_implementation_tag,
                   language_version_tag,
                   abi_tag,
-                  platform_tag))
+                  platform_tag)),
       ]
+
     else:
       _LOGGER.info('Downloading source distribution of the SDK from PyPi')
       cmd_args.extend(['--no-binary', ':all:'])

diff --git a/sdks/python/apache_beam/runners/portability/stager_test.py b/sdks/python/apache_beam/runners/portability/stager_test.py
@@ -137,6 +137,8 @@ def pip_fake(args):
             # Per PEP-0427 in wheel filenames non-alphanumeric characters
             # in distribution name are replaced with underscore.
             distribution_name = distribution_name.replace('-', '_')
+            if args[17] == 'manylinux2014_x86_64':
+              args[17] = 'manylinux_2_17_x86_64.' + args[17]
             package_file = '%s-%s-%s%s-%s-%s.whl' % (
                 distribution_name,
                 distribution_version,

diff --git a/sdks/python/container/boot.go b/sdks/python/container/boot.go
@@ -224,7 +224,7 @@ func setupAcceptableWheelSpecs() error {
 	if err != nil {
 		return err
 	}
-	re := regexp.MustCompile(`Python (\d)\.(\d).*`)
+	re := regexp.MustCompile(`Python (\d)\.(\d+).*`)
 	pyVersions := re.FindStringSubmatch(string(stdoutStderr[:]))
 	if len(pyVersions) != 3 {
 		return fmt.Errorf("cannot get parse Python version from %s", stdoutStderr)
@@ -233,9 +233,9 @@ func setupAcceptableWheelSpecs() error {
 	var wheelName string
 	switch pyVersion {
 	case "36", "37":
-		wheelName = fmt.Sprintf("cp%s-cp%sm-manylinux1_x86_64.whl", pyVersion, pyVersion)
+		wheelName = fmt.Sprintf("cp%s-cp%sm-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", pyVersion, pyVersion)
 	default:
-		wheelName = fmt.Sprintf("cp%s-cp%s-manylinux1_x86_64.whl", pyVersion, pyVersion)
+		wheelName = fmt.Sprintf("cp%s-cp%s-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", pyVersion, pyVersion)
 	}
 	acceptableWhlSpecs = append(acceptableWhlSpecs, wheelName)
 	return nil

diff --git a/sdks/python/setup.py b/sdks/python/setup.py
@@ -95,7 +95,7 @@ def get_version():
 different technologies and user communities.
 '''
 
-RECOMMENDED_MIN_PIP_VERSION = '7.0.0'
+RECOMMENDED_MIN_PIP_VERSION = '19.3.0'
 try:
   _PIP_VERSION = get_distribution('pip').version
   if parse_version(_PIP_VERSION) < parse_version(RECOMMENDED_MIN_PIP_VERSION):