diff --git a/sdks/python/apache_beam/runners/portability/stager.py b/sdks/python/apache_beam/runners/portability/stager.py index e06c71c917d2..fbcf4d5a2e59 100644 --- a/sdks/python/apache_beam/runners/portability/stager.py +++ b/sdks/python/apache_beam/runners/portability/stager.py @@ -699,6 +699,8 @@ def _get_platform_for_default_sdk_container(): # Base image pip_version = pkg_resources.get_distribution('pip').version if parse_version(pip_version) >= parse_version('19.3'): + # pip can only recognize manylinux2014_x86_64 wheels + # from version 19.3. return 'manylinux2014_x86_64' else: return 'manylinux2010_x86_64' @@ -830,13 +832,15 @@ def _create_beam_sdk(sdk_remote_location, temp_dir): try: abi_suffix = 'm' if sys.version_info < (3, 8) else '' # Stage binary distribution of the SDK, for now on a best-effort basis. + platform_tag = Stager._get_platform_for_default_sdk_container() sdk_local_file = Stager._download_pypi_sdk_package( temp_dir, fetch_binary=True, language_version_tag='%d%d' % (sys.version_info[0], sys.version_info[1]), abi_tag='cp%d%d%s' % - (sys.version_info[0], sys.version_info[1], abi_suffix)) + (sys.version_info[0], sys.version_info[1], abi_suffix), + platform_tag=platform_tag) sdk_binary_staged_name = Stager.\ _desired_sdk_filename_in_staging_location(sdk_local_file) _LOGGER.info( @@ -873,10 +877,10 @@ def _create_beam_sdk(sdk_remote_location, temp_dir): def _download_pypi_sdk_package( temp_dir, fetch_binary=False, - language_version_tag='27', + language_version_tag='39', language_implementation_tag='cp', - abi_tag='cp27mu', - platform_tag='manylinux1_x86_64'): + abi_tag='cp39', + platform_tag='manylinux2014_x86_64'): """Downloads SDK package from PyPI and returns path to local path.""" package_name = Stager.get_sdk_package_name() try: @@ -911,7 +915,10 @@ def _download_pypi_sdk_package( '--platform', platform_tag ]) - # Example wheel: apache_beam-2.4.0-cp27-cp27mu-manylinux1_x86_64.whl + # Example wheel: with manylinux14 tag. + # apache_beam-2.43.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl # pylint: disable=line-too-long + if platform_tag == 'manylinux2014_x86_64': + platform_tag = 'manylinux_2_17_x86_64.' + platform_tag expected_files = [ os.path.join( temp_dir, @@ -921,8 +928,9 @@ def _download_pypi_sdk_package( language_implementation_tag, language_version_tag, abi_tag, - platform_tag)) + platform_tag)), ] + else: _LOGGER.info('Downloading source distribution of the SDK from PyPi') cmd_args.extend(['--no-binary', ':all:']) diff --git a/sdks/python/apache_beam/runners/portability/stager_test.py b/sdks/python/apache_beam/runners/portability/stager_test.py index b221bb1ec6f6..84b6c5ddbb6f 100644 --- a/sdks/python/apache_beam/runners/portability/stager_test.py +++ b/sdks/python/apache_beam/runners/portability/stager_test.py @@ -137,6 +137,8 @@ def pip_fake(args): # Per PEP-0427 in wheel filenames non-alphanumeric characters # in distribution name are replaced with underscore. distribution_name = distribution_name.replace('-', '_') + if args[17] == 'manylinux2014_x86_64': + args[17] = 'manylinux_2_17_x86_64.' + args[17] package_file = '%s-%s-%s%s-%s-%s.whl' % ( distribution_name, distribution_version, diff --git a/sdks/python/container/boot.go b/sdks/python/container/boot.go index 8e4cf772c0e0..47b9ea186c93 100644 --- a/sdks/python/container/boot.go +++ b/sdks/python/container/boot.go @@ -224,7 +224,7 @@ func setupAcceptableWheelSpecs() error { if err != nil { return err } - re := regexp.MustCompile(`Python (\d)\.(\d).*`) + re := regexp.MustCompile(`Python (\d)\.(\d+).*`) pyVersions := re.FindStringSubmatch(string(stdoutStderr[:])) if len(pyVersions) != 3 { return fmt.Errorf("cannot get parse Python version from %s", stdoutStderr) @@ -233,9 +233,9 @@ func setupAcceptableWheelSpecs() error { var wheelName string switch pyVersion { case "36", "37": - wheelName = fmt.Sprintf("cp%s-cp%sm-manylinux1_x86_64.whl", pyVersion, pyVersion) + wheelName = fmt.Sprintf("cp%s-cp%sm-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", pyVersion, pyVersion) default: - wheelName = fmt.Sprintf("cp%s-cp%s-manylinux1_x86_64.whl", pyVersion, pyVersion) + wheelName = fmt.Sprintf("cp%s-cp%s-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", pyVersion, pyVersion) } acceptableWhlSpecs = append(acceptableWhlSpecs, wheelName) return nil diff --git a/sdks/python/setup.py b/sdks/python/setup.py index c91fb2e71a85..c4c7f106c39f 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -95,7 +95,7 @@ def get_version(): different technologies and user communities. ''' -RECOMMENDED_MIN_PIP_VERSION = '7.0.0' +RECOMMENDED_MIN_PIP_VERSION = '19.3.0' try: _PIP_VERSION = get_distribution('pip').version if parse_version(_PIP_VERSION) < parse_version(RECOMMENDED_MIN_PIP_VERSION):