Use wheels for Dataflow postcommit tests

.github/workflows/build_wheels.yml

@@ -260,6 +260,7 @@ jobs:
       if: ${{matrix.arch == 'aarch64'}}
       name: Set up QEMU
     - name: Install cibuildwheel
+      # note: sync cibuildwheel version with gradle task sdks:python:bdistPy* steps
       run: pip install cibuildwheel==2.9.0
     - name: Build wheel
       working-directory: apache-beam-source

.test-infra/jenkins/job_PostCommit_Python.groovy

@@ -42,6 +42,7 @@ ALL_SUPPORTED_VERSIONS.each { pythonVersion ->
             rootBuildScriptDir(commonJobProperties.checkoutDir)
             tasks(":python${versionSuffix}PostCommit")
             commonJobProperties.setGradleSwitches(delegate)
+            switches("-PuseWheelDistribution")
           }
         }
       }

.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Dataflow_V2.groovy

@@ -39,6 +39,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Py_VR_Dataflow_V2', 'Run Pyt
           tasks(':sdks:python:test-suites:dataflow:validatesRunnerBatchTestsV2')
           tasks(':sdks:python:test-suites:dataflow:validatesRunnerStreamingTestsV2')
           switches('-PuseRunnerV2')
+          switches('-PuseWheelDistribution')
           commonJobProperties.setGradleSwitches(delegate)
         }
       }

.test-infra/jenkins/job_PreCommit_PythonIT.groovy

@@ -22,6 +22,7 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder(
     scope: this,
     nameBase: 'Python_Integration',
     gradleTask: ':pythonPreCommitIT',
+    gradleSwitches: ['-PuseWheelDistribution'],
     timeoutMins: 180,
     triggerPathPatterns: [
       '^model/.*$',

gradle.properties

@@ -38,5 +38,7 @@ javaVersion=1.8
 docker_image_default_repo_root=apache
 docker_image_default_repo_prefix=beam_
 
+# supported flink versions
 flink_versions=1.12,1.13,1.14,1.15,1.16
-
+# supported python versions
+python_versions=3.7,3.8,3.9,3.10,3.11

sdks/python/build.gradle

@@ -39,6 +39,7 @@ build.dependsOn tasks.named("buildPython")
 // Create a Python source distribution tarball.
 def tarball = "apache-beam.tar.gz"
 def sdist = tasks.register("sdist") {
+  description "Create a Python source distribution tarball"
   dependsOn setupVirtualenv
 
   doLast {
@@ -64,6 +65,50 @@ artifacts {
   distTarBall file: file("${buildDir}/${tarball}"), builtBy: sdist
 }
 
+// Create Python wheels for given platform and Python version
+// build identifiers for cibuildwheel
+def platform_identifiers_map = [
+  linux: 'manylinux_*64*', // e.g. manylinux_x86_64, manylinux_aarch64
+  macos:'macosx_*64*',     // e.g. macosx_x86_64, macosx_arm64
+  windows: 'win_*64*',     // e.g. win_amd64, win_arm64
+]
+
+platform_identifiers_map.each { platform, idsuffix ->
+  def archs = 'auto'
+  // note: A fix for arm64 platform in gradle environment. For some reason the
+  // task fails with "Invalid archs option {<Architecture.arm64: 'arm64'>}."
+  // even though os.arch is 'aarch64'
+  // Running cibuildwheel command directly in shell it succeeded, however
+  if (platform == 'linux' && 'aarch64'.equalsIgnoreCase(System.getProperty("os.arch"))) {
+    archs = 'aarch64'
+  }
+  getVersionsAsList('python_versions').each { it ->
+    def pyversion = it.replace('.', '')
+
+    project.tasks.register("bdistPy${pyversion}${platform}") {
+      description "Build a Python wheel distribution for Py${pyversion} ${platform}"
+      dependsOn setupVirtualenv
+      // need sdist task to generate protos
+      dependsOn ':sdks:python:sdist'
+
+      // generated installable Python SDK package
+      doLast {
+        exec {
+          environment CIBW_BUILD: "cp${pyversion}-${idsuffix}"
+          environment CIBW_ENVIRONMENT: "SETUPTOOLS_USE_DISTUTILS=stdlib"
+          environment CIBW_BEFORE_BUILD: "pip install cython numpy && pip install --upgrade setuptools"
+          // note: sync cibuildwheel version with GitHub Action
+          // .github/workflow/build_wheel.yml:build_wheels "Install cibuildwheel" step
+          executable 'sh'
+          args '-c', ". ${envdir}/bin/activate && " +
+              "pip install cibuildwheel==2.9.0 && " +
+              "cibuildwheel --print-build-identifiers --platform ${platform} --archs ${archs} && " +
+              "cibuildwheel --output-dir ${buildDir} --platform ${platform} --archs ${archs}"
+        }
+      }
+    }
+  }
+}
 
 /*************************************************************************************************/
 // Non-testing builds and analysis tasks

sdks/python/test-suites/dataflow/build.gradle

@@ -74,7 +74,7 @@ task validatesRunnerStreamingTestsV2 {
 }
 
 task validatesContainerTests {
-  getVersionsAsList('dataflow_validates_container_tests').each {
+  getVersionsAsList('python_versions').each {
     dependsOn.add(":sdks:python:test-suites:dataflow:py${getVersionSuffix(it)}:validatesContainer")
   }
 }