Generate bdist wheels for musllinux_1_1 (#6025)

Z3Prover · May 9, 2022 · 361888f · NikolajBjorner · May 28, 2022 · NikolajBjorner
1 parent dcc01b8
commit 361888f
Show file tree

Hide file tree

Showing 3 changed files with 68 additions and 17 deletions.
diff --git a/scripts/nightly.yaml b/scripts/nightly.yaml
@@ -86,21 +86,31 @@ stages:
         artifactName: 'UbuntuDoc'
         targetPath: $(Build.ArtifactStagingDirectory)
 
-  - job: Manylinux
-    displayName: "Manylinux build"
+  - job: LinuxBuilds
+    strategy:
+      matrix:
+        manyLinux:
+          name: ManyLinux
+          image: "quay.io/pypa/manylinux2010_x86_64:latest"
+          python: "/opt/python/cp37-cp37m/bin/python"
+        muslLinux:
+          name: MuslLinux
+          image: "quay.io/pypa/musllinux_1_1_x86_64:latest"
+          python: "/opt/python/cp310-cp310/bin/python"
+    displayName: "$(name) build"
     pool:
       vmImage: "Ubuntu-18.04"
-    container: "quay.io/pypa/manylinux2010_x86_64:latest"
+    container: $(image)
     variables:
-      python: "/opt/python/cp37-cp37m/bin/python"
+      python: $(python)
     steps:
     - script: $(python) scripts/mk_unix_dist.py --nodotnet --nojava
     - script: git clone https://github.com/z3prover/z3test z3test
     - script: $(python) z3test/scripts/test_benchmarks.py build-dist/z3 z3test/regressions/smt2
     - script: cp dist/*.zip $(Build.ArtifactStagingDirectory)/
     - task: PublishPipelineArtifact@0
       inputs:
-        artifactName: 'Manylinux'
+        artifactName: '$(name)Build'
         targetPath: $(Build.ArtifactStagingDirectory)
 
   - job: Windows32
@@ -395,6 +405,11 @@ stages:
       inputs:
         artifactName: 'Manylinux'
         targetPath: $(Agent.TempDirectory)
+    - task: DownloadPipelineArtifact@2
+      displayName: 'Download MuslLinux Build'
+      inputs:
+        artifact: 'MuslLinuxBuild'
+        path: $(Agent.TempDirectory)
     - task: DownloadPipelineArtifact@2
       inputs:
         artifactName: 'Mac'
@@ -405,13 +420,15 @@ stages:
         targetPath: $(Agent.TempDirectory)
     - script: cd $(Agent.TempDirectory); mkdir osx-x64-bin; cd osx-x64-bin; unzip ../*x64-osx*.zip
     - script: cd $(Agent.TempDirectory); mkdir osx-arm64-bin; cd osx-arm64-bin; unzip ../*arm64-osx*.zip
-    - script: cd $(Agent.TempDirectory); mkdir linux-bin; cd linux-bin; unzip ../*glibc*.zip
+    - script: cd $(Agent.TempDirectory); mkdir libc-bin; cd libc-bin; unzip ../*glibc*.zip
+    - script: cd $(Agent.TempDirectory); mkdir musl-bin; cd musl-bin; unzip ../*-linux.zip
     - script: cd $(Agent.TempDirectory); mkdir win32-bin; cd win32-bin; unzip ../*x86-win*.zip
     - script: cd $(Agent.TempDirectory); mkdir win64-bin; cd win64-bin; unzip ../*x64-win*.zip
     - script: python3 -m pip install --user -U setuptools wheel
     - script: cd src/api/python; python3 setup.py sdist
     # take a look at this PREMIUM HACK I came up with to get around the fact that the azure variable syntax overloads the bash syntax for subshells
-    - script: cd src/api/python; echo $(Agent.TempDirectory)/linux-bin/* | xargs printf 'PACKAGE_FROM_RELEASE=%s\n' | xargs -I '{}' env '{}' python3 setup.py bdist_wheel
+    - script: cd src/api/python; echo $(Agent.TempDirectory)/libc-bin/* | xargs printf 'PACKAGE_FROM_RELEASE=%s\n' | xargs -I '{}' env '{}' python3 setup.py bdist_wheel
+    - script: cd src/api/python; echo $(Agent.TempDirectory)/musl-bin/* | xargs printf 'PACKAGE_FROM_RELEASE=%s\n' | xargs -I '{}' env '{}' python3 setup.py bdist_wheel
     - script: cd src/api/python; echo $(Agent.TempDirectory)/win32-bin/* | xargs printf 'PACKAGE_FROM_RELEASE=%s\n' | xargs -I '{}' env '{}' python3 setup.py bdist_wheel
     - script: cd src/api/python; echo $(Agent.TempDirectory)/win64-bin/* | xargs printf 'PACKAGE_FROM_RELEASE=%s\n' | xargs -I '{}' env '{}' python3 setup.py bdist_wheel
     - script: cd src/api/python; echo $(Agent.TempDirectory)/osx-x64-bin/* | xargs printf 'PACKAGE_FROM_RELEASE=%s\n' | xargs -I '{}' env '{}' python3 setup.py bdist_wheel

diff --git a/scripts/release.yml b/scripts/release.yml
@@ -129,13 +129,23 @@ stages:
         artifactName: 'UbuntuDoc'
         targetPath: $(Build.ArtifactStagingDirectory)
 
-  - job: ManyLinuxBuild
-    displayName: "ManyLinux build"
+  - job: LinuxBuilds
+    strategy:
+      matrix:
+        manyLinux:
+          name: ManyLinux
+          image: "quay.io/pypa/manylinux2010_x86_64:latest"
+          python: "/opt/python/cp37-cp37m/bin/python"
+        muslLinux:
+          name: MuslLinux
+          image: "quay.io/pypa/musllinux_1_1_x86_64:latest"
+          python: "/opt/python/cp310-cp310/bin/python"
+    displayName: "$(name) build"
     pool:
       vmImage: "ubuntu-latest"
-    container: "quay.io/pypa/manylinux2010_x86_64:latest"
+    container: $(image)
     variables:
-      python: "/opt/python/cp37-cp37m/bin/python"
+      python: $(python)
     steps:
     - task: PythonScript@0
       displayName: Build
@@ -160,7 +170,7 @@ stages:
         targetFolder: $(Build.ArtifactStagingDirectory)
     - task: PublishPipelineArtifact@0
       inputs:
-        artifactName: 'ManyLinuxBuild'
+        artifactName: '$(name)Build'
         targetPath: $(Build.ArtifactStagingDirectory)
 
   - template: build-win-signed.yml
@@ -400,6 +410,11 @@ stages:
       inputs:
         artifact: 'ManyLinuxBuild'
         path: $(Agent.TempDirectory)
+    - task: DownloadPipelineArtifact@2
+      displayName: 'Download MuslLinux Build'
+      inputs:
+        artifact: 'MuslLinuxBuild'
+        path: $(Agent.TempDirectory)
     - task: DownloadPipelineArtifact@2
       displayName: 'Download Win32 Build'
       inputs:
@@ -411,14 +426,16 @@ stages:
         artifact: 'WindowsBuild-x64'
         path: $(Agent.TempDirectory)
     - script: cd $(Agent.TempDirectory); mkdir osx-bin; cd osx-bin; unzip ../*osx*.zip
-    - script: cd $(Agent.TempDirectory); mkdir linux-bin; cd linux-bin; unzip ../*glibc*.zip
+    - script: cd $(Agent.TempDirectory); mkdir libc-bin; cd libc-bin; unzip ../*glibc*.zip
+    - script: cd $(Agent.TempDirectory); mkdir musl-bin; cd musl-bin; unzip ../*-linux.zip
     - script: cd $(Agent.TempDirectory); mkdir win32-bin; cd win32-bin; unzip ../*x86-win*.zip
     - script: cd $(Agent.TempDirectory); mkdir win64-bin; cd win64-bin; unzip ../*x64-win*.zip
     - script: python3 -m pip install --user -U setuptools wheel
     - script: cd src/api/python; python3 setup.py sdist
     # take a look at this PREMIUM HACK I came up with to get around the fact that the azure variable syntax overloads the bash syntax for subshells
     - script: cd src/api/python; echo $(Agent.TempDirectory)/osx-bin/* | xargs printf 'PACKAGE_FROM_RELEASE=%s\n' | xargs -I '{}' env '{}' python3 setup.py bdist_wheel
-    - script: cd src/api/python; echo $(Agent.TempDirectory)/linux-bin/* | xargs printf 'PACKAGE_FROM_RELEASE=%s\n' | xargs -I '{}' env '{}' python3 setup.py bdist_wheel
+    - script: cd src/api/python; echo $(Agent.TempDirectory)/libc-bin/* | xargs printf 'PACKAGE_FROM_RELEASE=%s\n' | xargs -I '{}' env '{}' python3 setup.py bdist_wheel
+    - script: cd src/api/python; echo $(Agent.TempDirectory)/musl-bin/* | xargs printf 'PACKAGE_FROM_RELEASE=%s\n' | xargs -I '{}' env '{}' python3 setup.py bdist_wheel
     - script: cd src/api/python; echo $(Agent.TempDirectory)/win32-bin/* | xargs printf 'PACKAGE_FROM_RELEASE=%s\n' | xargs -I '{}' env '{}' python3 setup.py bdist_wheel
     - script: cd src/api/python; echo $(Agent.TempDirectory)/win64-bin/* | xargs printf 'PACKAGE_FROM_RELEASE=%s\n' | xargs -I '{}' env '{}' python3 setup.py bdist_wheel
     - task: PublishPipelineArtifact@0
@@ -534,4 +551,4 @@ stages:
         secureFile: 'pypircs'
     - script: python3 -m pip install --upgrade pip
     - script: python3 -m pip install --user -U setuptools importlib_metadata wheel twine 
-    - script: python3 -m twine upload --config-file $(pypircs.secureFilePath) -r $(pypiReleaseServer) dist/*
+    - script: python3 -m twine upload --config-file $(pypircs.secureFilePath) -r $(pypiReleaseServer) dist/*
diff --git a/src/api/python/setup.py b/src/api/python/setup.py
@@ -250,6 +250,16 @@ def run(self):
 #try: os.makedirs(os.path.join(ROOT_DIR, 'build'))
 #except OSError: pass
 
+# platform.freedesktop_os_release was added in 3.10
+os_id = ''
+if hasattr(platform, 'freedesktop_os_release'):
+    try:
+        osr = platform.freedesktop_os_release()
+        print(osr)
+        os_id = osr['ID']
+    except OSError:
+        pass
+
 if 'bdist_wheel' in sys.argv and '--plat-name' not in sys.argv:
     if RELEASE_DIR is None:
         name = get_platform()
@@ -271,13 +281,20 @@ def run(self):
         # extract the architecture of the release from the directory name
         arch = RELEASE_METADATA[1]
         distos = RELEASE_METADATA[2]
-        if distos in ('debian', 'ubuntu') or 'linux' in distos:
-            raise Exception("Linux binary distributions must be built on centos to conform to PEP 513")
+        if distos in ('debian', 'ubuntu'):
+            raise Exception(
+                "Linux binary distributions must be built on centos to conform to PEP 513 or alpine if targetting musl"
+            )
         elif distos == 'glibc':
             if arch == 'x64':
                 plat_name = 'manylinux1_x86_64'
             else:
                 plat_name = 'manylinux1_i686'
+        elif distos == 'linux' and os_id == 'alpine':
+            if arch == 'x64':
+                plat_name = 'musllinux_1_1_x86_64'
+            else:
+                plat_name = 'musllinux_1_1_i686'
         elif distos == 'win':
             if arch == 'x64':
                 plat_name = 'win_amd64'