APPLE: Upgrade third party packages for Apple Silicon cross compilation

VERSIONS.md

@@ -20,16 +20,16 @@ Our test machines have the following software versions installed
 | Boost         | 1.70.0               | 1.76.0                       | 1.70.0                         |
 | Intel TBB     | 2019 Update 6        | 2018 Update 1, 2019 Update 6 | 2019 Update 6                  |
 | OpenSubdiv    | 3.4.4                | 3.4.4                        | 3.4.4                          |
-| OpenImageIO   | 2.1.16.0             | 2.1.16.0                     | 2.1.16.0                       |
+| OpenImageIO   | 2.3.15.0             | 2.3.15.0                     | 2.3.15.0                       |
 | OpenColorIO   | 1.1.0                | 1.1.0                        | 1.1.0                          |
 | OSL           | 1.10.9               |                              |                                |
 | Ptex          | 2.3.2                | 2.1.33                       | 2.1.33                         |
 | Qt for Python | PySide2 5.14.1       | PySide6 6.3.1                | PySide2 5.14.1                 |
 | PyOpenGL      | 3.1.5                | 3.1.5                        | 3.1.5                          |
 | Embree        | 3.2.2                | 3.13.3                       | 3.2.2                          |
 | RenderMan     | 24.0                 | 24.0                         | 24.0                           |
-| Alembic       | 1.7.10               | 1.7.10                       | 1.7.10                         |
-| OpenEXR       | 2.4.4                | 2.4.4                        | 2.5.2                          |
+| Alembic       | 1.8.3                | 1.8.3                        | 1.8.3                          |
+| OpenEXR       | 2.5.5                | 2.5.5                        | 2.5.5                          |
 | MaterialX     | 1.38.4               | 1.38.4                       | 1.38.4                         |
 | Jinja2        | 2.0                  |                              |                                |
 | Flex          | 2.5.39               |                              |                                |

build_scripts/apple_utils.py

@@ -0,0 +1,66 @@
+#!/usr/bin/python
+import sys
+import locale
+import os
+import shlex
+import subprocess
+
+def GetLocale():
+    return sys.stdout.encoding or locale.getdefaultlocale()[1] or "UTF-8"
+
+def GetCommandOutput(command):
+    """Executes the specified command and returns output or None."""
+    try:
+        return subprocess.check_output(
+            shlex.split(command), 
+            stderr=subprocess.STDOUT).decode(GetLocale(), 'replace').strip()
+    except subprocess.CalledProcessError:
+        pass
+    return None
+
+def GetMacArmArch():
+    return os.environ.get('MACOS_ARM_ARCHITECTURE') or "arm64"
+
+def GetMacArch():
+    macArch = GetCommandOutput('arch').strip()
+    if macArch == "i386" or macArch == "x86_64":
+        macArch = "x86_64"
+    else:
+        macArch = GetMacArmArch()
+    return macArch
+
+devout = open(os.devnull, 'w')
+
+def ExtractFilesRecursive(path, cond):
+    files = []
+    for r, d, f in os.walk(path):
+        for file in f:
+            if cond(os.path.join(r,file)):
+                files.append(os.path.join(r, file))
+    return files
+
+def CodesignFiles(files):
+    SDKVersion  = subprocess.check_output(['xcodebuild', '-version']).strip()[6:10]
+    codeSignIDs = subprocess.check_output(['security', 'find-identity', '-vp', 'codesigning'])
+
+    codeSignID = "-"
+    if os.environ.get('CODE_SIGN_ID'):
+        codeSignID = os.environ.get('CODE_SIGN_ID')
+    elif float(SDKVersion) >= 11.0 and codeSignIDs.find(b'Apple Development') != -1:
+        codeSignID = "Apple Development"
+    elif codeSignIDs.find(b'Mac Developer') != -1:
+        codeSignID = "Mac Developer"
+
+    for f in files:
+        subprocess.call(['codesign', '-f', '-s', '{codesignid}'
+              .format(codesignid=codeSignID), f],
+              stdout=devout, stderr=devout)
+
+def Codesign(install_path, verbose_output=False):
+    if verbose_output:
+        global devout
+        devout = sys.stdout
+
+    files = ExtractFilesRecursive(install_path, 
+                 (lambda file: '.so' in file or '.dylib' in file))
+    CodesignFiles(files)

build_scripts/build_usd.py

@@ -43,6 +43,7 @@
 import sysconfig
 import tarfile
 import zipfile
+import apple_utils
 
 if sys.version_info.major >= 3:
     from urllib.request import urlopen
@@ -90,8 +91,6 @@ def Linux():
     return platform.system() == "Linux"
 def MacOS():
     return platform.system() == "Darwin"
-def Arm():
-    return platform.processor() == "arm"
 
 def Python3():
     return sys.version_info.major == 3
@@ -888,9 +887,10 @@ def InstallTBB_LinuxOrMacOS(context, force, buildArgs):
         AppendCXX11ABIArg("CXXFLAGS", context, buildArgs)
 
         # Ensure that the tbb build system picks the proper architecture.
-        if MacOS() and Arm():
-            buildArgs.append("arch=arm64")
-
+        if MacOS():
+            if apple_utils.GetMacArch() != "x86_64":
+                buildArgs.append("arch=arm64")
+
         # TBB does not support out-of-source builds in a custom location.
         Run('make -j{procs} {buildArgs}'
             .format(procs=context.numJobs, 
@@ -921,18 +921,26 @@ def InstallTBB_LinuxOrMacOS(context, force, buildArgs):
 
 if Windows():
     JPEG_URL = "https://github.com/libjpeg-turbo/libjpeg-turbo/archive/1.5.1.zip"
+elif MacOS():
+    JPEG_URL = "https://github.com/libjpeg-turbo/libjpeg-turbo/archive/2.0.1.zip"
 else:
     JPEG_URL = "https://www.ijg.org/files/jpegsrc.v9b.tar.gz"
 
 def InstallJPEG(context, force, buildArgs):
-    if Windows():
+    if Windows() or MacOS():
         InstallJPEG_Turbo(context, force, buildArgs)
     else:
         InstallJPEG_Lib(context, force, buildArgs)
 
 def InstallJPEG_Turbo(context, force, buildArgs):
     with CurrentWorkingDirectory(DownloadURL(JPEG_URL, context, force)):
-        RunCMake(context, force, buildArgs)
+        extraJPEGArgs = buildArgs
+        if MacOS():
+            extraJPEGArgs.append("-DWITH_SIMD=FALSE")
+
+        RunCMake(context, force, extraJPEGArgs)
+        return os.getcwd()
+
 
 def InstallJPEG_Lib(context, force, buildArgs):
     with CurrentWorkingDirectory(DownloadURL(JPEG_URL, context, force)):
@@ -981,17 +989,11 @@ def InstallTIFF(context, force, buildArgs):
 ############################################################
 # PNG
 
-PNG_URL = "https://github.com/glennrp/libpng/archive/refs/tags/v1.6.29.tar.gz"
+PNG_URL = "https://github.com/glennrp/libpng/archive/refs/heads/libpng16.zip"
 
 def InstallPNG(context, force, buildArgs):
-    macArgs = []
-    if MacOS() and Arm():
-        # ensure libpng's build doesn't erroneously activate inappropriate
-        # Neon extensions
-        macArgs = ["-DPNG_HARDWARE_OPTIMIZATIONS=OFF", 
-                   "-DPNG_ARM_NEON=off"] # case is significant
     with CurrentWorkingDirectory(DownloadURL(PNG_URL, context, force)):
-        RunCMake(context, force, buildArgs + macArgs)
+        RunCMake(context, force, buildArgs)
 
 PNG = Dependency("PNG", InstallPNG, "include/png.h")
 
@@ -1092,7 +1094,7 @@ def InstallPtex_LinuxOrMacOS(context, force, buildArgs):
 def InstallBLOSC(context, force, buildArgs):
     with CurrentWorkingDirectory(DownloadURL(BLOSC_URL, context, force)):
         macArgs = []
-        if MacOS() and Arm():
+        if MacOS() and apple_utils.GetMacArch() != "x86_64":
             # Need to disable SSE for macOS ARM targets.
             macArgs = ["-DDEACTIVATE_SSE2=ON"]
         RunCMake(context, force, buildArgs + macArgs)
@@ -1135,7 +1137,8 @@ def InstallOpenVDB(context, force, buildArgs):
 ############################################################
 # OpenImageIO
 
-OIIO_URL = "https://github.com/OpenImageIO/oiio/archive/Release-2.1.16.0.zip"
+# OIIO 2.3.15 adds fixes for Apple Silicon cross compilation.
+OIIO_URL = "https://github.com/OpenImageIO/oiio/archive/refs/tags/v2.3.15.0.zip"
 
 def InstallOpenImageIO(context, force, buildArgs):
     with CurrentWorkingDirectory(DownloadURL(OIIO_URL, context, force)):
@@ -1205,8 +1208,9 @@ def InstallOpenColorIO(context, force, buildArgs):
         # When building for Apple Silicon we need to make sure that
         # the correct build architecture is specified for OCIO and
         # and for TinyXML and YAML.
-        if MacOS() and Arm():
-            arch = 'arm64'
+        if MacOS():
+            arch = apple_utils.GetMacArch()
+
             PatchFile("CMakeLists.txt",
                     [('CMAKE_ARGS      ${TINYXML_CMAKE_ARGS}',
                     'CMAKE_ARGS      ${TINYXML_CMAKE_ARGS}\n' +
@@ -1236,6 +1240,16 @@ def InstallOpenColorIO(context, force, buildArgs):
             pass
         else:
             extraArgs.append('-DCMAKE_CXX_FLAGS=-w')
+
+        if MacOS():
+            #if using version 2 of OCIO we patch a different config path as it resides elsewere
+            PatchFile("src/core/Config.cpp",
+                       [("cacheidnocontext_ = cacheidnocontext_;", 
+                         "cacheidnocontext_ = rhs.cacheidnocontext_;")])
+
+            extraArgs.append('-DCMAKE_CXX_FLAGS="-Wno-unused-function -Wno-unused-const-variable -Wno-unused-private-field"')
+            if (apple_utils.GetMacArch() != "x86_64"):
+                extraArgs.append('-DOCIO_USE_SSE=OFF')
 
         if MacOS() and Arm():
             extraArgs.append('-DOCIO_USE_SSE=OFF')
@@ -1356,6 +1370,7 @@ def GetPySideInstructions():
                 'update your PYTHONPATH to indicate where it is '
                 'located.')
 
+
 PYSIDE = PythonDependency("PySide", GetPySideInstructions,
                           moduleNames=["PySide", "PySide2", "PySide6"])
 
@@ -1376,10 +1391,14 @@ def InstallHDF5(context, force, buildArgs):
 ############################################################
 # Alembic
 
-ALEMBIC_URL = "https://github.com/alembic/alembic/archive/1.7.10.zip"
+ALEMBIC_URL = "https://github.com/alembic/alembic/archive/1.8.3.zip"
 
 def InstallAlembic(context, force, buildArgs):
     with CurrentWorkingDirectory(DownloadURL(ALEMBIC_URL, context, force)):
+        if MacOS():
+            PatchFile("CMakeLists.txt", 
+                      [("ADD_DEFINITIONS(-Wall -Werror -Wextra -Wno-unused-parameter)",
+                        "ADD_DEFINITIONS(-Wall -Wextra -Wno-unused-parameter)")])
         cmakeOptions = ['-DUSE_BINARIES=OFF', '-DUSE_TESTS=OFF']
         if context.enableHDF5:
             # HDF5 requires the H5_BUILT_AS_DYNAMIC_LIB macro be defined if
@@ -1743,6 +1762,10 @@ def InstallUSD(context, force, buildArgs):
 group.add_argument("--toolset", type=str,
                    help=("CMake toolset to use when building libraries with "
                          "cmake"))
+if MacOS():
+    codesignDefault = False if apple_utils.GetMacArch() == "x64_64" else True
+    group.add_argument("--codesign", dest="macos_codesign", default=codesignDefault,
+                       help=("Use codesigning for macOS builds (defaults to enabled on Apple Silicon)"))
 
 if Linux():
     group.add_argument("--use-cxx11-abi", type=int, choices=[0, 1],
@@ -1993,6 +2016,8 @@ def __init__(self, args):
         self.useCXX11ABI = \
             (args.use_cxx11_abi if hasattr(args, "use_cxx11_abi") else None)
         self.safetyFirst = args.safety_first
+        self.macOSCodesign = \
+            (args.macos_codesign if hasattr(args, "macos_codesign") else False)
 
         # Dependencies that are forced to be built
         self.forceBuildAll = args.force_all
@@ -2308,6 +2333,7 @@ def FormatBuildArguments(buildArgs):
                   else "Release w/ Debug Info" if context.buildRelWithDebug
                   else ""),
     buildImaging=("On" if context.buildImaging else "Off"),
+    macOSCodesign=("On" if context.macOSCodesign else "Off"),
     enablePtex=("On" if context.enablePtex else "Off"),
     enableOpenVDB=("On" if context.enableOpenVDB else "Off"),
     buildOIIO=("On" if context.buildOIIO else "Off"),
@@ -2387,6 +2413,10 @@ def FormatBuildArguments(buildArgs):
         os.path.join(context.instDir, "lib")
     ])
 
+if MacOS():
+    if context.macOSCodesign:
+        apple_utils.Codesign(context.usdInstDir, verbosity > 1)
+
 Print("""
 Success! To use USD, please ensure that you have:""")