Share nasm path across jobs, add node-specific env for python and mor…

…e comments on deploy.py.

Jenkinsfile

@@ -63,7 +63,9 @@ if (BUILD_LINUX) {
 def windowsNode(platform) {
   node('OGS_Win_001') {
     try {
-      withEnv(["WORKSPACE_ON_NODE=${env.WORKSPACE}"]) {
+      withEnv(["WORKSPACE_ON_NODE=${env.WORKSPACE}",
+               "PATH+PY=C:\\Program Files\\Python37;C:\\Program Files\\Python37\\Scripts",
+               "PYTHONBIN=python"]) {
         checkoutStage('windows')
 
         def buildInfo = getBuildInfo('windows', platform, 'RelWithDebInfo')
@@ -88,7 +90,9 @@ def windowsNode(platform) {
 def macNode(platform) {
   node('OGS_Mac') {
     try {
-      withEnv(["WORKSPACE_ON_NODE=${env.WORKSPACE}"]) {
+      withEnv(["WORKSPACE_ON_NODE=${env.WORKSPACE}",
+               "PATH+PY=/Library/Frameworks/Python.framework/Versions/3.9/bin",
+               "PYTHONBIN=python3"]) {
         checkoutStage('mac')
 
         def buildInfo = getBuildInfo('mac', platform, 'RelWithDebInfo')
@@ -114,7 +118,8 @@ def macNode(platform) {
 def linuxNode(platform) {
   node('aws-centos-gpu') {
     try {
-      withEnv(["WORKSPACE_ON_NODE=${env.WORKSPACE}"]) {
+      withEnv(["WORKSPACE_ON_NODE=${env.WORKSPACE}",
+               "PYTHONBIN=python3"]) {
         // Always clean up the workspace of the instance,
         // in case the same instance is allocated after previous run is aborted
         dir('.') { deleteDir() }
@@ -139,6 +144,7 @@ def linuxNode(platform) {
       }
     }
     finally {
+      // Place any mandatory cleanup here if has.
     }
   }
 }
@@ -187,14 +193,6 @@ def checkoutGit() {
 def getBuildInfo(os, platform, config) {
   String sep = isUnix() ? '/' : '\\\\'
 
-  // Static agents uses the common workspace/USDarts for USD artifacts,
-  // and dynamic agents just use allocated workspace
-  String USDarts = WORKSPACE_ON_NODE
-  def static_agents = ['OGS_Win_001', 'OGS_Mac_001', 'OGS_Mac_002']
-  if (static_agents.contains(env.NODE_NAME)) {
-    USDarts = WORKSPACE_ON_NODE + sep + '..' + sep + 'USDarts'
-  }
-
   if (config == 'Debug') {
     buildVariant = 'debug'
   }
@@ -205,8 +203,19 @@ def getBuildInfo(os, platform, config) {
     buildVariant = 'release'
   }
 
+  // Linux docker image uses allocated workspace for USD artifacts,
+  // and Windows and Mac agents use the common workspace/USDarts
+  String USDarts = ''
+  if ((env.NODE_NAME).contains('aws-centos')) {
+    USDarts                  = WORKSPACE_ON_NODE
+  }
+  else {
+    USDarts                  = WORKSPACE_ON_NODE + sep + '..' + sep + 'USDarts'
+  }
+
   // Store for future references
   def buildInfo = [:]
+  buildInfo.Python           = PYTHONBIN
   buildInfo.Platform         = "${os}_${platform}_${config}"
   buildInfo.BuildVariant     = buildVariant
   buildInfo.CMakeConfig      = config
@@ -235,9 +244,9 @@ for folder in [r'${buildInfo.USDinst}', r'${buildInfo.USDgen_build}']:
         print("removed " + folder)"""
   writeFile file: "delete_arts.py", text: delete_arts
   if (isUnix()) {
-    sh 'python3 delete_arts.py'
+    sh """${buildInfo.Python} delete_arts.py"""
   } else {
-    bat 'python delete_arts.py'
+    bat """${buildInfo.Python} delete_arts.py"""
   }
 }
 
@@ -286,6 +295,7 @@ def checkoutStage(os) {
       throw err
     }
     finally {
+      // Place any mandatory cleanup here if has.
     }
   }
 }
@@ -308,36 +318,33 @@ def buildStage(buildInfo, runBuild) {
       throw err
     }
     finally {
+      // Place any mandatory cleanup here if has.
     }
   }
 }
 
 def windowsBuild(buildInfo) {
   bat """
     :: Set up nasm for OIIO plugin
-    call curl -o nasm-2.15.05-win64.zip https://www.nasm.us/pub/nasm/releasebuilds/2.15.05/win64/nasm-2.15.05-win64.zip
-    call 7z x -y -aoa nasm-2.15.05-win64.zip
-    call set PATH=${WORKSPACE}\\nasm-2.15.05;%PATH%
+    if not exist ${buildInfo.USDgen_src}\\nasm-2.15.05-win64.zip call curl -o ${buildInfo.USDgen_src}\\nasm-2.15.05-win64.zip https://www.nasm.us/pub/nasm/releasebuilds/2.15.05/win64/nasm-2.15.05-win64.zip
+    if not exist ${buildInfo.USDgen_src}\\nasm-2.15.05 call 7z x -y -aoa ${buildInfo.USDgen_src}\\nasm-2.15.05-win64.zip -o${buildInfo.USDgen_src}
+    call set PATH=${buildInfo.USDgen_src}\\nasm-2.15.05;%PATH%
     :: Build USD with tests
     call C:\\"Program Files (x86)"\\"Microsoft Visual Studio"\\2019\\Enterprise\\VC\\Auxiliary\\Build\\vcvars64.bat
-    call set PYTHONLOCATION=C:\\"Program Files"\\Python37
-    call set PYTHONBIN=%PYTHONLOCATION%\\python.exe
-    call set PATH=%PYTHONLOCATION%;%PYTHONLOCATION%\\Scripts;%PATH%
     call set BOOST_ROOT=
-    call %PYTHONBIN% --version
-    call %PYTHONBIN% -m pip install --upgrade pip
-    call %PYTHONBIN% -m pip install PySide2
-    call %PYTHONBIN% -m pip install PyOpenGL
-    call %PYTHONBIN% build_scripts/build_usd.py --build ${buildInfo.USDgen_build} --src ${buildInfo.USDgen_src} ${buildInfo.USDinst} --materialx --tests --build-variant ${buildInfo.BuildVariant} --build-args USD,"-DPXR_ENABLE_PRECOMPILED_HEADERS=OFF -DPXR_USE_PYTHON_3=ON" -v
+    call ${buildInfo.Python} --version
+    call ${buildInfo.Python} -m pip install --upgrade pip
+    call ${buildInfo.Python} -m pip install PySide2
+    call ${buildInfo.Python} -m pip install PyOpenGL
+    call ${buildInfo.Python} build_scripts/build_usd.py --build ${buildInfo.USDgen_build} --src ${buildInfo.USDgen_src} ${buildInfo.USDinst} --materialx --tests --build-variant ${buildInfo.BuildVariant} --build-args USD,"-DPXR_ENABLE_PRECOMPILED_HEADERS=OFF -DPXR_USE_PYTHON_3=ON" -v
     """
 }
 
 def macBuild(buildInfo) {
   sh """
-    export PATH=/Library/Frameworks/Python.framework/Versions/3.9/bin:$PATH
-    python3 -m pip install pyside2 ;
-    python3 -m pip install PyOpenGL ;
-    python3 build_scripts/build_usd.py --generator Xcode --toolset "buildsystem=1" --build ${buildInfo.USDgen_build} --src ${buildInfo.USDgen_src} ${buildInfo.USDinst} --materialx --tests --build-variant ${buildInfo.BuildVariant} --build-args USD,"-DPXR_USE_PYTHON_3=ON" -v
+    ${buildInfo.Python} -m pip install pyside2 ;
+    ${buildInfo.Python} -m pip install PyOpenGL ;
+    ${buildInfo.Python} build_scripts/build_usd.py --generator Xcode --toolset "buildsystem=1" --build ${buildInfo.USDgen_build} --src ${buildInfo.USDgen_src} ${buildInfo.USDinst} --materialx --tests --build-variant ${buildInfo.BuildVariant} --build-args USD,"-DPXR_USE_PYTHON_3=ON" -v
     """
 }
 
@@ -349,10 +356,10 @@ def linuxBuild(buildInfo) {
     apt-get install -y python-setuptools;
     apt-get install -y libglew-dev libxrandr-dev libxcursor-dev libxinerama-dev libxi-dev;
     apt-get install -y zlib1g-dev
-    python3 -m pip install --upgrade pip
-    python3 -m pip install PySide2 ;
-    python3 -m pip install PyOpenGL ;
-    python3 build_scripts/build_usd.py --build ${buildInfo.USDgen_build} --src ${buildInfo.USDgen_src} ${buildInfo.USDinst} --materialx --tests --build-variant ${buildInfo.BuildVariant} --build-args USD,"-DPXR_USE_PYTHON_3=ON" -v
+    ${buildInfo.Python} -m pip install --upgrade pip
+    ${buildInfo.Python} -m pip install PySide2 ;
+    ${buildInfo.Python} -m pip install PyOpenGL ;
+    ${buildInfo.Python} build_scripts/build_usd.py --build ${buildInfo.USDgen_build} --src ${buildInfo.USDgen_src} ${buildInfo.USDinst} --materialx --tests --build-variant ${buildInfo.BuildVariant} --build-args USD,"-DPXR_USE_PYTHON_3=ON" -v
     """
 }
 
@@ -378,14 +385,13 @@ def testStage(buildInfo, runTest) {
       throw err
     }
     finally {
+      // Place any mandatory cleanup here if has.
     }
   }
 }
 
 def windowsTest(buildInfo) {
   bat """
-    call set PYTHONLOCATION=C:\\"Program Files"\\Python37
-    call set PATH=%PYTHONLOCATION%;%PYTHONLOCATION%\\Scripts;%PATH%
     call set PATH=%PATH%;${buildInfo.USDinst}\\lib;${buildInfo.USDinst}\\bin;${buildInfo.USDinst}\\share\\usd\\examples\\plugin;
     call set PYTHONPATH=${buildInfo.USDinst}\\lib\\python
     call cmake --build ${buildInfo.USDgen_build_usd} --config ${buildInfo.CMakeConfig} --target RUN_TESTS
@@ -395,7 +401,6 @@ def windowsTest(buildInfo) {
 def macTest(buildInfo) {
   sh """
     export PATH=$PATH:${buildInfo.USDinst}/lib:
-    export PATH=/Library/Frameworks/Python.framework/Versions/3.9/bin:$PATH
     PYTHONPATH=${buildInfo.USDinst}/lib/python:
     LD_LIBRARY_PATH=/Library/Frameworks/Python.framework/Versions/3.9/lib:
     cmake --build ${buildInfo.USDgen_build_usd} --config ${buildInfo.CMakeConfig} --target RUN_TESTS
@@ -405,7 +410,7 @@ def macTest(buildInfo) {
 def linuxTest(buildInfo) {
   sh """
     apt-get install -y '^libxcb.*-dev' libx11-xcb-dev libxrender-dev libxkbcommon-dev libxkbcommon-x11-dev;
-    python3 -m pip install numpy
+    ${buildInfo.Python} -m pip install numpy
     export PATH=$PATH:${buildInfo.USDinst}/lib:
     export PYTHONPATH=${buildInfo.USDinst}/lib/python:
     export QT_QPA_PLATFORM_PLUGIN_PATH=/usr/local/lib/python3.6/dist-packages/PySide2/Qt/plugins:
@@ -428,7 +433,7 @@ def deployStage(buildInfo, runDeploy) {
     stage("Versioning ${buildInfo.Platform}") {
       try {
         COMMONSHELL.shell """\
-                          python adsk-build-scripts/deploy.py --add_version --source ${buildInfo.USDinst}
+                          ${buildInfo.Python} adsk-build-scripts/deploy.py --add_version --source ${buildInfo.USDinst}
                           """
         postStatus('SUCCESS', "${buildInfo.Platform}_versioning")
       }
@@ -438,13 +443,14 @@ def deployStage(buildInfo, runDeploy) {
         throw err
       }
       finally {
+        // Place any mandatory cleanup here if has.
       }
     }
 
     stage("Garasign ${buildInfo.Platform}") {
       try {
         COMMONSHELL.shell """\
-                          python adsk-build-scripts/deploy.py --add_sign --source ${buildInfo.USDinst}
+                          ${buildInfo.Python} adsk-build-scripts/deploy.py --add_sign --source ${buildInfo.USDinst}
                           """
         sign_args = [
           'include_list': "toSign.txt",
@@ -467,6 +473,7 @@ def deployStage(buildInfo, runDeploy) {
         throw err
       }
       finally {
+        // Place any mandatory cleanup here if has.
       }
     }
   }
@@ -482,12 +489,12 @@ def deployStage(buildInfo, runDeploy) {
       throw err
     }
     finally {
+      // Place any mandatory cleanup here if has.
     }
   }
 }
 
 def genericDeploy(buildInfo) {
-  def pyExe = isUnix() ? "python3" : 'python'
   def artifactory_env_path = env.BRANCH_NAME
   def remotePath = "${ARTIFACTORY_FOLDER}/${artifactory_env_path}"
   def targetDir = 'archive'
@@ -496,7 +503,7 @@ def genericDeploy(buildInfo) {
 
   withCredentials([DEPLOY_CREDENTIALS]) {
     COMMONSHELL.shell """\
-                      ${pyExe} adsk-build-scripts/deploy.py --deploy --packagename ${PROJECT_NAME} \
+                      ${buildInfo.Python} adsk-build-scripts/deploy.py --deploy --packagename ${PROJECT_NAME} \
                         --branch ${env.BRANCH_NAME} --git_url ${git_url} --git_commit ${commit} \
                         --platform ${buildInfo.Platform} --source ${buildInfo.USDinst} --path ${targetDir} \
                         --server ${ARTIFACTORY_URL} --remotepath ${remotePath} \
@@ -527,8 +534,7 @@ String readCommandOutput(command) {
 }
 
 def postStatus(result, stageName, additionalHTML = '') {
-  // Use the Jenkins stash for inter-node communication. This is probably not what the gods
-  // intended, but it's what I've got.
+  // Use the Jenkins stash for inter-node communication.
 
   // Accessible magenta/green alternative to the traditional red/green combination
   // http://www.visualisingdata.com/2015/11/colour-swatch-alternatives-to-green-and-red/ [November 2016]

adsk-build-scripts/deploy.py

@@ -1,5 +1,41 @@
 #!/usr/bin/env python
 
+
+# Utilities for managing deployment tasks, including key modules of:
+#
+# read_package_configs (internal): get configuration details of a certain package
+#     1) Read configs from packageConfig.json
+#     2) Read a certain file inside of USD installation folder for version and macro information
+#     3) Generate a generic resource string for the package
+#     4) Generate the list of dlls targeted versioning for the package
+#     5) Generate the list of folders targeted signing for the package
+#
+# add_version_to_file (internal): Add version information to a dll
+#     1) Create dll specific rc file from a generic resource string
+#     2) Call ResourceHacker to compile a res file from the rc file
+#     3) Call ResourceHacker to add resource to the dll
+#
+# deploy_to_artifactory (internal): Deploy archive to Artifactory
+#     1) Calculate a hash value of the archive
+#     2) Check file existence with the given hash value in Artifactory
+#     3) Deploy no-existed file to Artifactory (with properties if provided)
+#
+# add_version: Add version information to a certain package
+#     1) Call read_package_configs to get config details
+#     2) Multi-thread versioning task for all the targeted files
+#     3) Clean up temp rc and res files
+#
+# add_sign: Add signature and timestamp information to a certain package
+#     1) Call read_package_configs to get config details
+#     2) Generate the list of signing files of the package
+#     3) Write all the full path of files to toSign.txt
+#
+# deploy: Zip package and deploy to Artifactory
+#     1) Call read_package_configs to get config details - mostly for version information
+#     2) Zip source directory into an archive
+#     3) Call deploy_to_artifactory to deploy the archive
+
+
 from __future__ import print_function
 
 import argparse
@@ -397,7 +433,7 @@ def deploy_to_artifactory(file_path, target_url,
     }
 
     # Deploy by checksum runs in two phases: first we query with the checksum *WITHOUT* data.
-    # If there is no existing file with the given SHA1 it returns status code 404.
+    # If there is no existing file with the given SHA256 it returns status code 404.
     # If we try to do everything at once (send the put request with the checksums AND the data)
     # when the file does not exist, Artifactory will reset the connection.
     # See this bash/curl example from JFrog:
@@ -472,25 +508,27 @@ def main():
         help='API key for artifactory (preferred method)')
     cargs = parser.parse_args()
 
-    # Source install path is required for versioning and deployment
+    # Source install path is required for all the tasks
     source_dir = os.path.abspath(cargs.source)
     if not os.path.exists(source_dir):
         error_msg('Local install path ' + source_dir + ' does not exist.')
         sys.exit(2)
 
-    # Handle special step first
+    # Handle standalone steps and skip rest of deployment steps after that
     if cargs.add_version:
         add_version(source_dir, cargs.packagename)
     elif cargs.add_sign:
         add_sign(source_dir, cargs.packagename)
 
+    # Check local path to store the archive
     if not os.path.exists(cargs.path):
         os.mkdir(cargs.path)
     if not os.path.isdir(cargs.path):
         error_msg('Local deploy path ' + cargs.path + ' is not a directory.')
         sys.exit(2)
     target_dir = os.path.abspath(cargs.path)
 
+    # Validate parameters for remote deployment
     has_credentials = cargs.credentials or cargs.apikey
     create_remote_archive = cargs.server is not None and cargs.remotepath is not None
     if create_remote_archive and not has_credentials:
@@ -499,36 +537,38 @@ def main():
 
     configs = read_package_configs(source_dir, cargs.packagename)
     for config in configs:
-        name = "{0}{1}{2}{3}".format(cargs.packagename,
+        # Naming the zip with all the information provided, e.g. usd_windows_x64_RelWithDebInfo_0.22.8.0_9089e8b.zip
+        # TODO: Have a more proper naming conversion, such as incrementing the last BUILD digit for USD package,
+        # which also requires additional changes on the above versioning step
+        zip_name = "{0}{1}{2}{3}.zip".format(cargs.packagename,
                                      '_' + cargs.platform if cargs.platform else '',
                                      '_' + '.'.join(config[1]),
                                      '_' + cargs.git_commit[:7] if cargs.git_commit else '')
 
         # Zip source directory into an archive
-        archive = os.path.join(target_dir, name + '.zip')
+        archive = os.path.join(target_dir, zip_name)
         if not os.path.exists(archive):
             zip_directory(source_dir, archive)
             info_msg('Deployed local archive: ' + archive)
 
         # Upload resulting archive to artifactory if requested
         if create_remote_archive:
-            remote_file_path = "{0}/{1}".format(cargs.remotepath, name + '.zip')
+            remote_file_path = "{0}/{1}".format(cargs.remotepath, zip_name)
             target_url = cargs.server + '/' + remote_file_path
             kauth = {"apikey": cargs.apikey}
             if cargs.apikey is None:
                 credentials = cargs.credentials.split(':')
                 kauth["username"] = credentials[0]
                 kauth["password"] = credentials[1]
 
+            # https://www.jfrog.com/confluence/display/JFROG/Using+Properties+in+Deployment+and+Resolution
             properties = dict()
             if cargs.branch:
                 properties['git.branch'] = cargs.branch
             if cargs.git_commit:
                 properties['git.commit'] = cargs.git_commit
             if cargs.git_url:
                 properties['git.url'] = cargs.git_url
-
-            # https://www.jfrog.com/confluence/display/JFROG/Using+Properties+in+Deployment+and+Resolution
             if properties:
                 target_url = '{0};{1}'.format(target_url, ";".join(['{}={}'.format(k,v) for k,v in properties.items()]))