Merged in SIM-2981 (pull request apache#3)

SIM-2981: Fix Versioning For Python Packages

Approved-by: Jeffrey Uong

Jenkinsfile

@@ -3,12 +3,14 @@ library('sima-jenkins-lib')
 
 def main() {
   def job_name = env.JOB_NAME.split('/')[1]
-  def currentBranchName = env.CHANGE_ID ? env.CHANGE_BRANCH : env.BRANCH_NAME
 
   properties([
       parameters([
-          string(name: "COPY_MLA_BRANCH_PKG", defaultValue: currentBranchName, description: 'Copy specified mla pkg'),
-          string(name: "COPY_N2A_COMPILER_BRANCH_PKG", defaultValue: currentBranchName, description: 'Copy specified n2a_compiler pkg')
+          booleanParam(
+              name: 'SKIP_N2A_COMPILER_BUILD',
+              description: 'Skips building n2a_compiler',
+              defaultValue: false
+          )
       ]),
   ])
 
@@ -20,9 +22,6 @@ def main() {
     def image
     stage("DockerBuild") {
       image = utils.dockerBuild("docker/Dockerfile", 'simaai/' + job_name, "docker_creds", "docker_build.log", { ->
-        utils.getPackage('sima-ai','mla', params.COPY_MLA_BRANCH_PKG, '*.deb')
-        utils.getPackage('sima-ai','n2a_compiler', params.COPY_N2A_COMPILER_BRANCH_PKG, '*.whl')
-        sh "ls -alh"
       })
     }
 
@@ -34,12 +33,14 @@ def main() {
       image["image"].inside("-m 32g -c 8") {
         utils.cmakeBuild("build", "-DCMAKE_CXX_COMPILER_LAUNCHER=ccache", {}, { src_dir ->
           stage("Python Bindings") {
-            sh """#!/bin/bash -ex
-cd ..
-make cython
-cd python
+            dir("../python") {
+              utils.setPythonBuildEnv([]) {
+                sh """#!/bin/bash -ex
+rm -rf dist build
 python3 setup.py bdist_wheel
 """
+              }
+            }
           }
         }, "../sima-regres.cmake", "clean all")
         stage("Package") {
@@ -48,9 +49,22 @@ python3 setup.py bdist_wheel
         }
       }
     }
+
+    stage("Promotion") {
+      if (env.BRANCH_NAME=="sima") {
+        utils.docker_promote(image['image'], 'docker_creds', '')
+      }
+    }
+
+  }
+
+  stage("Upstream") {
+    utils.buildUpstream("n2a_compiler", params.SKIP_N2A_COMPILER_BUILD, [])
   }
 }
 
 utils.job_wrapper( {
   main()
 })
+
+return this

docker/Dockerfile

@@ -3,7 +3,7 @@ FROM simaai/ubuntu:latest
 ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get update && apt-get -y install \
     build-essential ccache \
-    wget curl awscli cmake unzip && \
+    wget curl awscli cmake unzip git && \
     rm -rf /var/lib/apt/lists/*
 RUN \
   sed -i 's/# \(.*multiverse$\)/\1/g' /etc/apt/sources.list && \
@@ -21,16 +21,6 @@ RUN \
     -o /usr/local/share/ca-certificates/letsencryptx3.crt && \
   update-ca-certificates
 
-ARG MLA_BRANCH="master"
-ARG MLA_VERSION="0.0.1"
-ADD mla-${MLA_VERSION}-Linux.deb /tmp
-RUN apt-get update && apt-get install -y /tmp/mla-${MLA_VERSION}-Linux.deb && rm -rf /var/lib/apt/lists/*
-
-ARG N2A_COMPILER_BRANCH="master"
-ARG N2A_COMPILER_VERSION="0.0.1"
-ADD sima_mlc-${N2A_COMPILER_VERSION}_${N2A_COMPILER_BRANCH}-py3-none-any.whl /tmp
-RUN pip3 install /tmp/sima_mlc-${N2A_COMPILER_VERSION}_${N2A_COMPILER_BRANCH}-py3-none-any.whl
-
 RUN apt-get update && \
   apt-get install -y \
     libopenblas-dev \

python/MANIFEST.in

@@ -0,0 +1 @@
+include VERSION.in

python/VERSION.in

@@ -0,0 +1,3 @@
+major: 0
+minor: 1
+patch: 0

python/setup.py

@@ -149,9 +149,43 @@ def get_package_data_files():
     # Relay standard libraries
     return ['relay/std/prelude.rly', 'relay/std/core.rly']
 
+import yaml
+import subprocess
+
+def get_version(pkg_dir, main_branch = "master"):
+  with open("VERSION.in") as fh:
+    vinfo = yaml.load(fh.read(), Loader=yaml.BaseLoader)
+    version = '.'.join([vinfo["major"], vinfo["minor"], vinfo["patch"]])
+
+  if 'GIT_HASH' in os.environ:
+    # In tox the .git directory is not available so do this instead
+    vinfo['git_hash'] = os.environ['GIT_HASH']
+  else:
+    proc = subprocess.Popen(['git','log','-1','--format=%h'], stdout=subprocess.PIPE)
+    proc.wait()
+    if proc.returncode != 0:
+      raise RuntimeError("ERROR: unable to execute git log")
+    vinfo['git_hash'] = proc.stdout.read().decode('utf-8').rstrip()
+
+  with open(pkg_dir + "/VERSION","w") as fh:
+    fh.write(yaml.dump(vinfo))
+
+  # This comes from Jenkins for upstream/downstream builds
+  if 'DEV_VERSION' in os.environ:
+    version = version + "+" + os.environ['DEV_VERSION']
+
+  return version
+
+
+def get_package(env, pkg_name, pkg_version):
+  if env in os.environ and os.environ[env] != "latest":
+    return "%s==%s+%s" % (pkg_name, pkg_version, os.environ[env])
+  else:
+    return "%s==%s" % (pkg_name, pkg_version)
+
 
 setup(name='sima-tvm',
-      version=__version__,
+      version=get_version('tvm', 'sima'),
       description="TVM: An End to End Tensor IR/DSL Stack for Deep Learning Systems",
       zip_safe=False,
       install_requires=[
@@ -173,7 +207,7 @@ def get_package_data_files():
 
       packages=find_packages(),
       package_dir={'tvm': 'tvm'},
-      package_data={'tvm': get_package_data_files()},
+      package_data={'tvm': get_package_data_files() + ["VERSION"]},
       distclass=BinaryDistribution,
       url='https://github.com/apache/incubator-tvm',
       ext_modules=config_cython(),

python/tvm/.gitignore

@@ -0,0 +1 @@
+VERSION