diff --git a/jenkins.sh b/jenkins.sh
index 6f2758315bf..c4200552a72 100755
--- a/jenkins.sh
+++ b/jenkins.sh
@@ -14,12 +14,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-set -x
+set -xe
 shopt -s globstar
 
+app_version=""
+
 delete_app_version() {
-  yes | gcloud --project="$GOOGLE_PROJECT_ID" \
-    app versions delete "$GOOGLE_VERSION_ID"
+  if [ -n "${app_version}" ] || [ $# -gt 0 ]; then
+    yes | gcloud --project="${GOOGLE_PROJECT_ID}" \
+      app versions delete "${1-${app_version}}"
+  fi
 }
 handle_error() {
   errcode=$? # Remember the error code so we can exit with it after cleanup
@@ -27,7 +31,7 @@ handle_error() {
   # Clean up
   delete_app_version
 
-  exit $errcode
+  exit ${errcode}
 }
 trap handle_error ERR
 
@@ -36,10 +40,18 @@ shellcheck ./**/*.sh
 
 # Find all jenkins.sh's and run them.
 find . -mindepth 2 -maxdepth 5 -name jenkins.sh -type f | while read path; do
+  dir="${path%/jenkins.sh}"
+  app_version="jenkins-${dir//[^a-z]/}"
   (
-  pushd "${path%jenkins.sh}"
-  /bin/bash ./jenkins.sh
+  pushd "${dir}"
+  # Need different app versions because flex can't deploy over an existing
+  # version
+  GOOGLE_VERSION_ID="${app_version}" /bin/bash ./jenkins.sh
+  echo "Return code: $?"
+
+  # Clean up the app version in the background
+  nohup delete_app_version "${app_version}" &
   )
 done
 
-delete_app_version
+wait