diff --git a/jenkins.sh b/jenkins.sh
index 6f2758315bf..e9f3b09cf20 100755
--- a/jenkins.sh
+++ b/jenkins.sh
@@ -14,12 +14,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-set -x
+set -xe
 shopt -s globstar
 
+app_version=""
+
 delete_app_version() {
-  yes | gcloud --project="$GOOGLE_PROJECT_ID" \
-    app versions delete "$GOOGLE_VERSION_ID"
+  if [ -n "$app_version" ] || [ $# -gt 0 ]; then
+    yes | gcloud --project="$GOOGLE_PROJECT_ID" \
+      app versions delete "${1-$app_version}"
+  fi
 }
 handle_error() {
   errcode=$? # Remember the error code so we can exit with it after cleanup
@@ -36,10 +40,18 @@ shellcheck ./**/*.sh
 
 # Find all jenkins.sh's and run them.
 find . -mindepth 2 -maxdepth 5 -name jenkins.sh -type f | while read path; do
+  dir="${path%/jenkins.sh}"
+  app_version="jenkins-${dir//[^a-z]/}"
   (
-  pushd "${path%jenkins.sh}"
-  /bin/bash ./jenkins.sh
+  pushd "$dir"
+  # Need different app versions because flex can't deploy over an existing
+  # version
+  GOOGLE_VERSION_ID="$app_version" /bin/bash ./jenkins.sh
+  echo "Return code: $?"
+
+  # Clean up the app version in the background
+  nohup delete_app_version "$app_version" &
   )
 done
 
-delete_app_version
+wait