Un-Revert OSS branch build for Cloud workflow (#11808)

* Revert "Revert "Build OSS branch for deploying to Cloud env (#11474)""

This reverts commit 55e3c1e.

* add action to get dev branch tag to OSS project instead of doing it in cloud

* remove dev branch version action, going to do this in cloud afterall

.github/actions/build-and-push-branch/action.yml

@@ -0,0 +1,63 @@
+name: "Build OSS Branch and Push Minimum Required OSS Images"
+description: "Build jars and docker images tagged for a particular branch. Primarily used for running OSS branch code in Cloud."
+inputs:
+  branch_version_tag:
+    description: 'Used to tag jars and docker images with a branch-specific version (should use the form "dev-<commit_hash>" to pass AirbyteVersion validation)'
+    required: false
+  dockerhub_token:
+    description: "Used to log in to dockerhub for pushing images"
+    required: true
+runs:
+  using: "composite"
+  steps:
+    - name: "Parse Input"
+      id: parse-input
+      shell: bash
+      run: |-
+        # if the *branch_version_tag* input param is not specified, then generate it as 'dev-<commit_hash>`
+        #
+        [[ "${{ inputs.branch_version_tag }}" != '' ]] && echo "::set-output name=branch_version_tag::${{ inputs.branch_version_tag }}" \
+          || { short_hash=$(git rev-parse --short HEAD); echo "::set-output name=branch_version_tag::dev-$short_hash"; }
+
+    - uses: actions/setup-java@v1
+      with:
+        java-version: "17"
+
+    - uses: actions/setup-node@v1
+      with:
+        node-version: "16.13.0"
+
+    - name: Set up CI Gradle Properties
+      run: |
+        mkdir -p ~/.gradle/
+        cat > ~/.gradle/gradle.properties <<EOF
+        org.gradle.jvmargs=-Xmx8g -Xss4m --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
+          --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \
+          --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \
+          --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \
+          --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
+        org.gradle.workers.max=8
+        org.gradle.vfs.watch=false
+        EOF
+      shell: bash
+
+    - name: Build
+      run: VERSION=${{ steps.parse-input.outputs.branch_version_tag }} SUB_BUILD=PLATFORM ./gradlew build --scan
+      shell: bash
+
+    - name: Publish to Maven Local
+      run: VERSION=${{ steps.parse-input.outputs.branch_version_tag }} SUB_BUILD=PLATFORM ./gradlew publishToMavenLocal
+      shell: bash
+
+    - name: Login to Docker (on Master)
+      uses: docker/login-action@v1
+      with:
+        username: airbytebot
+        password: ${{ inputs.dockerhub_token }}
+
+    - name: Push Docker Images
+      run: |
+        GIT_REVISION=$(git rev-parse HEAD)
+        [ [ -z "$GIT_REVISION" ] ] && echo "Couldn't get the git revision..." && exit 1
+        VERSION=${{ steps.parse-input.outputs.branch_version_tag }} GIT_REVISION=$GIT_REVISION docker-compose -f docker-compose-cloud.build.yaml push
+      shell: bash

airbyte-bootloader/Dockerfile

@@ -1,10 +1,14 @@
 ARG JDK_VERSION=17.0.1
 FROM openjdk:${JDK_VERSION}-slim
 
+ARG VERSION=0.35.65-alpha
+
 ENV APPLICATION airbyte-bootloader
+ENV VERSION ${VERSION}
 
 WORKDIR /app
 
-ADD bin/${APPLICATION}-0.35.65-alpha.tar /app
+ADD bin/${APPLICATION}-${VERSION}.tar /app
+
 
-ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-0.35.65-alpha/bin/${APPLICATION}"]
+ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-${VERSION}/bin/${APPLICATION}"]

airbyte-bootloader/build.gradle

@@ -80,7 +80,7 @@ task copyGeneratedTar(type: Copy) {
     into 'build/docker/bin'
 }
 
-Task dockerBuildTask = getDockerBuildTask("bootloader",  "$project.projectDir")
+Task dockerBuildTask = getDockerBuildTask("bootloader", "$project.projectDir", "$rootProject.ext.version", "$rootProject.ext.image_tag")
 dockerBuildTask.dependsOn(copyGeneratedTar)
 assemble.dependsOn(dockerBuildTask)
 

airbyte-cli/build.gradle

@@ -1,3 +1,3 @@
-Task dockerBuildTask = getDockerBuildTask("cli", "$project.projectDir")
+Task dockerBuildTask = getDockerBuildTask("cli", "$project.projectDir", "$rootProject.ext.version", "$rootProject.ext.image_tag")
 dockerBuildTask.dependsOn(copyDocker)
 assemble.dependsOn(dockerBuildTask)

airbyte-commons/src/main/java/io/airbyte/commons/version/AirbyteVersion.java

@@ -12,7 +12,7 @@
  */
 public class AirbyteVersion {
 
-  public static final String DEV_VERSION = "dev";
+  public static final String DEV_VERSION_PREFIX = "dev";
   public static final String AIRBYTE_VERSION_KEY_NAME = "airbyte_version";
 
   private final String version;
@@ -25,7 +25,7 @@ public AirbyteVersion(final String version) {
     this.version = version;
     final String[] parsedVersion = version.replace("\n", "").strip().split("-")[0].split("\\.");
 
-    if (version.equals(DEV_VERSION)) {
+    if (isDev()) {
       this.major = null;
       this.minor = null;
       this.patch = null;
@@ -66,7 +66,7 @@ public String getPatchVersion() {
    * Only the major and minor part of the Version is taken into account.
    */
   public int compatibleVersionCompareTo(final AirbyteVersion another) {
-    if (version.equals(DEV_VERSION) || another.version.equals(DEV_VERSION))
+    if (isDev() || another.isDev())
       return 0;
     final int majorDiff = compareVersion(major, another.major);
     if (majorDiff != 0) {
@@ -100,7 +100,7 @@ public boolean lessThan(final AirbyteVersion other) {
    * Compares two Airbyte Version to check if they are equivalent (including patch version).
    */
   public int patchVersionCompareTo(final AirbyteVersion another) {
-    if (version.equals(DEV_VERSION) || another.version.equals(DEV_VERSION)) {
+    if (isDev() || another.isDev()) {
       return 0;
     }
     final int majorDiff = compareVersion(major, another.major);
@@ -118,7 +118,7 @@ public int patchVersionCompareTo(final AirbyteVersion another) {
    * Compares two Airbyte Version to check if only the patch version was updated.
    */
   public boolean checkOnlyPatchVersionIsUpdatedComparedTo(final AirbyteVersion another) {
-    if (version.equals(DEV_VERSION) || another.version.equals(DEV_VERSION)) {
+    if (isDev() || another.isDev()) {
       return false;
     }
     final int majorDiff = compareVersion(major, another.major);
@@ -133,7 +133,7 @@ public boolean checkOnlyPatchVersionIsUpdatedComparedTo(final AirbyteVersion ano
   }
 
   public boolean isDev() {
-    return version.equals(DEV_VERSION);
+    return version.startsWith(DEV_VERSION_PREFIX);
   }
 
   /**

airbyte-config/init/build.gradle

@@ -19,6 +19,6 @@ task copyScripts(type: Copy) {
     into 'build/docker/bin/scripts'
 }
 
-Task dockerBuildTask = getDockerBuildTask("init",  "$project.projectDir")
+Task dockerBuildTask = getDockerBuildTask("init", "$project.projectDir", "$rootProject.ext.version", "$rootProject.ext.image_tag")
 dockerBuildTask.dependsOn(copyScripts)
 assemble.dependsOn(dockerBuildTask)

airbyte-container-orchestrator/Dockerfile

@@ -25,13 +25,16 @@ RUN curl -fsSLo /usr/share/keyrings/kubernetes-archive-keyring.gpg https://packa
 RUN echo "deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | tee /etc/apt/sources.list.d/kubernetes.list
 RUN apt-get update && apt-get install -y kubectl
 
+ARG VERSION=0.35.65-alpha
+
 ENV APPLICATION airbyte-container-orchestrator
-ENV AIRBYTE_ENTRYPOINT "/app/${APPLICATION}-0.35.65-alpha/bin/${APPLICATION}"
+ENV VERSION=${VERSION}
+ENV AIRBYTE_ENTRYPOINT "/app/${APPLICATION}-${VERSION}/bin/${APPLICATION}"
 
 WORKDIR /app
 
 # Move orchestrator app
-ADD bin/${APPLICATION}-0.35.65-alpha.tar /app
+ADD bin/${APPLICATION}-${VERSION}.tar /app
 
 # wait for upstream dependencies to become available before starting server
-ENTRYPOINT ["/bin/bash", "-c", "/app/${APPLICATION}-0.35.65-alpha/bin/${APPLICATION}"]
+ENTRYPOINT ["/bin/bash", "-c", "/app/${APPLICATION}-${VERSION}/bin/${APPLICATION}"]

airbyte-container-orchestrator/build.gradle

@@ -45,6 +45,6 @@ task copyGeneratedTar(type: Copy) {
     into 'build/docker/bin'
 }
 
-Task dockerBuildTask = getDockerBuildTask("container-orchestrator",  "$project.projectDir")
+Task dockerBuildTask = getDockerBuildTask("container-orchestrator", "$project.projectDir", "$rootProject.ext.version", "$rootProject.ext.image_tag")
 dockerBuildTask.dependsOn(copyGeneratedTar)
 assemble.dependsOn(dockerBuildTask)

airbyte-db/lib/build.gradle

@@ -79,6 +79,6 @@ task copyInitSql(type: Copy) {
     into 'build/docker/bin'
 }
 
-Task dockerBuildTask = getDockerBuildTask("db",  "$project.projectDir")
+Task dockerBuildTask = getDockerBuildTask("db", "$project.projectDir", "$rootProject.ext.version", "$rootProject.ext.image_tag")
 dockerBuildTask.dependsOn(copyInitSql)
 assemble.dependsOn(dockerBuildTask)

airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/IntegrationRunner.java

@@ -297,8 +297,8 @@ private static ITransaction createSentryTransaction(final Class<?> connectorClas
     final String version = parseConnectorVersion(env.getOrDefault("WORKER_CONNECTOR_IMAGE", ""));
     final String airbyteVersion = env.getOrDefault(EnvConfigs.AIRBYTE_VERSION, "");
     final String airbyteRole = env.getOrDefault(EnvConfigs.AIRBYTE_ROLE, "");
-    final boolean isDev = version.equals(AirbyteVersion.DEV_VERSION)
-        || airbyteVersion.equals(AirbyteVersion.DEV_VERSION)
+    final boolean isDev = version.startsWith(AirbyteVersion.DEV_VERSION_PREFIX)
+        || airbyteVersion.startsWith(AirbyteVersion.DEV_VERSION_PREFIX)
         || airbyteRole.equals("airbyter");
     if (isDev) {
       LOGGER.debug("Skip Sentry transaction for dev environment");

airbyte-metrics/reporter/Dockerfile

@@ -1,11 +1,15 @@
 ARG JDK_VERSION=17.0.1
 FROM openjdk:${JDK_VERSION}-slim AS metrics-reporter
 
+ARG VERSION=0.35.65-alpha
+
 ENV APPLICATION airbyte-metrics-reporter
+ENV VERSION ${VERSION}
 
 WORKDIR /app
 
-ADD bin/${APPLICATION}-0.35.65-alpha.tar /app
+ADD bin/${APPLICATION}-${VERSION}.tar /app
+
 
 # wait for upstream dependencies to become available before starting server
-ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-0.35.65-alpha/bin/${APPLICATION}"]
+ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-${VERSION}/bin/${APPLICATION}"]

airbyte-metrics/reporter/build.gradle

@@ -25,6 +25,6 @@ task copyGeneratedTar(type: Copy) {
     into 'build/docker/bin'
 }
 
-Task dockerBuildTask = getDockerBuildTask("metrics-reporter",  "$project.projectDir")
+Task dockerBuildTask = getDockerBuildTask("metrics-reporter", "$project.projectDir", "$rootProject.ext.version", "$rootProject.ext.image_tag")
 dockerBuildTask.dependsOn(copyGeneratedTar)
 assemble.dependsOn(dockerBuildTask)

airbyte-scheduler/app/Dockerfile

@@ -1,11 +1,14 @@
 ARG JDK_VERSION=17.0.1
 FROM openjdk:${JDK_VERSION}-slim AS scheduler
 
+ARG VERSION=0.35.65-alpha
+
 ENV APPLICATION airbyte-scheduler
+ENV VERSION ${VERSION}
 
 WORKDIR /app
 
-ADD bin/${APPLICATION}-0.35.65-alpha.tar /app
+ADD bin/${APPLICATION}-${VERSION}.tar /app
 
 # wait for upstream dependencies to become available before starting server
-ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-0.35.65-alpha/bin/${APPLICATION}"]
+ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-${VERSION}/bin/${APPLICATION}"]

airbyte-scheduler/app/build.gradle

@@ -65,6 +65,6 @@ task copyGeneratedTar(type: Copy) {
     into 'build/docker/bin'
 }
 
-Task dockerBuildTask = getDockerBuildTask("scheduler",  "$project.projectDir")
+Task dockerBuildTask = getDockerBuildTask("scheduler", "$project.projectDir", "$rootProject.ext.version", "$rootProject.ext.image_tag")
 dockerBuildTask.dependsOn(copyGeneratedTar)
 assemble.dependsOn(dockerBuildTask)

airbyte-server/Dockerfile

@@ -3,11 +3,14 @@ FROM openjdk:${JDK_VERSION}-slim AS server
 
 EXPOSE 8000
 
+ARG VERSION=0.35.65-alpha
+
 ENV APPLICATION airbyte-server
+ENV VERSION ${VERSION}
 
 WORKDIR /app
 
-ADD bin/${APPLICATION}-0.35.65-alpha.tar /app
+ADD bin/${APPLICATION}-${VERSION}.tar /app
 
 # wait for upstream dependencies to become available before starting server
-ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-0.35.65-alpha/bin/${APPLICATION}"]
+ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-${VERSION}/bin/${APPLICATION}"]

airbyte-server/build.gradle

@@ -145,7 +145,7 @@ task copyGeneratedTar(type: Copy) {
     into 'build/docker/bin'
 }
 
-Task dockerBuildTask = getDockerBuildTask("server", "$project.projectDir")
+Task dockerBuildTask = getDockerBuildTask("server", "$project.projectDir", "$rootProject.ext.version", "$rootProject.ext.image_tag")
 dockerBuildTask.dependsOn(copyGeneratedTar)
 assemble.dependsOn(dockerBuildTask)
 

airbyte-temporal/build.gradle

@@ -5,6 +5,6 @@ task copyScripts(type: Copy) {
     into 'build/docker/bin/scripts'
 }
 
-Task dockerBuildTask = getDockerBuildTask("temporal", "$project.projectDir")
+Task dockerBuildTask = getDockerBuildTask("temporal", "$project.projectDir", "$rootProject.ext.version", "$rootProject.ext.image_tag")
 dockerBuildTask.dependsOn(copyScripts)
 assemble.dependsOn(dockerBuildTask)

airbyte-webapp/build.gradle

@@ -82,9 +82,9 @@ copyAssets.dependsOn npm_run_build
 assemble.dependsOn copyDocs
 copyDocker.dependsOn(npm_run_build)
 
-Task dockerBuildTask = getDockerBuildTask("webapp",  "$project.projectDir")
+Task dockerBuildTask = getDockerBuildTask("webapp", "$project.projectDir", "$rootProject.ext.version", "$rootProject.ext.image_tag")
 dockerBuildTask.dependsOn(copyBuild)
 dockerBuildTask.dependsOn(copyNginx)
 dockerBuildTask.dependsOn(copyDocs)
 dockerBuildTask.dependsOn(copyAssets)
-assemble.dependsOn(dockerBuildTask)
+assemble.dependsOn(dockerBuildTask)

airbyte-workers/Dockerfile

@@ -25,12 +25,15 @@ RUN curl -fsSLo /usr/share/keyrings/kubernetes-archive-keyring.gpg https://packa
 RUN echo "deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | tee /etc/apt/sources.list.d/kubernetes.list
 RUN apt-get update && apt-get install -y kubectl
 
+ARG VERSION=0.35.65-alpha
+
 ENV APPLICATION airbyte-workers
+ENV VERSION ${VERSION}
 
 WORKDIR /app
 
 # Move worker app
-ADD bin/${APPLICATION}-0.35.65-alpha.tar /app
+ADD bin/${APPLICATION}-${VERSION}.tar /app
 
 # wait for upstream dependencies to become available before starting server
-ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-0.35.65-alpha/bin/${APPLICATION}"]
+ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-${VERSION}/bin/${APPLICATION}"]

airbyte-workers/build.gradle

@@ -76,7 +76,7 @@ task copyGeneratedTar(type: Copy) {
     into 'build/docker/bin'
 }
 
-Task dockerBuildTask = getDockerBuildTask("worker",  "$project.projectDir")
+Task dockerBuildTask = getDockerBuildTask("worker", "$project.projectDir", "$rootProject.ext.version", "$rootProject.ext.image_tag")
 dockerBuildTask.dependsOn(copyGeneratedTar)
 assemble.dependsOn(dockerBuildTask)
 

build.gradle

@@ -30,6 +30,15 @@ if (!env.containsKey('VERSION')) {
     throw new Exception('Version not specified in .env file...')
 }
 
+// `version` is used as the application build version for artifacts like jars
+// `image_tag` is used as the docker tag applied to built images.
+// These values are the same for building an specific Airbyte release or branch via the 'VERSION' environment variable.
+// For local development builds, the 'VERSION' environment variable is unset, and built images are tagged with 'dev'.
+ext {
+    version = System.getenv("VERSION") ?: env.VERSION
+    image_tag = System.getenv("VERSION") ?: 'dev'
+}
+
 def createLicenseWith = { File license, String startComment, String endComment, String lineComment, boolean isPython ->
     /*
     In java, we don't have a second linter/styling tool other than spotless so it doesn't really
@@ -129,9 +138,8 @@ spotless {
 check.dependsOn 'spotlessApply'
 
 @SuppressWarnings('GroovyAssignabilityCheck')
-def Task getDockerBuildTask(String artifactName, String projectDir) {
+def Task getDockerBuildTask(String artifactName, String projectDir, String buildVersion, String buildTag) {
     return task ("buildDockerImage-$artifactName"(type: DockerBuildImage) {
-        def buildTag = System.getenv('VERSION') ?: 'dev'
         def jdkVersion = System.getenv('JDK_VERSION') ?: '17.0.1'
 
         def arch = System.getProperty("os.arch").toLowerCase()
@@ -149,6 +157,7 @@ def Task getDockerBuildTask(String artifactName, String projectDir) {
         buildArgs.put('DOCKER_BUILD_ARCH', buildArch)
         buildArgs.put('ALPINE_IMAGE', alpineImage)
         buildArgs.put('POSTGRES_IMAGE', postgresImage)
+        buildArgs.put('VERSION', buildVersion)
     })
 }
 
@@ -170,7 +179,7 @@ allprojects {
     group = "io.${rootProject.name}${sub.isEmpty() ? '' : ".$sub"}"
     project.archivesBaseName = "${project.group}-${project.name}"
 
-    version = env.VERSION
+    version = rootProject.ext.version
 }
 
 // Java projects common configurations