SPYT-746 Separating release build and uploading artifacts to public r…

…epositories * Changelog entry Type: feature Component: spyt Separating release build and uploading artifacts to public repositories commit_hash:f182c7871fb2237df7475782ba6e3614e9324853
ytsaurus · Nov 11, 2024 · 7369a59 · 7369a59
1 parent cd8cc03
commit 7369a59
Show file tree

Hide file tree

Showing 8 changed files with 102 additions and 108 deletions.
diff --git a/.github/workflows/spyt_release.yaml b/.github/workflows/spyt_release.yaml
@@ -60,8 +60,8 @@ jobs:
           echo "RELEASE_VERSION=$RELEASE_VERSION" >> $GITHUB_OUTPUT
           echo "RELEASE_VERSION=$RELEASE_VERSION"
 
-  spyt_release:
-    name: Run SPYT release
+  build_spyt:
+    name: Build SPYT artifacts
     needs:
       - start-vm
       - prepare_env
@@ -77,45 +77,64 @@ jobs:
           server-password: OSSRH_PASSWORD
           gpg-private-key: ${{ secrets.GPG_PRIVATE_KEY }}
           gpg-passphrase: GPG_PASSWORD
+        env:
+          OSSRH_USERNAME: ${{ secrets.OSSRH_USERNAME }}
+          OSSRH_PASSWORD: ${{ secrets.OSSRH_PASSWORD }}
+          GPG_PASSWORD: ${{ secrets.GPG_PASSWORD }}
+
+      - name: Run sbt build
+        run: |
+          sbt -DconfigGeneration=False \
+            -DcustomSpytVersion=${{ needs.prepare_env.outputs.release_version }} \
+            spytBuildRelease
 
+  publish_to_pypi:
+    name: Publish SPYT artifacts to PyPi
+    needs:
+      - start-vm
+      - prepare_env
+      - build_spyt
+    if: ${{ always() && (needs.build_spyt.result == 'success') }}
+    runs-on: ${{ needs.start-vm.outputs.label }}
+    steps:
       - name: Set up python
         run: |
           ln -sf /usr/bin/python3.11 /usr/bin/python3
           python3 -m pip install twine
-          python3 -m pip install ytsaurus-client==0.13.5
+          python3 -m pip install ytsaurus-client==0.13.20
+      
+      - name: Publish to PyPi
+        run: |
+          sbt -Dpypi.password=${{ secrets.PYPI_API_TOKEN }} publishToPypi
 
+  publish_to_maven_central:
+    name: Publish SPYT artifacts to Maven Central
+    needs:
+      - start-vm
+      - prepare_env
+      - build_spyt
+    if: ${{ always() && (needs.build_spyt.result == 'success') }}
+    runs-on: ${{ needs.start-vm.outputs.label }}
+    steps:
       - name: Write credentials
         run: |
           mkdir -p ~/.sbt
           echo "realm=Sonatype Nexus Repository Manager" > ~/.sbt/.ossrh_credentials
           echo "host=s01.oss.sonatype.org" >> ~/.sbt/.ossrh_credentials
           echo "user=${{ secrets.OSSRH_USERNAME }}" >> ~/.sbt/.ossrh_credentials
           echo "password=${{ secrets.OSSRH_PASSWORD }}" >> ~/.sbt/.ossrh_credentials
-
-      - name: Run sbt build
+      
+      - name: Publish to Maven Central
         run: |
-          sbt \
-            -DpublishRepo=True \
-            -DpublishMavenCentral=True \
-            -DinnerSidecarConfig=False \
-            -DconfigGeneration=False \
-            -DpublishYt=False \
-            -DcustomSpytVersion=${{ needs.prepare_env.outputs.release_version }} \
-            -Dgpg.passphrase=${{ secrets.GPG_PASSWORD }} \
-            -Dpypi.password=${{ secrets.PYPI_API_TOKEN }} \
-            spytPublishRelease
-        env:
-          OSSRH_USERNAME: ${{ secrets.OSSRH_USERNAME }}
-          OSSRH_PASSWORD: ${{ secrets.OSSRH_PASSWORD }}
-          GPG_PASSWORD: ${{ secrets.GPG_PASSWORD }}
-
-  build_spyt_image:
-    name: Build SPYT docker image
+          sbt -Dgpg.passphrase=${{ secrets.GPG_PASSWORD }} publishToMavenCentral
+  
+  build_and_publish_docker_image:
+    name: Build and publish SPYT docker image
     needs:
       - start-vm
       - prepare_env
-      - spyt_release
-    if: ${{ always() && (needs.spyt_release.result == 'success') }}
+      - build_spyt
+    if: ${{ always() && (needs.build_spyt.result == 'success') }}
     runs-on: ${{ needs.start-vm.outputs.label }}
     steps:
 
@@ -140,8 +159,10 @@ jobs:
     name: Stop VM
     needs:
       - start-vm
-      - build_spyt_image
-    if: ${{ always() && needs.build_spyt_image.result == 'success' }}
+      - publish_to_pypi
+      - publish_to_maven_central
+      - build_and_publish_docker_image
+    if: ${{ always() && needs.publish_to_pypi.result == 'success' && needs.publish_to_maven_central.result == 'success' && needs.build_and_publish_docker_image.result == 'success' }}
     runs-on: trampoline
     steps:
       - name: Stop YC runner

diff --git a/build.sbt b/build.sbt
@@ -199,8 +199,10 @@ lazy val `spyt-package` = (project in file("spyt-package"))
       val clusterConfigArtifacts = spyt.ClusterConfig.artifacts(streams.value.log, versionValue,
         (Compile / resourceDirectory).value)
 
+      val spytPackageZip = getBuildDirectory(baseDirectory.value.getParentFile) / s"${(Universal / packageName).value}.zip"
+
       Seq(
-        YtPublishFile((Universal / packageBin).value, basePath, None, isTtlLimited = isTtlLimited),
+        YtPublishFile(spytPackageZip, basePath, None, isTtlLimited = isTtlLimited),
         YtPublishFile(setupSpytEnvScript.value, basePath, None, isTtlLimited = isTtlLimited, isExecutable = true)
       ) ++ clusterConfigArtifacts
     }
@@ -228,41 +230,41 @@ lazy val root = (project in file("."))
       streams.value.log.info(s"Preparing build directory in ${baseDirectory.value}")
       deleteBuildDirectory(baseDirectory.value)
     },
-    spytPublish := Def.taskDyn {
-      val task1 = if (publishYtEnabled) {
-        `spyt-package` / publishYt
-      } else {
-        streams.value.log.info("Publishing SPYT files to YT is skipped because of disabled publishYt")
-        `spyt-package` / spytArtifacts
-      }
-      val task2 = Def.task {
-        if (publishRepoEnabled) {
-          (`spyt-package` / pythonBuildAndUpload).value
-        } else {
-          streams.value.log.info("Publishing spyt client to pypi is skipped because of disabled publishRepo")
+
+    spytDistributive := Def.taskDyn {
+      val spytPackageArchives = `spyt-package` / spytArtifacts
+      val pythonWhl = Def.task {
           val pythonDist = (`spyt-package` / pythonBuild).value
           makeLinkToBuildDirectory(pythonDist, baseDirectory.value, "ytsaurus-spyt")
-        }
       }
-      Def.sequential(task1, task2)
+      Def.sequential(spytPackageArchives, pythonWhl)
     }.value,
-    spytPublishLibraries := {
-      if (publishMavenCentralEnabled) {
-        Def.sequential(
-          `spark-adapter-api` / publishSigned,
-          `spark-adapter-impl-322` / publishSigned,
-          `spark-adapter-impl-330` / publishSigned,
-          `yt-wrapper` / publishSigned,
-          `file-system` / publishSigned,
-          `spark-patch` / publishSigned,
-          `data-source-base` / publishSigned,
-          `data-source-extended` / publishSigned,
-          `resource-manager` / publishSigned,
-          `cluster` / publishSigned,
-          `spark-submit` / publishSigned
-        ).value
+
+    publishToYt := {
+      if (publishYtEnabled) {
+        (`spyt-package` / publishYt).value
       } else {
-        streams.value.log.info("Publishing spyt libraries to maven is skipped because of disabled publishRepo")
+        streams.value.log.info("Publishing SPYT files to YT is skipped because no cluster was specified")
       }
+    },
+
+    publishToPypi := {
+      (`spyt-package` / pythonUpload).value
+    },
+
+    publishToMavenCentral := {
+      Def.sequential(
+        `spark-adapter-api` / publishSigned,
+        `spark-adapter-impl-322` / publishSigned,
+        `spark-adapter-impl-330` / publishSigned,
+        `yt-wrapper` / publishSigned,
+        `file-system` / publishSigned,
+        `spark-patch` / publishSigned,
+        `data-source-base` / publishSigned,
+        `data-source-extended` / publishSigned,
+        `resource-manager` / publishSigned,
+        `cluster` / publishSigned,
+        `spark-submit` / publishSigned
+      ).value
     }
   )
diff --git a/e2e-test/run-tests.sh b/e2e-test/run-tests.sh
@@ -65,8 +65,7 @@ done
 
 if [ "$rebuild" = "true" ]; then
     # Build SPYT artifacts
-    sbt -DpublishRepo=False -DpublishMavenCentral=False -DpublishYt=False \
-        -DcustomSpytVersion=$spyt_version spytPublishRelease
+    sbt -DcustomSpytVersion=$spyt_version spytBuildRelease
 fi;
 
 if [ "$start_yt_local" = "true" ]; then

diff --git a/project/ReleaseUtils.scala b/project/ReleaseUtils.scala
@@ -12,23 +12,15 @@ import java.io.File
 
 object ReleaseUtils {
 
-  lazy val logSpytVersion: ReleaseStep = { st: State =>
-    st.log.info(s"SPYT scala version: ${st.extract.get(spytVersion)}")
-    st.log.info(s"SPYT python version: ${st.extract.get(spytPythonVersion)}")
-    st
-  }
-
   lazy val dumpVersions: ReleaseStep = { st: State =>
     val scalaVersion = st.extract.get(spytVersion)
     val pythonVersion = st.extract.get(spytPythonVersion)
     st.log.info(s"SPYT scala version dump: $scalaVersion")
     st.log.info(s"SPYT python version dump: $pythonVersion")
-    if (!publishYtEnabled) {
-      dumpVersionsToBuildDirectory(
-        Map("scala" -> scalaVersion, "python" -> pythonVersion),
-        st.extract.get(baseDirectory), "version.json"
-      )
-    }
+    dumpVersionsToBuildDirectory(
+      Map("scala" -> scalaVersion, "python" -> pythonVersion),
+      st.extract.get(baseDirectory), "version.json"
+    )
     st
   }
 

diff --git a/project/SpytPlugin.scala b/project/SpytPlugin.scala
@@ -27,12 +27,14 @@ object SpytPlugin extends AutoPlugin {
 
     val spytPublishSnapshot = taskKey[Unit]("Publish spyt with snapshot version")
 
-    val spytPublishRelease = taskKey[Unit]("Publish spyt with release version")
+    val spytDistributive = taskKey[Unit]("Build SPYT distributive")
+    val spytBuildRelease = taskKey[Unit]("Build spyt with release version")
 
     val prepareBuildDirectory = taskKey[Unit]("")
-    val spytPublish = taskKey[Unit]("Publish spyt")
 
-    val spytPublishLibraries = taskKey[Unit]("Publish spyt libraries")
+    val publishToYt = taskKey[Unit]("Publish spyt distributive to YTsaurus cluster")
+    val publishToPypi = taskKey[Unit]("Publish SPYT Python libraries to PyPi")
+    val publishToMavenCentral = taskKey[Unit]("Publish SPYT Java libraries to Maven Central")
 
     val spytUpdatePythonVersion = taskKey[Unit]("Update versions in data-source/version.py")
 
@@ -44,13 +46,11 @@ object SpytPlugin extends AutoPlugin {
     val spytVersionPyFile = settingKey[File]("Spyt client version")
     val spytBuildDirectory = settingKey[File]("Build directory")
 
-    def publishRepoEnabled: Boolean = Option(System.getProperty("publishRepo")).exists(_.toBoolean)
-    def publishMavenCentralEnabled: Boolean = Option(System.getProperty("publishMavenCentral")).exists(_.toBoolean)
-    def publishYtEnabled: Boolean = Option(System.getProperty("publishYt")).forall(_.toBoolean)
+    def publishYtEnabled: Boolean = Option(System.getProperty("proxies")).exists(_.nonEmpty)
     def customSpytVersion: Option[String] = Option(System.getProperty("customSpytVersion"))
     def gpgPassphrase: Option[String] = Option(System.getProperty("gpg.passphrase"))
 
-    private def getBuildDirectory(rootDirectory: File): File = {
+    def getBuildDirectory(rootDirectory: File): File = {
       rootDirectory / "build_output"
     }
 
@@ -125,7 +125,7 @@ object SpytPlugin extends AutoPlugin {
       runProcess(state.value, spytSnapshotProcess)
     },
 
-    spytPublishRelease := {
+    spytBuildRelease := {
       runProcess(state.value, spytReleaseProcess)
     }
   )

diff --git a/project/SpytRelease.scala b/project/SpytRelease.scala
@@ -9,33 +9,16 @@ import spyt.SpytPlugin.autoImport._
 
 object SpytRelease {
 
-  lazy val spytReleaseProcess: Seq[ReleaseStep] = testProcess ++ Seq(
+  lazy val spytReleaseProcess: Seq[ReleaseStep] = Seq(
     ReleaseStep(releaseStepTask(prepareBuildDirectory)),
     minorReleaseVersions,
     setReleaseSpytVersion,
   ) ++ setCustomVersions ++ Seq(
     ReleaseStep(releaseStepTask(spytUpdatePythonVersion)),
-    ReleaseStep(releaseStepTask(spytPublish)),
-    ReleaseStep(releaseStepTask(spytPublishLibraries)),
-    dumpVersions,
-    setNextSpytVersion,
-    ReleaseStep(releaseStepTask(spytUpdatePythonVersion)),
-    logSpytVersion
+    ReleaseStep(releaseStepTask(spytDistributive)),
+    dumpVersions
   )
 
-  private lazy val testProcess: Seq[ReleaseStep] = {
-    val skipTests = Option(System.getProperty("skipTests")).forall(_.toBoolean)
-    if (skipTests) {
-      Nil
-    } else {
-      Seq(
-        checkSnapshotDependencies,
-        runClean,
-        runTest
-      )
-    }
-  }
-
   private def releaseMinorVersions(versions: SettingKey[Versions],
                                    st: State,
                                    versionSetting: SettingKey[String]): State = {

diff --git a/project/SpytSnapshot.scala b/project/SpytSnapshot.scala
@@ -18,8 +18,8 @@ object SpytSnapshot {
     spytSnapshotVersions,
     setSpytSnapshotVersion,
     releaseStepTask(spytUpdatePythonVersion),
-    releaseStepTask(spytPublish),
-    dumpVersions
+    releaseStepTask(spytDistributive),
+    releaseStepTask(publishToYt)
   )
 
   case class SnapshotVersion(main: String,

diff --git a/yt-publish-plugin/src/main/scala/YtPublishPlugin.scala b/yt-publish-plugin/src/main/scala/YtPublishPlugin.scala
@@ -163,10 +163,7 @@ object YtPublishPlugin extends AutoPlugin {
     }
 
     def ytProxies: Seq[String] = {
-      val envProxy = Option(System.getenv("YT_PROXY"))
-      val propProxy = Option(System.getProperty("proxies"))
-      val proxy = propProxy.orElse(envProxy)
-      proxy match {
+      Option(System.getProperty("proxies")) match {
         case Some(value) => value.split(",").toSeq
         case None => Nil
       }
@@ -176,8 +173,8 @@ object YtPublishPlugin extends AutoPlugin {
     def configGenerationEnabled: Boolean = Option(System.getProperty("configGeneration")).forall(_.toBoolean)
     def innerSidecarConfigEnabled: Boolean = Option(System.getProperty("innerSidecarConfig")).exists(_.toBoolean)
 
-    val publishYt = taskKey[Unit]("Publish to yt directory")
-    val publishYtArtifacts = taskKey[Seq[YtPublishArtifact]]("Yt publish artifacts")
+    val publishYt = taskKey[Unit]("Publish SPYT artifacts to YTsaurus cluster")
+    val publishYtArtifacts = taskKey[Seq[YtPublishArtifact]]("Artifacts to publish on the cluster")
   }
 
   import autoImport._
@@ -272,7 +269,7 @@ object YtPublishPlugin extends AutoPlugin {
             }
       }
       if (ytProxies.isEmpty) {
-        log.warn("No yt proxies provided. Use `proxies` property or `YT_PROXY` environment variable")
+        log.warn("No yt proxies provided via `proxies` property.")
       }
       ytProxies.par.foreach { proxy =>
         val (ytClient, connector) = createYtClient(proxy, creds)