NO-ISSUE: Jenkins jobs updates and fixes for Apache 10 release

.ci/jenkins/Jenkinsfile.build-image

@@ -87,19 +87,23 @@ pipeline {
 
                     if (getMavenArtifactRepository()) {
                         echo "Setup Repository url to '${getMavenArtifactRepository()}'"
-                        runPythonCommand("python scripts/update-repository.py  --repo-url ${getMavenArtifactRepository()}")
+                        runPythonCommand("python scripts/update-repository.py --repo-urls ${getMavenArtifactRepository()}")
                     }
                 }
             }
         }
 
-        stage('Update kogito version') {
+        stage('Update project version') {
             when {
                 expression { params.UPDATE_KOGITO_VERSION != '' }
             }
             steps {
                 script {
-                    runPythonCommand("python scripts/manage-kogito-version.py --bump-to ${params.UPDATE_KOGITO_VERSION} --confirm")
+                    versionCmd = "python scripts/manage-kogito-version.py --bump-to ${params.UPDATE_KOGITO_VERSION} --confirm"
+                    if (isRelease()) {
+                        versionCmd += " --examples-ref ${getSourceBranch()}"
+                    }
+                    sh versionCmd
                 }
             }
         }
@@ -128,7 +132,7 @@ pipeline {
 
                     int freePort = cloud.findFreePort()
                     env.localRegistryUrl = cloud.startLocalRegistry(freePort)
-                    cloud.prepareForDockerMultiplatformBuild([env.localRegistryUrl],[cloud.getDockerIOMirrorRegistryConfig()], false)
+                    cloud.prepareForDockerMultiplatformBuild([env.localRegistryUrl], [cloud.getDockerIOMirrorRegistryConfig()], false)
 
                     // Generate the Dockerfile
                     runPythonCommand("make build-image ${getMakeBuildImageArgs()} image_name=${getBuildImageName()} ignore_test=true ignore_tag=true build_options='--dry-run'")
@@ -184,7 +188,7 @@ pipeline {
                 script {
                     updateGithubCommitStatus('PENDING', 'Tests in progress')
 
-                    String testImageTag = "docker.io/apache/incubator-kie-${getBuildImageName()}:${getImageVersion()}"
+                    String testImageTag = "${getDeployImageRegistry()}/${getDeployImageNamespace()}/incubator-kie-${getBuildImageName()}:${getImageVersion()}"
                     String builtImageTag = getBuiltImageTag()
                     // Pull and tag to test image
                     sh """
@@ -208,15 +212,17 @@ pipeline {
             }
             steps {
                 script {
-                    def resultingFileName = "incubator-kie-${getImageArtifactReleaseVersion()}-${getBuildImageName()}.tar.gz"
-                    def signatureFileName = "${resultingFileName}.asc"
+                    String resultingFileName = "incubator-kie-${getImageArtifactReleaseVersion()}-${getBuildImageName()}-image.tar.gz"
+                    String signatureFileName = "${resultingFileName}.asc"
+                    String checksumFileName = "${resultingFileName}.sha512"
                     sh """
                         docker pull ${getBuiltImageTag()}
-                        docker save ${getBuiltImageTag()} | gzip > ${resultingFileName}
+                        docker tag ${getBuiltImageTag()} ${getBuiltDeployImageTag()}
+                        docker save ${getBuiltDeployImageTag()} | gzip > ${resultingFileName}
                     """
-                    release.gpgImportKeyFromStringWithoutPassword(getReleaseGpgSignKeyCredsId())
-                    release.gpgSignFileDetachedSignatureWithoutPassword(resultingFileName, signatureFileName)
-                    release.svnUploadFileToRepository(getReleaseSvnRepository(), getReleaseSvnCredsId(), getImageArtifactReleaseVersion(), resultingFileName, signatureFileName)
+                    releaseUtils.gpgImportKeyFromStringWithoutPassword(getReleaseGpgSignKeyCredsId())
+                    releaseUtils.gpgSignFileDetachedSignatureWithoutPassword(resultingFileName, signatureFileName)
+                    releaseUtils.svnUploadFileToRepository(getReleaseSvnRepository(), getReleaseSvnCredsId(), getImageArtifactReleaseVersion(), resultingFileName, signatureFileName, checksumFileName)
                 }
             }
             post {
@@ -269,9 +275,13 @@ String getImageVersion() {
     return runPythonCommand('make display-image-version', true).trim()
 }
 
+String getBuiltDeployImageTag(String imageTag = '') {
+    return "${getDeployImageRegistry()}/${getDeployImageNamespace()}/${getFinalImageName()}:${imageTag ?: getDeployImageTag()}"
+}
+
 String getBuiltImageTag(String imageTag = '') {
     if (shouldDeployImage()) {
-        return "${getDeployImageRegistry()}/${getDeployImageNamespace()}/${getFinalImageName()}:${imageTag ?: getDeployImageTag()}"
+        return getBuiltDeployImageTag(imageTag)
     }
     return "${env.localRegistryUrl}/${getBuildImageName()}:${githubscm.getCommitHash()}"
 }
@@ -316,7 +326,7 @@ String getDeployImageTag() {
 }
 
 String getFinalImageName() {
-    return "incubator-kie-" + getBuildImageName() + (getDeployImageNameSuffix() ? "-${getDeployImageNameSuffix()}" : '')
+    return 'incubator-kie-' + getBuildImageName() + (getDeployImageNameSuffix() ? "-${getDeployImageNameSuffix()}" : '')
 }
 
 boolean isDeployLatestTag() {
@@ -440,7 +450,7 @@ String getTriggeringProjectName() {
     if (currentBuild.upstreamBuilds) {
         return currentBuild.upstreamBuilds[0].projectName
     }
-    return ""
+    return ''
 }
 String getReleaseGpgSignKeyCredsId() {
     return env.RELEASE_GPG_SIGN_KEY_CREDS_ID
@@ -459,5 +469,9 @@ String getReleaseSvnCredsId() {
 }
 
 String getImageArtifactReleaseVersion() {
-    return env.IMAGE_ARTIFACT_RELEASE_VERSION
+    return params.IMAGE_ARTIFACT_RELEASE_VERSION
+}
+
+boolean isRelease() {
+    return env.RELEASE ? env.RELEASE.toBoolean() : false
 }

.ci/jenkins/Jenkinsfile.deploy

@@ -18,15 +18,13 @@
 
 deployProperties = [:]
 
-changesDone = false
-
 BUILT_IMAGES = Collections.synchronizedList([])
 BUILD_FAILED_IMAGES = Collections.synchronizedList([])
 TEST_FAILED_IMAGES = Collections.synchronizedList([])
 
 pipeline {
     agent {
-        docker { 
+        docker {
             image env.AGENT_DOCKER_BUILDER_IMAGE
             args env.AGENT_DOCKER_BUILDER_ARGS
             label util.avoidFaultyNodes()
@@ -73,22 +71,6 @@ pipeline {
             }
         }
 
-        stage('Prepare for PR') {
-            when {
-                expression { return isRelease() }
-            }
-            steps {
-                script {
-                    dir(getRepoName()) {
-                        if (githubscm.isBranchExist('origin', getPRBranch())) {
-                            githubscm.removeRemoteBranch('origin', getPRBranch(), getGitAuthorPushCredsId())
-                        }
-                        githubscm.createBranch(getPRBranch())
-                    }
-                }
-            }
-        }
-
         stage('Setup Quarkus platform version') {
             when {
                 expression { isRelease() }
@@ -117,14 +99,12 @@ pipeline {
                     dir(getRepoName()) {
                         versionCmd = "python scripts/manage-kogito-version.py --bump-to ${getProjectVersion()} --confirm"
                         if (getBuildBranch() != 'main') {
-                            versionCmd += " --examples-ref ${!isRelease() ? 'nightly-' : ''}${getBuildBranch()}"
+                            versionCmd += " --examples-ref ${getBuildBranch()}"
                         }
                         if (getKogitoArtifactsVersion()) {
                             versionCmd += " --artifacts-version ${getKogitoArtifactsVersion()}"
                         }
                         sh versionCmd
-
-                        commitAndPushChanges("Update project version to ${getProjectVersion()}")
                     }
                 }
             }
@@ -173,62 +153,16 @@ pipeline {
             }
         }
 
-        stage('Create PR') {
+        stage('Commit and Tag changes') {
             when {
-                expression {
-                    return changesDone && isRelease()
-                }
+                expression { return isRelease() }
             }
             steps {
                 script {
-                    dir(getRepoName()) {
-                        def commitMsg = "[${getBuildBranch()}] Update Maven artifacts"
-                        def prBody = "Generated by build ${BUILD_TAG}: ${BUILD_URL}."
-                        if (isRelease()) {
-                            commitMsg = "[${getBuildBranch()}] Update project version to ${getProjectVersion()}"
-                            prBody += '\nPlease do not merge, it should be merged automatically.'
-                        } else if (currentBuild.currentResult != 'SUCCESS') {
-                            commitMsg += " (${currentBuild.currentResult})"
-                            prBody += '\n\nSuccessful images:\n'
-                            getBuiltImages().each {
-                                prBody += "- ${it}\n"
-                            }
-                            if (getBuildFailedImages()) {
-                                prBody += '\nBuild failures on those images:\n'
-                                getBuildFailedImages().each {
-                                    prBody += "- ${it}\n"
-                                }
-                            } else {
-                                prBody += '\nImages were all successfully built but some other problem occured in the pipeline execution...\n'
-                            }
-                            if (getTestFailedImages()) {
-                                prBody += '\nTest failures on those images:\n'
-                                getTestFailedImages().each {
-                                    prBody += "- ${it}\n"
-                                }
-                            } else {
-                                prBody += '\nImages were all successfully built but some other problem occured in the pipeline execution...\n'
-                            }
-                            prBody += '\nSee build url above for more information'
+                    if (!getBuildFailedImages()) {
+                        dir(getRepoName()) {
+                            commitAndTagChanges("[${getBuildBranch()}] Update version to ${getProjectVersion()}")
                         }
-                        String prLink = githubscm.createPR(commitMsg, prBody, getBuildBranch(), getGitAuthorPushCredsId())
-                        deployProperties["${getRepoName()}.pr.link"] = prLink
-
-                        // Create a new tag
-                        githubscm.setUserConfigFromCreds(getGitAuthorPushCredsId())
-                        githubscm.tagRepository(getGitTagName())
-                        githubscm.pushRemoteTag('origin', getGitTagName(), getGitAuthorPushCredsId())
-                    }
-                }
-            }
-            post {
-                always {
-                    script {
-                        setDeployPropertyIfNeeded("${getRepoName()}.pr.source.uri", "https://github.com/${getGitAuthor()}/${getRepoName()}")
-                        setDeployPropertyIfNeeded("${getRepoName()}.pr.source.ref", getPRBranch())
-                        setDeployPropertyIfNeeded("${getRepoName()}.pr.target.uri", "https://github.com/${getGitAuthor()}/${getRepoName()}")
-                        setDeployPropertyIfNeeded("${getRepoName()}.pr.target.ref", getBuildBranch())
-                        setDeployPropertyIfNeeded("${getRepoName()}.tag.name", getGitTagName())
                     }
                 }
             }
@@ -284,11 +218,11 @@ void checkoutRepo() {
     checkout(githubscm.resolveRepository(getRepoName(), getGitAuthor(), getBuildBranch(), false, getGitAuthorCredsId()))
 }
 
-void commitAndPushChanges(String commitMsg) {
+void commitAndTagChanges(String commitMsg) {
     githubscm.setUserConfigFromCreds(getGitAuthorPushCredsId())
     githubscm.commitChanges(commitMsg)
-    githubscm.pushObject('origin', getPRBranch(), getGitAuthorPushCredsId())
-    changesDone = true
+    githubscm.tagRepository(getGitTagName())
+    githubscm.pushRemoteTag('origin', getGitTagName(), getGitAuthorPushCredsId())
 }
 
 void createBuildAndTestStageClosure(String image) {
@@ -297,16 +231,18 @@ void createBuildAndTestStageClosure(String image) {
             List buildParams = []
             buildParams.add(string(name: 'DISPLAY_NAME', value: "${params.DISPLAY_NAME} - ${image}"))
             buildParams.add(string(name: 'BUILD_IMAGE_NAME', value: image))
-            buildParams.add(string(name: 'SOURCE_AUTHOR', value: isRelease() ? getGitAuthor() : getGitAuthor()))
-            buildParams.add(string(name: 'SOURCE_BRANCH', value: isRelease() ? getPRBranch() : getBuildBranch()))
+            buildParams.add(string(name: 'SOURCE_AUTHOR', value: getGitAuthor()))
+            buildParams.add(string(name: 'SOURCE_BRANCH', value: getBuildBranch()))
 
-            buildParams.add(string(name: 'MAVEN_ARTIFACTS_REPOSITORY', value: env.MAVEN_ARTIFACT_REPOSITORY ?: (isRelease() ? env.DEFAULT_STAGING_REPOSITORY : '')))
+            buildParams.add(string(name: 'MAVEN_ARTIFACTS_REPOSITORY', value: env.MAVEN_ARTIFACT_REPOSITORY))
             buildParams.add(string(name: 'BUILD_KOGITO_APPS_URI', value: params.APPS_URI))
             buildParams.add(string(name: 'BUILD_KOGITO_APPS_REF', value: params.APPS_REF))
             buildParams.add(booleanParam(name: 'SKIP_TESTS', value: params.SKIP_TESTS))
             buildParams.add(string(name: 'TESTS_KOGITO_EXAMPLES_URI', value: params.EXAMPLES_URI))
             buildParams.add(string(name: 'TESTS_KOGITO_EXAMPLES_REF', value: params.EXAMPLES_REF))
 
+            buildParams.add(string(name: 'UPDATE_KOGITO_VERSION', value: isRelease() ? getProjectVersion() : ''))
+
             buildParams.add(booleanParam(name: 'DEPLOY_IMAGE', value: isDeployImage()))
             buildParams.add(booleanParam(name: 'DEPLOY_IMAGE_USE_OPENSHIFT_REGISTRY', value: isDeployImageInOpenshiftRegistry()))
             buildParams.add(string(name: 'DEPLOY_IMAGE_REGISTRY_USER_CREDENTIALS_ID', value: getDeployImageRegistryUserCredentialsId()))
@@ -364,7 +300,7 @@ List getTestFailedImages() {
 ////////////////////////////////////////////////////////////////////////
 
 boolean isDeployImage() {
-    return !Boolean.valueOf(env.DISABLE_IMAGES_DEPLOY)
+    return !env.DISABLE_IMAGES_DEPLOY
 }
 
 boolean isDeployImageInOpenshiftRegistry() {

.ci/jenkins/dsl/jobs.groovy

@@ -151,8 +151,8 @@ void setupDeployJob(JobType jobType) {
             stringParam('KOGITO_ARTIFACTS_VERSION', '', 'Optional. If artifacts\' version is different from PROJECT_VERSION.')
             if (jobType == JobType.RELEASE) {
                 stringParam('QUARKUS_PLATFORM_VERSION', '', 'Allow to override the Quarkus Platform version')
-                stringParam('GIT_TAG_NAME', '', 'Git tag to be created')
             }
+            stringParam('GIT_TAG_NAME', '', 'Optional if not RELEASE. Tag to be created in the repository')
 
             stringParam('KOGITO_PR_BRANCH', '', 'PR branch name')
             booleanParam('SEND_NOTIFICATION', false, 'In case you want the pipeline to send a notification on CI channel for this run.')
@@ -176,8 +176,8 @@ void setupBuildImageJob(JobType jobType) {
 
         RELEASE_GPG_SIGN_KEY_CREDS_ID: Utils.getReleaseGpgSignKeyCredentialsId(this),
         RELEASE_GPG_SIGN_PASSPHRASE_CREDS_ID: Utils.getReleaseGpgSignPassphraseCredentialsId(this),
-        RELEASE_SVN_REPOSITORY: Utils.getReleaseSvnCredentialsId(this),
-        RELEASE_SVN_CREDS_ID: Utils.getReleaseSvnStagingRepository(this)
+        RELEASE_SVN_REPOSITORY: Utils.getReleaseSvnStagingRepository(this),
+        RELEASE_SVN_CREDS_ID: Utils.getReleaseSvnCredentialsId(this)
     ])
     KogitoJobTemplate.createPipelineJob(this, jobParams)?.with {
         logRotator {
@@ -216,7 +216,7 @@ void setupBuildImageJob(JobType jobType) {
             stringParam('DEPLOY_IMAGE_TAG', '', 'Image tag to use to deploy images')
             booleanParam('DEPLOY_WITH_LATEST_TAG', false, 'Set to true if you want the deployed images to also be with the `latest` tag')
             booleanParam('EXPORT_AND_GPG_SIGN_IMAGE', jobType == JobType.RELEASE, 'Set to true if should images be exported and signed.')
-            stringParam('IMAGE_ARTIFACT_RELEASE_VERSION', '', 'Set the release version to be attached to the images artifacts names')
+            stringParam('IMAGE_ARTIFACT_RELEASE_VERSION', '', 'Optional if not RELEASE. Set the release version to be attached to the images artifacts names')
         }
     }
 }

scripts/common.py

@@ -430,6 +430,19 @@ def update_maven_repo_in_setup_maven(repo_url, replace_default_repository):
         replacement = 'export MAVEN_REPO_URL="{}"'.format(repo_url)
     update_in_file(SETUP_MAVEN_SCRIPT, pattern, replacement)
 
+def update_maven_repos_in_setup_maven(repos_url):
+    """
+    Update maven repositories into setup-maven.sh script
+    :param repos_url: Maven repositories urls
+    """
+    repo_list = repos_url.split(',')
+    print("Set maven repos {} in setup-maven script".format(repo_list))
+    pattern = re.compile(r'(# export MAVEN_REPO_URL=.*)')
+    replacement = f"export MAVEN_REPOS={','.join(['REPO_' + str(i) for i, _ in enumerate(repo_list)])}\n"
+    for i, value in enumerate(repo_list):
+        replacement += f"export REPO_{i}_MAVEN_REPO_URL={value}\n"
+    update_in_file(SETUP_MAVEN_SCRIPT, pattern, replacement)
+
 def update_env_value(env_name, env_value):
     """
     Update environment value into the given yaml module/image file

scripts/update-repository.py

@@ -32,6 +32,7 @@
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description='Update Maven information in repo from the given artifact url and '
                                                  'version.')
+    parser.add_argument('--repo-urls', dest='repo_urls', help='Defines the urls of the repositories to setup maven, comma (,) separated')
     parser.add_argument('--repo-url', dest='repo_url', help='Defines the url of the repository to setup into the tests')
     parser.add_argument('--replace-default-repo', dest='replace_default_repo', default=False, action='store_true',
                         help='Enable if repo-url should replace the default repository')
@@ -53,6 +54,9 @@
     parser.add_argument('--tests-only', dest='tests_only', default=False, action='store_true', help='Update product modules/images')
     args = parser.parse_args()
 
+    if args.repo_urls:
+        common.update_maven_repos_in_setup_maven(args.repo_urls)
+
     if args.repo_url:
         common.update_maven_repo_in_build_config(args.repo_url, args.replace_default_repo)
         common.update_maven_repo_in_setup_maven(args.repo_url, args.replace_default_repo)