feat：蓝盾本地开发实现 #9576

helm-charts/core/ci/base/default_env.yaml

@@ -54,3 +54,5 @@ bkCiSm4Enabled: false
 bkCiMonitorRegister: false
 bkCiBuildDataClearSwitch: false
 bkCiOpenapiApiAuth: false
+bkCiLocalRunEnabled: false
+bkCiLocalRunTarget: host.docker.internal:8081

scripts/bkenv.properties

@@ -231,6 +231,10 @@ BK_CI_KUBERNETES_WEBCONSOLE_PROXY=
 BK_CI_SM4_KEY=s31^dDjd!3k
 # BK_CI_SM4_ENABLED
 BK_CI_SM4_ENABLED=false
+# BK_CI_LOCAL_RUN_ENABLED
+BK_CI_LOCAL_RUN_ENABLED=false
+# BK_CI_LOCAL_RUN_TARGET
+BK_CI_LOCAL_RUN_TARGET=host.docker.internal:8081
 
 ##########
 # 3-微服务配置

src/backend/ci/.gitignore

@@ -17,3 +17,5 @@ build.yml
 .codecc
 *.log.gz
 i18n/
+core/multijar/boot-multijar/src/main/resources/application-*.yml
+core/multijar/boot-multijar/bkenv.properties

src/backend/ci/buildSrc/build.gradle.kts

@@ -22,6 +22,7 @@ dependencies {
     implementation("org.apache.logging.log4j:log4j-core:2.17.1")
     implementation("org.owasp:dependency-check-gradle:7.1.0.1")
     implementation("com.google.cloud.tools:jib-gradle-plugin:3.3.1")
+    implementation("org.springframework.boot:spring-boot-gradle-plugin:2.6.13")
     implementation("com.github.jk1:gradle-license-report:1.17") // 2.x need gradle-7.0
     implementation("com.mysql:mysql-connector-j:8.0.33")
 }

src/backend/ci/buildSrc/src/main/kotlin/plugins/task-docker-build.gradle.kts

@@ -31,19 +31,15 @@ plugins {
 val toImageRepo = System.getProperty("to.image.repo")
 val toImageTag = System.getProperty("to.image.tag")
 var toImage = System.getProperty("jib.to.image")
-
 // 加这个判断 , 主要是为了编译kts时不报错
 if (toImage.isNullOrBlank() || (toImageRepo.isNullOrBlank() && toImageTag.isNullOrBlank())) {
     val service = name.replace("boot-", "").replace("-tencent", "")
-
     if (toImage.isNullOrBlank() && !toImageRepo.isNullOrBlank()) {
         toImage = toImageRepo.let {
             if (toImageRepo.endsWith("/")) it else it + "/"
         } + "bkci-" + service + ":" + toImageTag
     }
-
     val configNamespace = System.getProperty("config.namespace")
-
     val jvmFlagList = System.getProperty("jvmFlags.file")?.let { File(it).readLines() } ?: emptyList()
 
     val finalJvmFlags = mutableListOf(
@@ -53,7 +49,7 @@ if (toImage.isNullOrBlank() || (toImageRepo.isNullOrBlank() && toImageTag.isNull
         "-XX:+PrintTenuringDistribution",
         "-XX:+PrintGCDetails",
         "-XX:+PrintGCDateStamps",
-        "-XX:MaxGCPauseMillis=200",
+        "-XX:MaxGCPauseMillis=100",
         "-XX:+UseG1GC",
         "-XX:NativeMemoryTracking=summary",
         "-XX:+HeapDumpOnOutOfMemoryError",
@@ -81,6 +77,7 @@ if (toImage.isNullOrBlank() || (toImageRepo.isNullOrBlank() && toImageTag.isNull
         "-Dio.undertow.legacy.cookie.ALLOW_HTTP_SEPARATORS_IN_V0=true",
         "-Dserver.port=80"
     )
+
     finalJvmFlags.addAll(jvmFlagList)
 
     jib {

src/backend/ci/buildSrc/src/main/kotlin/plugins/task-multi-boot-jar.gradle.kts

@@ -0,0 +1,50 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2019 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+tasks.register("multiBootJar") {
+    val finalModules = System.getProperty("devops.multi.from") ?: localRunMultiModules
+    val finalModulesList = finalModules.split(",").toMutableList()
+    rootProject.subprojects.filter {
+        isSpecifiedModulePath(it.path, finalModulesList)
+    }.forEach { subProject -> addDependencies(subProject.path) }
+    dependsOn("copyToRelease")
+}
+
+fun isSpecifiedModulePath(path: String, multiModuleList: List<String>): Boolean {
+    if (!path.contains("core"))
+        return false
+    return path.contains("biz") && multiModuleList.any { module -> path.contains(module) }
+}
+
+fun addDependencies(path: String) {
+    dependencies {
+        add("implementation", project(path))
+    }
+}
+
+val localRunMultiModules = "process,auth,image,environment,repository,ticket,project," +
+    "notify,quality,dispatch,dispatch-docker,dispatch-kubernetes,artifactory," +
+    "monitoring,plugin,websocket,worker,misc,store,log"

src/backend/ci/buildSrc/src/main/kotlin/plugins/task-multi-boot-run.gradle.kts

@@ -0,0 +1,69 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2019 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+import org.springframework.boot.gradle.tasks.bundling.BootJar
+import org.springframework.boot.gradle.tasks.run.BootRun
+import java.util.Properties
+
+tasks.register<BootRun>("multiBootRun") {
+    doFirst {
+        systemProperty("devops.multi.from", localRunMultiServices)
+        systemProperty("spring.datasource.url", System.getProperty("mysqlURL"))
+        systemProperty("spring.datasource.username", System.getProperty("mysqlUser"))
+        systemProperty("spring.datasource.password",  System.getProperty("mysqlPasswd"))
+        systemProperty("spring.main.allow-circular-references", "true")
+        systemProperty("spring.cloud.config.enabled", "false")
+        systemProperty("spring.cloud.config.fail-fast", "true")
+        systemProperty("spring.jmx.enabled", "true")
+        systemProperty("jasypt.encryptor.bootstrap", "false")
+        systemProperty("sun.jnu.encoding", "UTF-8")
+        systemProperty("file.encoding", "UTF-8")
+        systemProperty("spring.cloud.consul.enabled", "false")
+        systemProperty("spring.cloud.consul.discovery.enabled", "false")
+        systemProperty("server.port", "8081")
+        systemProperty("local.run", "true")
+        systemProperty("service.log.dir", joinPath(projectDir.absolutePath, "log"))
+    }
+    dependsOn("multiBootJar")
+    val bootJarTask = tasks.getByName<BootJar>("bootJar")
+    mainClass.set(bootJarTask.mainClass)
+    classpath = bootJarTask.classpath
+}
+
+fun loadProperties(basePath: String): Properties {
+    val properties = Properties()
+    properties.load(file(basePath).inputStream())
+    return properties
+}
+
+/**
+ * 返回路径
+ */
+fun joinPath(vararg folders: String) = folders.joinToString(File.separator)
+tasks.getByName("compileKotlin").dependsOn("replacePlaceholders")
+
+val localRunMultiServices = "process,auth,environment,repository,ticket,project,notify,quality,dispatch," +
+    "dispatch-kubernetes,artifactory,monitoring,plugin,websocket,worker,misc,store,log,image"

src/backend/ci/buildSrc/src/main/kotlin/plugins/task-render-template.gradle.kts

@@ -0,0 +1,108 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2019 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+import java.util.Properties
+
+tasks.register("replacePlaceholders") {
+    doLast {
+        val destDir = joinPath(projectDir.absolutePath, "src", "main", "resources")
+        // 每次启动时，首先清空application文件，再根据模板文件重新渲染
+        deleteFiles(destDir)
+
+        val rootDirPath = rootDir.absolutePath.replace(
+            "${File.separator}src${File.separator}backend${File.separator}ci",
+            ""
+        )
+        // 基础变量
+        val baseBkEnvPath = joinPath(rootDirPath, "scripts", "bkenv.properties")
+        val bkEnvProperties = loadProperties(baseBkEnvPath)
+        val bkEnvFileContent = renderTemplate(baseBkEnvPath, bkEnvProperties)
+        bkEnvProperties.load(bkEnvFileContent.byteInputStream())
+        // 自定义变量
+        val bkEnvPath = joinPath(projectDir.absolutePath, "bkenv.properties")
+        file(bkEnvPath).let {
+            if (it.exists()) {
+                bkEnvProperties.load(it.inputStream())
+            }
+        }
+        // 渲染模板
+        val templatesDir = joinPath(rootDirPath, "support-files", "templates")
+        generateFiles(templatesDir, destDir, bkEnvProperties)
+    }
+}
+
+tasks.register("deleteApplicationTemplate") {
+    val destDir = joinPath(projectDir.absolutePath, "src", "main", "resources")
+    deleteFiles(destDir)
+}
+
+fun deleteFiles(dir: String) {
+    val destDirFile = File(dir)
+    destDirFile.listFiles()?.forEach { file ->
+        if (file.isFile && file.name != "application.yml") {
+            file.delete()
+        }
+    }
+}
+
+fun loadProperties(basePath: String): Properties {
+    val properties = Properties()
+    properties.load(file(basePath).inputStream())
+    return properties
+}
+
+fun renderTemplate(templatePath: String, properties: Properties): String {
+    var templateContent = file(templatePath).readText()
+    properties.forEach { key, value ->
+        templateContent = templateContent.replace("$${key}", value.toString())
+    }
+    return templateContent
+}
+
+fun generateFiles(templatesDir: String, destDir: String, properties: Properties) {
+    val templatesDirFile = file(templatesDir)
+    templatesDirFile.walkTopDown().forEach { templateFile ->
+        if (templateFile.isFile && templateFile.name.endsWith(".yml")) {
+            var content = templateFile.readText()
+            properties.forEach { key, value ->
+                content = content.replace("__${key}__", value.toString())
+            }
+            val destFileName = if (templateFile.name == "#etc#ci#common.yml") {
+                "application-common.yml"
+            } else {
+                templateFile.name.replace("#etc#ci#", "")
+            }
+            val destFile = File(joinPath(destDir, destFileName))
+            destFile.parentFile.mkdirs()
+            destFile.writeText(content)
+        }
+    }
+}
+
+/**
+ * 返回路径
+ */
+fun joinPath(vararg folders: String) = folders.joinToString(File.separator)

src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/common/AuthCoreConfiguration.kt

@@ -38,6 +38,7 @@ import com.tencent.devops.common.client.ClientTokenService
 import com.tencent.devops.common.event.dispatcher.pipeline.mq.MeasureEventDispatcher
 import com.tencent.devops.common.event.dispatcher.pipeline.mq.MQ
 import com.tencent.devops.common.web.mq.EXTEND_RABBIT_TEMPLATE_NAME
+import com.tencent.devops.common.event.dispatcher.pipeline.mq.Tools
 import org.springframework.amqp.core.Binding
 import org.springframework.amqp.core.BindingBuilder
 import org.springframework.amqp.core.DirectExchange
@@ -120,17 +121,18 @@ class AuthCoreConfiguration {
         @Autowired refreshListener: AuthRefreshEventListener,
         @Autowired messageConverter: Jackson2JsonMessageConverter
     ): SimpleMessageListenerContainer {
-        val container = SimpleMessageListenerContainer(connectionFactory)
-        container.setQueueNames(authRefreshQueue.name)
-        container.setConcurrentConsumers(1)
-        container.setMaxConcurrentConsumers(10)
-        container.setAmqpAdmin(rabbitAdmin)
-        container.setStartConsumerMinInterval(5000)
-        container.setConsecutiveActiveTrigger(5)
         val adapter = MessageListenerAdapter(refreshListener, refreshListener::execute.name)
         adapter.setMessageConverter(messageConverter)
-        container.setMessageListener(adapter)
-        return container
+        return Tools.createSimpleMessageListenerContainerByAdapter(
+            connectionFactory = connectionFactory,
+            queue = authRefreshQueue,
+            rabbitAdmin = rabbitAdmin,
+            adapter = adapter,
+            startConsumerMinInterval = 5000,
+            consecutiveActiveTrigger = 5,
+            concurrency = 1,
+            maxConcurrency = 5
+        )
     }
 
     @Bean

src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/util/PropertyUtil.kt

@@ -72,4 +72,9 @@ object PropertyUtil {
         properties.load(propertyFileContent.reader())
         return properties[propertyKey]?.toString()
     }
+
+    /**
+     * 判断是否为本地运行
+     */
+    fun isLocalRun() = System.getProperty("local.run") == "true"
 }

src/backend/ci/core/common/common-client/src/main/resources/META-INF/spring.factories

@@ -1,2 +1,2 @@
 org.springframework.boot.autoconfigure.EnableAutoConfiguration=\
-com.tencent.devops.common.client.ClientAutoConfiguration
+com.tencent.devops.common.client.ClientAutoConfiguration

src/backend/ci/core/common/common-db/src/main/kotlin/com/tencent/devops/common/db/DBAutoConfiguration.kt

@@ -33,6 +33,7 @@ import com.zaxxer.hikari.HikariDataSource
 import org.springframework.beans.factory.annotation.Value
 import org.springframework.boot.autoconfigure.AutoConfigureBefore
 import org.springframework.boot.autoconfigure.AutoConfigureOrder
+import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingClass
 import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration
 import org.springframework.boot.autoconfigure.jooq.JooqAutoConfiguration
 import org.springframework.context.annotation.Bean
@@ -54,6 +55,7 @@ import javax.sql.DataSource
 @AutoConfigureBefore(DataSourceAutoConfiguration::class, JooqAutoConfiguration::class)
 @Import(DBBaseConfiguration::class)
 @EnableTransactionManagement
+@ConditionalOnMissingClass("com.tencent.devops.multijar.MultijarDslContextConfiguration")
 class DBAutoConfiguration {
 
     @Value("\${spring.datasource.url:#{null}}")

src/backend/ci/core/common/common-dispatch-sdk/src/main/kotlin/com/tencent/devops/common/dispatch.sdk/config/MQConfiguration.kt

@@ -48,9 +48,11 @@ import org.springframework.amqp.support.converter.Jackson2JsonMessageConverter
 import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.beans.factory.annotation.Value
 import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean
+import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingClass
 import org.springframework.context.annotation.Bean
 
 @Suppress("ALL")
+@ConditionalOnMissingClass("com.tencent.devops.multijar.MultijarDslContextConfiguration")
 class MQConfiguration @Autowired constructor() {
 
     @Value("\${dispatch.demoteQueue.concurrency:2}")