Skip to content

Commit

Permalink
feat: 指定模块统一打包插件 TencentBlueKing#8672
Browse files Browse the repository at this point in the history
  • Loading branch information
fcfang123 committed Oct 18, 2023
1 parent 0dc76b2 commit 07f8c40
Show file tree
Hide file tree
Showing 18 changed files with 68 additions and 68 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import com.tencent.devops.buildless.pojo.RejectedExecutionType
import com.tencent.devops.buildless.rejected.RejectedExecutionFactory
import com.tencent.devops.buildless.service.BuildLessContainerService
import com.tencent.devops.buildless.utils.CommonUtils
import com.tencent.devops.buildless.utils.BuildlessRedisUtils
import com.tencent.devops.buildless.utils.RedisUtils
import com.tencent.devops.buildless.utils.ThreadPoolName
import com.tencent.devops.buildless.utils.ThreadPoolUtils
import org.slf4j.LoggerFactory
Expand All @@ -22,7 +22,7 @@ import java.util.concurrent.locks.ReentrantLock

@Component
class ContainerPoolExecutor @Autowired constructor(
private val buildlessRedisUtils: BuildlessRedisUtils,
private val redisUtils: RedisUtils,
private val buildLessConfig: BuildLessConfig,
private val rejectedExecutionFactory: RejectedExecutionFactory,
private val buildLessContainerService: BuildLessContainerService
Expand All @@ -35,7 +35,7 @@ class ContainerPoolExecutor @Autowired constructor(
}

logger.info("$buildId|$vmSeqId|$executionCount left push buildLessReadyTask")
buildlessRedisUtils.leftPushBuildLessReadyTask(
redisUtils.leftPushBuildLessReadyTask(
BuildLessTask(
projectId = projectId,
pipelineId = pipelineId,
Expand Down Expand Up @@ -85,7 +85,7 @@ class ContainerPoolExecutor @Autowired constructor(

fun getContainerStatus(containerId: String): BuildLessPoolInfo? {
synchronized(containerId.intern()) {
return buildlessRedisUtils.getBuildLessPoolContainer(containerId)
return redisUtils.getBuildLessPoolContainer(containerId)
}
}

Expand Down Expand Up @@ -124,7 +124,7 @@ class ContainerPoolExecutor @Autowired constructor(
}

retry@ while (true) {
val idlePoolSize = buildlessRedisUtils.getIdlePoolSize()
val idlePoolSize = redisUtils.getIdlePoolSize()

// 无空闲容器时执行拒绝策略
logger.info("${buildLessStartInfo.buildId}|${buildLessStartInfo.vmSeqId} idlePoolSize: $idlePoolSize")
Expand All @@ -135,10 +135,10 @@ class ContainerPoolExecutor @Autowired constructor(
}

// 再次check idlePoolSize,有变更则retry
if (idlePoolSize != buildlessRedisUtils.getIdlePoolSize()) continue@retry
if (idlePoolSize != redisUtils.getIdlePoolSize()) continue@retry

// 已经进入需求池,资源池减1
buildlessRedisUtils.increIdlePool(-1)
redisUtils.increIdlePool(-1)
return false
}
} finally {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import java.util.concurrent.ThreadPoolExecutor
import java.util.concurrent.TimeUnit

@Configuration
class BuildlessScheduleConfig : SchedulingConfigurer {
class ScheduleConfig : SchedulingConfigurer {
override fun configureTasks(taskRegistrar: ScheduledTaskRegistrar) {
val scheduler = ScheduledThreadPoolExecutor(5)
scheduler.maximumPoolSize = 5
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,19 +29,19 @@ package com.tencent.devops.buildless.rejected

import com.tencent.devops.buildless.pojo.BuildLessStartInfo
import com.tencent.devops.buildless.pojo.BuildLessTask
import com.tencent.devops.buildless.utils.BuildlessRedisUtils
import com.tencent.devops.buildless.utils.RedisUtils
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.stereotype.Service

@Service
class JumpPolicy @Autowired constructor(
private val buildlessRedisUtils: BuildlessRedisUtils
private val redisUtils: RedisUtils
) : RejectedExecutionHandler {
override fun rejectedExecution(buildLessStartInfo: BuildLessStartInfo): Boolean {
with(buildLessStartInfo) {
logger.info("$buildId|$vmSeqId|$executionCount jump allocate container.")
buildlessRedisUtils.rightPushBuildLessReadyTask(
redisUtils.rightPushBuildLessReadyTask(
BuildLessTask(
projectId = projectId,
pipelineId = pipelineId,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,14 +28,14 @@
package com.tencent.devops.buildless.schedule

import com.tencent.devops.buildless.ContainerPoolExecutor
import com.tencent.devops.buildless.utils.BuildlessRedisUtils
import com.tencent.devops.buildless.utils.RedisUtils
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.scheduling.annotation.Scheduled
import org.springframework.stereotype.Component

@Component
class ContainerPoolSchedule @Autowired constructor(
private val buildlessRedisUtils: BuildlessRedisUtils,
private val redisUtils: RedisUtils,
private val containerPoolExecutor: ContainerPoolExecutor
) {
/**
Expand All @@ -44,9 +44,9 @@ class ContainerPoolSchedule @Autowired constructor(
@Scheduled(cron = "0 0/1 * * * ?")
fun execute() {
// 校准空闲池大小
if (buildlessRedisUtils.getBuildLessReadyTaskSize() == 0L) {
val idleContainerPoolSize = buildlessRedisUtils.getBuildLessPoolContainerIdle()
buildlessRedisUtils.setIdlePool(idleContainerPoolSize)
if (redisUtils.getBuildLessReadyTaskSize() == 0L) {
val idleContainerPoolSize = redisUtils.getBuildLessPoolContainerIdle()
redisUtils.setIdlePool(idleContainerPoolSize)
}

containerPoolExecutor.addContainer()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ import org.springframework.scheduling.annotation.Scheduled
import org.springframework.stereotype.Component

@Component
class BuildlessSystemInfoRunner {
class SystemInfoRunner {

@Scheduled(cron = "0/5 * * * * ?")
fun startCollect() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ import com.tencent.devops.buildless.utils.ENV_JOB_BUILD_TYPE
import com.tencent.devops.buildless.utils.ENV_KEY_BK_TAG
import com.tencent.devops.buildless.utils.ENV_KEY_GATEWAY
import com.tencent.devops.buildless.utils.RandomUtil
import com.tencent.devops.buildless.utils.BuildlessRedisUtils
import com.tencent.devops.buildless.utils.RedisUtils
import com.tencent.devops.common.api.util.ShaUtils
import com.tencent.devops.common.service.BkTag
import com.tencent.devops.common.service.config.CommonConfig
Expand All @@ -76,7 +76,7 @@ import kotlin.streams.toList
@Service
class BuildLessContainerService(
private val bkTag: BkTag,
private val buildlessRedisUtils: BuildlessRedisUtils,
private val redisUtils: RedisUtils,
private val commonConfig: CommonConfig,
private val buildLessConfig: BuildLessConfig
) {
Expand Down Expand Up @@ -139,8 +139,8 @@ class BuildLessContainerService(
httpDockerCli.startContainerCmd(container.id).exec()

logger.info("===> created container: $container, containerName: $containerName. ")
buildlessRedisUtils.setBuildLessPoolContainer(container.id, ContainerStatus.IDLE)
buildlessRedisUtils.increIdlePool(1)
redisUtils.setBuildLessPoolContainer(container.id, ContainerStatus.IDLE)
redisUtils.increIdlePool(1)
logger.info("===> buildLessPoolKey hset ${container.id} ${ContainerStatus.IDLE.name}.")
} catch (e: Exception) {
logger.error("===> failed to created container.", e)
Expand Down Expand Up @@ -181,7 +181,7 @@ class BuildLessContainerService(
ignored
)
} finally {
buildlessRedisUtils.deleteBuildLessPoolContainer(containerId)
redisUtils.deleteBuildLessPoolContainer(containerId)
}
}

Expand All @@ -206,18 +206,18 @@ class BuildLessContainerService(
}.toList()

// 不在containerIds列表内的同步删除缓存
val buildLessPoolContainerMap = buildlessRedisUtils.getBuildLessPoolContainerList()
val buildLessPoolContainerMap = redisUtils.getBuildLessPoolContainerList()
buildLessPoolContainerMap.forEach { (key, _) ->
if (!containerIds.contains(key)) {
buildlessRedisUtils.deleteBuildLessPoolContainer(key)
redisUtils.deleteBuildLessPoolContainer(key)
}
}

// 在containerIds但是不在缓存中的,补充缓存
containerIds.forEach {
if (!buildLessPoolContainerMap.keys.contains(it)) {
logger.info("Supplemental cache buildLessPoolKey hset $it ${ContainerStatus.IDLE.name}.")
buildlessRedisUtils.setBuildLessPoolContainer(it, ContainerStatus.IDLE)
redisUtils.setBuildLessPoolContainer(it, ContainerStatus.IDLE)
}
}
}
Expand All @@ -231,7 +231,7 @@ class BuildLessContainerService(
for (container in containerInfo) {
val startTime = httpDockerCli.inspectContainerCmd(container.id).exec().state.startedAt
// 是否已运行超过12小时
val buildLessPoolInfo = buildlessRedisUtils.getBuildLessPoolContainer(container.id)
val buildLessPoolInfo = redisUtils.getBuildLessPoolContainer(container.id)
if (checkStartTime(startTime) &&
(buildLessPoolInfo == null || buildLessPoolInfo.status == ContainerStatus.IDLE)) {
timeoutContainerList.add(container.id)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ import com.tencent.devops.buildless.ContainerPoolExecutor
import com.tencent.devops.buildless.client.DispatchClient
import com.tencent.devops.buildless.pojo.BuildLessTask
import com.tencent.devops.buildless.utils.ContainerStatus
import com.tencent.devops.buildless.utils.BuildlessRedisUtils
import com.tencent.devops.buildless.utils.RedisUtils
import org.slf4j.LoggerFactory
import org.springframework.scheduling.annotation.Async
import org.springframework.scheduling.annotation.AsyncResult
Expand All @@ -44,7 +44,7 @@ import java.util.concurrent.Future
*/
@Service
class BuildLessTaskService(
private val buildlessRedisUtils: BuildlessRedisUtils,
private val redisUtils: RedisUtils,
private val dispatchClient: DispatchClient,
private val containerPoolExecutor: ContainerPoolExecutor
) {
Expand All @@ -60,7 +60,7 @@ class BuildLessTaskService(
return AsyncResult(buildLessPoolInfo.buildLessTask)
}

val buildLessTask = buildlessRedisUtils.popBuildLessReadyTask()
val buildLessTask = redisUtils.popBuildLessReadyTask()
if (buildLessTask != null) {
try {
logger.info("****> container: $containerId claim buildLessTask: $buildLessTask")
Expand All @@ -70,14 +70,14 @@ class BuildLessTaskService(
)

logger.info("****> claim task buildLessPoolKey hset $containerId ${ContainerStatus.BUSY.name}.")
buildlessRedisUtils.setBuildLessPoolContainer(containerId, ContainerStatus.BUSY, buildLessTask)
redisUtils.setBuildLessPoolContainer(containerId, ContainerStatus.BUSY, buildLessTask)

return AsyncResult(buildLessTask)
} catch (e: Exception) {
// 异常时任务重新回队列
logger.info("****> container: $containerId claim buildLessTask: $buildLessTask get error, " +
"retry.", e)
buildlessRedisUtils.leftPushBuildLessReadyTask(buildLessTask)
redisUtils.leftPushBuildLessReadyTask(buildLessTask)

continue
}
Expand Down Expand Up @@ -105,7 +105,7 @@ class BuildLessTaskService(
return
}

val buildLessTask = buildlessRedisUtils.popBuildLessReadyTask()
val buildLessTask = redisUtils.popBuildLessReadyTask()
if (buildLessTask != null) {
logger.info("****> container: $containerId claim buildLessTask: $buildLessTask")
dispatchClient.updateContainerId(
Expand All @@ -114,7 +114,7 @@ class BuildLessTaskService(
)

logger.info("****> claim task buildLessPoolKey hset $containerId ${ContainerStatus.BUSY.name}.")
buildlessRedisUtils.setBuildLessPoolContainer(containerId, ContainerStatus.BUSY, buildLessTask)
redisUtils.setBuildLessPoolContainer(containerId, ContainerStatus.BUSY, buildLessTask)

deferredResult.setResult(buildLessTask)
return
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ import org.springframework.beans.factory.annotation.Autowired
import org.springframework.stereotype.Component

@Component
class BuildlessRedisUtils @Autowired constructor(
class RedisUtils @Autowired constructor(
private val redisOperation: RedisOperation,
private val objectMapper: ObjectMapper
) {
Expand Down Expand Up @@ -139,6 +139,6 @@ class BuildlessRedisUtils @Autowired constructor(
}

companion object {
private val logger = LoggerFactory.getLogger(BuildlessRedisUtils::class.java)
private val logger = LoggerFactory.getLogger(RedisUtils::class.java)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ import javax.ws.rs.core.MediaType
@Path("/dockerhost")
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.APPLICATION_JSON)@Suppress("ALL")
interface DispatchBuildDockerHostResource {
interface BuildDockerHostResource {
@GET
@Path("/resource-config/pipelines/{pipelineId}/vmSeqs/{vmSeqId}")
@ApiOperation("获取蓝盾项目的docker性能配置")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ import javax.ws.rs.core.MediaType
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.APPLICATION_JSON)
@ServiceInterface("dispatch-docker") // 指明接入到哪个微服务
interface ServiceDispatchDockerHostResource {
interface ServiceDockerHostResource {

@ApiOperation("获取dockerhost列表")
@GET
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,21 +29,21 @@ package com.tencent.devops.dispatch.docker.controller

import com.tencent.devops.common.api.pojo.Result
import com.tencent.devops.common.web.RestResource
import com.tencent.devops.dispatch.docker.api.builds.DispatchBuildDockerHostResource
import com.tencent.devops.dispatch.docker.api.builds.BuildDockerHostResource
import com.tencent.devops.dispatch.docker.pojo.DockerIpInfoVO
import com.tencent.devops.dispatch.docker.pojo.resource.DockerResourceOptionsVO
import com.tencent.devops.dispatch.docker.service.DispatchDockerService
import com.tencent.devops.dispatch.docker.service.DispatchDockerHostBuildService
import com.tencent.devops.dispatch.docker.service.DockerHostBuildService
import com.tencent.devops.dispatch.docker.service.DockerResourceOptionsService
import com.tencent.devops.store.pojo.image.response.ImageRepoInfo
import org.springframework.beans.factory.annotation.Autowired

@RestResource
class DispatchBuildDockerHostResourceImpl @Autowired constructor(
private val dispatchDockerHostBuildService: DispatchDockerHostBuildService,
class BuildDockerHostResourceImpl @Autowired constructor(
private val dockerHostBuildService: DockerHostBuildService,
private val dispatchDockerService: DispatchDockerService,
private val dockerResourceOptionsService: DockerResourceOptionsService
) : DispatchBuildDockerHostResource {
) : BuildDockerHostResource {

override fun getResourceConfig(pipelineId: String, vmSeqId: String): Result<DockerResourceOptionsVO> {
return Result(dockerResourceOptionsService.getDockerResourceConfig(pipelineId, vmSeqId))
Expand All @@ -55,11 +55,11 @@ class DispatchBuildDockerHostResourceImpl @Autowired constructor(
vmSeqId: String,
poolNo: Int
): Result<List<String>> {
return Result(dispatchDockerHostBuildService.getQpcGitProjectList(projectId, buildId, vmSeqId, poolNo))
return Result(dockerHostBuildService.getQpcGitProjectList(projectId, buildId, vmSeqId, poolNo))
}

override fun log(buildId: String, red: Boolean, message: String, tag: String?, jobId: String?): Result<Boolean>? {
dispatchDockerHostBuildService.log(buildId, red, message, tag, jobId)
dockerHostBuildService.log(buildId, red, message, tag, jobId)
return Result(0, "success")
}

Expand All @@ -70,12 +70,12 @@ class DispatchBuildDockerHostResourceImpl @Autowired constructor(
tag: String?,
jobId: String?
): Result<Boolean>? {
dispatchDockerHostBuildService.log(buildId, red, message, tag, jobId)
dockerHostBuildService.log(buildId, red, message, tag, jobId)
return Result(0, "success")
}

override fun getPublicImages(): Result<List<ImageRepoInfo>> {
return dispatchDockerHostBuildService.getPublicImage()
return dockerHostBuildService.getPublicImage()
}

override fun refresh(dockerIp: String, dockerIpInfoVO: DockerIpInfoVO): Result<Boolean> {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,19 +30,19 @@ package com.tencent.devops.dispatch.docker.controller
import com.tencent.devops.common.api.pojo.Result
import com.tencent.devops.common.web.RestResource
import com.tencent.devops.dispatch.docker.api.op.OpDockerBuildResource
import com.tencent.devops.dispatch.docker.service.DispatchDockerHostBuildService
import com.tencent.devops.dispatch.docker.service.DockerHostBuildService
import com.tencent.devops.dispatch.docker.service.DockerHostQpcService
import org.springframework.beans.factory.annotation.Autowired

@RestResource
class OpDockerBuildResourceImpl @Autowired constructor(
private val dispatchDockerHostBuildService: DispatchDockerHostBuildService,
private val dockerHostBuildService: DockerHostBuildService,
private val dockerHostQpcService: DockerHostQpcService
) :
OpDockerBuildResource {

override fun enable(pipelineId: String, vmSeqId: Int?, enable: Boolean): Result<Boolean> {
dispatchDockerHostBuildService.enable(pipelineId, vmSeqId, enable)
dockerHostBuildService.enable(pipelineId, vmSeqId, enable)
return Result(true)
}

Expand Down
Loading

0 comments on commit 07f8c40

Please sign in to comment.