feat: 整合dispatch-docker, dispatch-kubernetes模块到dispatch #9548

helm-charts/core/ci/base/default_env.yaml

@@ -31,10 +31,12 @@ bkCiClusterTag: devops
 bkCiRepositoryGithubServer: repository
 bkCiDockerRoutingType: KUBERNETES
 bkCiDockerJobQuotaEnable: false
+bkCiBcsEnable: false
 bkCiBcsCpu: 8.0
 bkCiBcsMemory: 16048
 bkCiKubernetesCpu: 8
 bkCiKubernetesMemory: 16048
+bkCiKubernetesEnable: true
 bkCiKubernetesHost: http://kubernetes-manager
 bkCiKubernetesToken: landun
 bkCiDevopsToken: devops

scripts/bkenv.properties

@@ -197,6 +197,8 @@ BK_CI_REDIS_SSL=false
 BK_CI_REDIS_SENTINEL_ADDR=
 # BK_CI_REDIS_SENTINEL_MASTER_NAME无默认值. 无需修改.
 BK_CI_REDIS_SENTINEL_MASTER_NAME=
+# BK_CI_BCS_ENABLE无默认值，是否开启蓝鲸bcs资源调度，按需修改
+BK_CI_BCS_ENABLE=false
 # BK_CI_BCS_HOST无默认值，BCS API地址，按需修改
 BK_CI_BCS_HOST=
 # BK_CI_BCS_TOKEN无默认值，BCS API token，按需修改
@@ -205,6 +207,8 @@ BK_CI_BCS_TOKEN=
 BK_CI_BCS_CPU=16.0
 # BK_CI_BCS_MEMORY默认为32769，BCS构建机默认容器内存配额，按需修改
 BK_CI_BCS_MEMORY=32769
+# BK_CI_KUBERNETES_ENABLE无默认值，是否开启kubernetes资源调度，按需修改
+BK_CI_KUBERNETES_ENABLE=true
 # BK_CI_KUBERNETES_HOST无默认值，KUBERNETES API地址，按需修改
 BK_CI_KUBERNETES_HOST=
 # BK_CI_KUBERNETES_TOKEN无默认值，KUBERNETES API token，按需修改

src/backend/ci/boot-assembly/build.gradle.kts

@@ -34,7 +34,6 @@ dependencies {
     implementation(project(":core:artifactory:biz-artifactory"))
     implementation(project(":core:auth:biz-auth-blueking"))
     implementation(project(":core:dispatch:biz-dispatch"))
-    implementation(project(":core:dispatch-docker:biz-dispatch-docker-sample"))
     implementation(project(":core:environment:biz-environment"))
     implementation(project(":core:image:biz-image"))
     implementation(project(":core:log:biz-log-sample"))

src/backend/ci/buildSrc/src/main/kotlin/plugins/task-gen-jooq.gradle.kts

@@ -51,6 +51,10 @@ var moduleNames = when (val moduleName = name.split("-")[1]) {
         listOf("process", "project", "lambda", "store")
     }
 
+    "dispatch" -> {
+        listOf("dispatch", "dispatch_kubernetes")
+    }
+
     else -> listOf(moduleName)
 }
 

src/backend/ci/core/buildless/api-buildless/build.gradle.kts

@@ -28,7 +28,7 @@
 dependencies {
     api(project(":core:common:common-api"))
     api(project(":core:common:common-web"))
-    api(project(":core:dispatch-docker:api-dispatch-docker"))
+    api(project(":core:dispatch:api-dispatch-docker"))
 }
 
 plugins {

src/backend/ci/core/common/common-dispatch-sdk/src/main/kotlin/com/tencent/devops/common/dispatch.sdk/config/MQConfiguration.kt

@@ -47,15 +47,10 @@ import org.springframework.amqp.rabbit.listener.adapter.MessageListenerAdapter
 import org.springframework.amqp.support.converter.Jackson2JsonMessageConverter
 import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.beans.factory.annotation.Value
-import org.springframework.boot.autoconfigure.AutoConfigureOrder
 import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean
 import org.springframework.context.annotation.Bean
-import org.springframework.context.annotation.Configuration
-import org.springframework.core.Ordered
 
 @Suppress("ALL")
-@Configuration
-@AutoConfigureOrder(Ordered.LOWEST_PRECEDENCE)
 class MQConfiguration @Autowired constructor() {
 
     @Value("\${dispatch.demoteQueue.concurrency:2}")

src/backend/ci/core/common/common-dispatch-sdk/src/main/kotlin/com/tencent/devops/common/dispatch.sdk/config/SDKConfiguration.kt

@@ -28,23 +28,22 @@
 package com.tencent.devops.common.dispatch.sdk.config
 
 import com.fasterxml.jackson.databind.ObjectMapper
+import com.tencent.devops.common.client.Client
+import com.tencent.devops.common.dispatch.sdk.service.DispatchService
+import com.tencent.devops.common.dispatch.sdk.service.DockerRoutingSdkService
+import com.tencent.devops.common.dispatch.sdk.service.JobQuotaService
+import com.tencent.devops.common.dispatch.sdk.utils.ChannelUtils
 import com.tencent.devops.common.event.dispatcher.pipeline.PipelineEventDispatcher
 import com.tencent.devops.common.event.dispatcher.pipeline.mq.MQEventDispatcher
+import com.tencent.devops.common.log.utils.BuildLogPrinter
 import com.tencent.devops.common.redis.RedisOperation
-import com.tencent.devops.common.dispatch.sdk.service.DispatchService
+import com.tencent.devops.common.service.BkTag
+import com.tencent.devops.common.service.config.CommonConfig
 import org.springframework.amqp.rabbit.core.RabbitTemplate
 import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.beans.factory.annotation.Value
 import org.springframework.context.annotation.Bean
-import org.springframework.context.annotation.Configuration
-import com.tencent.devops.common.client.Client
-import com.tencent.devops.common.dispatch.sdk.service.DockerRoutingSdkService
-import com.tencent.devops.common.log.utils.BuildLogPrinter
-import com.tencent.devops.common.dispatch.sdk.service.JobQuotaService
-import com.tencent.devops.common.dispatch.sdk.utils.ChannelUtils
-import com.tencent.devops.common.service.config.CommonConfig
 
-@Configuration
 class SDKConfiguration {
     @Value("\${gateway.url:#{null}}")
     private val gateway: String? = ""
@@ -87,4 +86,9 @@ class SDKConfiguration {
     fun pipelineEventDispatcher(@Autowired rabbitTemplate: RabbitTemplate): PipelineEventDispatcher {
         return MQEventDispatcher(rabbitTemplate)
     }
+
+    @Bean
+    fun channelUtils(
+        @Autowired bkTag: BkTag
+    ) = ChannelUtils(bkTag)
 }

src/backend/ci/core/common/common-dispatch-sdk/src/main/kotlin/com/tencent/devops/common/dispatch.sdk/utils/ChannelUtils.kt

@@ -29,11 +29,8 @@ package com.tencent.devops.common.dispatch.sdk.utils
 
 import com.tencent.devops.common.pipeline.enums.ChannelCode
 import com.tencent.devops.common.service.BkTag
-import org.slf4j.LoggerFactory
 import org.springframework.beans.factory.annotation.Autowired
-import org.springframework.stereotype.Component
 
-@Component
 @Suppress("ALL")
 class ChannelUtils @Autowired constructor(
     private val bkTag: BkTag
@@ -48,8 +45,4 @@ class ChannelUtils @Autowired constructor(
             ChannelCode.BS
         }
     }
-
-    companion object {
-        private val logger = LoggerFactory.getLogger(ChannelUtils::class.java)
-    }
 }

src/backend/ci/core/common/common-dispatch-sdk/src/main/resources/META-INF/spring.factories

@@ -0,0 +1,3 @@
+org.springframework.boot.autoconfigure.EnableAutoConfiguration=\
+com.tencent.devops.common.dispatch.sdk.config.SDKConfiguration,\
+com.tencent.devops.common.dispatch.sdk.config.MQConfiguration

src/backend/ci/core/dispatch/api-dispatch-docker/build.gradle.kts

@@ -0,0 +1,36 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2019 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+dependencies {
+    api(project(":core:common:common-api"))
+    api(project(":core:common:common-web"))
+    api(project(":core:store:api-store-image"))
+}
+
+plugins {
+    `task-deploy-to-maven`
+}

src/backend/ci/core/dispatch/api-dispatch-docker/src/main/kotlin/com/tencent/devops/dispatch/docker/api/builds/BuildDockerHostResource.kt

@@ -0,0 +1,137 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2019 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.devops.dispatch.docker.api.builds
+
+import com.tencent.devops.common.api.pojo.Result
+import com.tencent.devops.dispatch.docker.pojo.DockerIpInfoVO
+import com.tencent.devops.dispatch.docker.pojo.resource.DockerResourceOptionsVO
+import com.tencent.devops.store.pojo.image.response.ImageRepoInfo
+import io.swagger.annotations.Api
+import io.swagger.annotations.ApiOperation
+import io.swagger.annotations.ApiParam
+import javax.ws.rs.Consumes
+import javax.ws.rs.GET
+import javax.ws.rs.POST
+import javax.ws.rs.Path
+import javax.ws.rs.PathParam
+import javax.ws.rs.Produces
+import javax.ws.rs.QueryParam
+import javax.ws.rs.core.MediaType
+
+@Api(tags = ["BUILD_DOCKER_HOST"], description = "构建-构建执行DOCKER_HOST资源")
+@Path("/dockerhost")
+@Produces(MediaType.APPLICATION_JSON)
+@Consumes(MediaType.APPLICATION_JSON)@Suppress("ALL")
+interface BuildDockerHostResource {
+    @GET
+    @Path("/resource-config/pipelines/{pipelineId}/vmSeqs/{vmSeqId}")
+    @ApiOperation("获取蓝盾项目的docker性能配置")
+    fun getResourceConfig(
+        @ApiParam("蓝盾流水线ID", required = true)
+        @PathParam("pipelineId")
+        pipelineId: String,
+        @ApiParam("JOB ID", required = true)
+        @PathParam("vmSeqId")
+        vmSeqId: String
+    ): Result<DockerResourceOptionsVO>
+
+    @GET
+    @Path("/qpc/projects/{projectId}/builds/{buildId}/vmSeqs/{vmSeqId}")
+    @ApiOperation("获取蓝盾项目的docker性能配置")
+    fun getQpcGitProjectList(
+        @ApiParam("蓝盾项目ID", required = true)
+        @PathParam("projectId")
+        projectId: String,
+        @ApiParam("构建ID", required = true)
+        @PathParam("buildId")
+        buildId: String,
+        @ApiParam("JOB ID", required = true)
+        @PathParam("vmSeqId")
+        vmSeqId: String,
+        @ApiParam("POOLNo", required = true)
+        @QueryParam("poolNo")
+        poolNo: Int
+    ): Result<List<String>>
+
+    @ApiOperation("上报日志信息")
+    @POST
+    @Path("/log")
+    fun log(
+        @ApiParam("buildId", required = true)
+        @QueryParam("buildId")
+        buildId: String,
+        @ApiParam("red", required = true)
+        @QueryParam("red")
+        red: Boolean,
+        @ApiParam("message", required = true)
+        @QueryParam("message")
+        message: String,
+        @ApiParam("tag", required = false)
+        @QueryParam("tag")
+        tag: String?,
+        @ApiParam("jobId", required = false)
+        @QueryParam("jobId")
+        jobId: String?
+    ): Result<Boolean>?
+
+    @ApiOperation("上报日志信息")
+    @POST
+    @Path("/postlog")
+    fun postLog(
+        @ApiParam("buildId", required = true)
+        @QueryParam("buildId")
+        buildId: String,
+        @ApiParam("red", required = false)
+        @QueryParam("red")
+        red: Boolean,
+        @ApiParam("message", required = true)
+        message: String,
+        @ApiParam("tag", required = false)
+        @QueryParam("tag")
+        tag: String?,
+        @ApiParam("jobId", required = false)
+        @QueryParam("jobId")
+        jobId: String?
+    ): Result<Boolean>?
+
+    @ApiOperation("获取公共镜像")
+    @GET
+    @Path("/public/images")
+    fun getPublicImages(): Result<List<ImageRepoInfo>>
+
+    @POST
+    @Path("/dockerIp/{dockerIp}/refresh")
+    @ApiOperation("刷新Docker构建机状态")
+    fun refresh(
+        @ApiParam("构建机信息", required = true)
+        @PathParam("dockerIp")
+        dockerIp: String,
+        @ApiParam("构建机信息", required = true)
+        dockerIpInfoVO: DockerIpInfoVO
+    ): Result<Boolean>
+}

src/backend/ci/core/dispatch/api-dispatch-docker/src/main/kotlin/com/tencent/devops/dispatch/docker/api/builds/BuildDockerResource.kt

@@ -0,0 +1,61 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2019 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.devops.dispatch.docker.api.builds
+
+import io.swagger.annotations.Api
+import io.swagger.annotations.ApiOperation
+import io.swagger.annotations.ApiParam
+import io.swagger.annotations.ApiResponse
+import io.swagger.annotations.ApiResponses
+import javax.ws.rs.Consumes
+import javax.ws.rs.GET
+import javax.ws.rs.Path
+import javax.ws.rs.Produces
+import javax.ws.rs.QueryParam
+import javax.ws.rs.core.MediaType
+import javax.ws.rs.core.Response
+
+@Api(tags = ["BUILD_DOCKER"], description = "构建-构建执行DOCKER资源")
+@Path("/build/dockers")
+@Produces(MediaType.APPLICATION_JSON)
+@Consumes(MediaType.APPLICATION_JSON)
+interface BuildDockerResource {
+
+    @ApiOperation("下载构建执行器")
+    @GET
+    @Path("/")
+    @Produces(MediaType.APPLICATION_OCTET_STREAM)
+    @ApiResponses(
+            ApiResponse(code = 304, message = "本地的构建执行器已是最新，无需下载")
+    )
+    fun download(
+        @ApiParam("本地eTag标签", required = false)
+        @QueryParam("eTag")
+        eTag: String?
+    ): Response
+}