Merge pull request #9800 from carlyin0801/issue_9397_pipeline_archive

feat:流水线归档 #9397

src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/constant/CommonConstants.kt

@@ -122,6 +122,7 @@ const val API_PERMISSION = "BK_CI_API_PERMISSION" // 请求API权限
 const val REQUEST_IP = "X-Forwarded-For" // 请求IP
 const val BK_CREATE = "bkCreate" // 创建
 const val BK_REVISE = "bkRevise" // 修改
+const val FAIL_MSG = "failMsg" // 失败信息
 
 const val KEY_START_TIME = "startTime"
 const val KEY_END_TIME = "endTime"
@@ -154,7 +155,9 @@ const val KEY_VERSION_NAME = "versionName"
 const val KEY_UPDATED_TIME = "updatedTime"
 const val KEY_DEFAULT_LOCALE_LANGUAGE = "defaultLocaleLanguage"
 const val KEY_PROJECT_ID = "projectId"
+const val KEY_PIPELINE_ID = "pipelineId"
 const val KEY_PIPELINE_NUM = "pipelineNum"
+const val KEY_ARCHIVE = "archive"
 const val KEY_BRANCH_TEST_FLAG = "branchTestFlag"
 
 const val BK_BUILD_ENV_START_FAILED = "bkBuildEnvStartFailed" // 构建环境启动失败

src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/constant/CommonMessageCode.kt

@@ -172,6 +172,7 @@ object CommonMessageCode {
     const val ERROR_YAML_FORMAT_EXCEPTION_ENV_VARIABLE_LENGTH_LIMIT_EXCEEDED = "2100121"
     const val ERROR_PROJECT_API_ACCESS_NO_PERMISSION = "2100122" // 项目[{0}]没有接口[{1}]的访问权限
     const val ERROR_INTERFACE_RETRY_NUM_EXCEEDED = "2100123" // 接口连续重试次数超过{0}次，请稍后再试
+    const val ERROR_PIPELINE_API_ACCESS_NO_PERMISSION = "2100124" // 流水线[{0}]没有接口[{1}]的访问权限
 
     const val BK_CONTAINER_TIMED_OUT = "bkContainerTimedOut" // 创建容器超时
     const val BK_CREATION_FAILED_EXCEPTION_INFORMATION = "bkCreationFailedExceptionInformation" // 创建失败，异常信息

src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/pojo/ShardingRuleTypeEnum.kt

@@ -29,5 +29,7 @@ package com.tencent.devops.common.api.pojo
 
 enum class ShardingRuleTypeEnum {
     DB,
-    TABLE
+    TABLE,
+    ARCHIVE_DB,
+    ARCHIVE_TABLE
 }

src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/util/DateTimeUtil.kt

@@ -70,6 +70,8 @@ object DateTimeUtil {
 
     const val YYYYMMDD = "yyyyMMdd"
 
+    const val YYYYMMDDHHMMSS = "yyyyMMddHHmmss"
+
     const val ONE_THOUSAND_MS = 1000L
 
     /**

src/backend/ci/core/common/common-auth/common-auth-api/src/main/kotlin/com/tencent/devops/common/auth/api/AuthPermission.kt

@@ -46,6 +46,8 @@ enum class AuthPermission(val value: String, val alias: String) {
     MKDIR("mkdir", "创建目录"), // 自定义目录，容器
     EXPERIENCE("experience", "转体验"), // 版本体验
     ENABLE("enable", "停用/启用"), // 质量红线
+    ARCHIVE("archive", "归档"), // 归档流水线
+    MANAGE_ARCHIVED_PIPELINE("manage-archived-pipeline", "管理已归档流水线"), // 管理已归档流水线
 
     VIEWS_MANAGER("views_manager", "视图管理"), // 项目视图管理
     WEB_CHECK("webcheck", "页面按钮校验"), // 页面按钮校验

src/backend/ci/core/common/common-db-sharding/src/main/kotlin/com/tencent/devops/common/db/config/BkShardingDataSourceConfiguration.kt

@@ -30,6 +30,8 @@ package com.tencent.devops.common.db.config
 import com.mysql.cj.jdbc.Driver
 import com.tencent.devops.common.api.constant.CommonMessageCode
 import com.tencent.devops.common.api.exception.ErrorCodeException
+import com.tencent.devops.common.api.util.JsonUtil.deepCopy
+import com.tencent.devops.common.db.pojo.ARCHIVE_DATA_SOURCE_NAME_PREFIX
 import com.tencent.devops.common.db.pojo.BindingTableGroupConfig
 import com.tencent.devops.common.db.pojo.DATA_SOURCE_NAME_PREFIX
 import com.tencent.devops.common.db.pojo.DataSourceConfig
@@ -57,6 +59,7 @@ import org.springframework.boot.autoconfigure.jooq.JooqAutoConfiguration
 import org.springframework.boot.context.properties.EnableConfigurationProperties
 import org.springframework.context.annotation.Bean
 import org.springframework.context.annotation.Configuration
+import org.springframework.context.annotation.Primary
 import org.springframework.core.Ordered
 import org.springframework.transaction.annotation.EnableTransactionManagement
 import java.util.Properties
@@ -89,6 +92,9 @@ class BkShardingDataSourceConfiguration {
     @Value("\${sharding.databaseShardingStrategy.migratingAlgorithmClassName:#{null}}")
     private val migratingDatabaseAlgorithmClassName: String? = null
 
+    @Value("\${sharding.databaseShardingStrategy.archiveAlgorithmClassName:#{null}}")
+    private val archiveDatabaseAlgorithmClassName: String? = null
+
     @Value("\${sharding.databaseShardingStrategy.shardingField:#{null}}")
     private val databaseShardingField: String? = null
 
@@ -98,6 +104,9 @@ class BkShardingDataSourceConfiguration {
     @Value("\${sharding.tableShardingStrategy.migratingAlgorithmClassName:#{null}}")
     private val migratingTableAlgorithmClassName: String? = null
 
+    @Value("\${sharding.tableShardingStrategy.archiveAlgorithmClassName:#{null}}")
+    private val archiveTableAlgorithmClassName: String? = null
+
     @Value("\${sharding.tableShardingStrategy.shardingField:#{null}}")
     private val tableShardingField: String? = null
 
@@ -110,6 +119,9 @@ class BkShardingDataSourceConfiguration {
     @Value("\${spring.datasource.idleTimeout:#{60000}}")
     private val datasourceIdleTimeout: Long = 60000
 
+    @Value("\${sharding.tableShardingStrategy.defaultShardingNum:#{5}}")
+    private val defaultTableShardingNum: Int = 5
+
     private fun dataSourceMap(
         dataSourcePrefixName: String,
         dataSourceConfigs: List<DataSourceConfig>,
@@ -157,6 +169,7 @@ class BkShardingDataSourceConfiguration {
     }
 
     @Bean
+    @Primary
     fun shardingDataSource(config: DataSourceProperties, registry: MeterRegistry): DataSource {
         return createShardingDataSource(
             dataSourcePrefixName = DATA_SOURCE_NAME_PREFIX,
@@ -192,6 +205,50 @@ class BkShardingDataSourceConfiguration {
         )
     }
 
+    @Bean
+    @ConditionalOnProperty(prefix = "sharding", name = ["archiveFlag"], havingValue = "Y")
+    fun archiveShardingDataSource(config: DataSourceProperties, registry: MeterRegistry): DataSource {
+        val archiveDataSourceConfigs = config.archiveDataSourceConfigs
+        val archiveTableRuleConfigs = config.archiveTableRuleConfigs
+        if (archiveDataSourceConfigs == null && archiveTableRuleConfigs == null) {
+            logger.warn("archiveDataSourceConfigs and archiveTableRuleConfigs cannot be empty at the same time")
+            throw ErrorCodeException(
+                errorCode = CommonMessageCode.SYSTEM_ERROR,
+                defaultMessage = "archiveDataSourceConfigs and archiveTableRuleConfigs cannot be empty at the same time"
+            )
+        }
+        return createShardingDataSource(
+            dataSourcePrefixName = ARCHIVE_DATA_SOURCE_NAME_PREFIX,
+            databaseAlgorithmClassName = archiveDatabaseAlgorithmClassName,
+            tableAlgorithmClassName = archiveTableAlgorithmClassName,
+            dataSourceConfigs = archiveDataSourceConfigs ?: config.dataSourceConfigs,
+            tableRuleConfigs = generateTableRuleConfigs(archiveTableRuleConfigs, config),
+            bindingTableGroupConfigs = config.archiveBindingTableGroupConfigs ?: config.bindingTableGroupConfigs,
+            registry = registry
+        )
+    }
+
+    private fun generateTableRuleConfigs(
+        tableRuleConfigs: List<TableRuleConfig>?,
+        config: DataSourceProperties
+    ): List<TableRuleConfig>? {
+        return if (tableRuleConfigs.isNullOrEmpty() && defaultTableShardingNum > 1) {
+            // 如果分表规则为空，则复用默认的分表规则
+            val finalTableRuleConfigs = mutableListOf<TableRuleConfig>()
+            config.tableRuleConfigs.forEach { tableRuleConfig ->
+                val finalTableRuleConfig = tableRuleConfig.deepCopy<TableRuleConfig>()
+                if (finalTableRuleConfig.broadcastFlag != true) {
+                    finalTableRuleConfig.tableShardingStrategy = TableShardingStrategyEnum.SHARDING
+                    finalTableRuleConfig.shardingNum = defaultTableShardingNum
+                }
+                finalTableRuleConfigs.add(finalTableRuleConfig)
+            }
+            finalTableRuleConfigs
+        } else {
+            tableRuleConfigs
+        }
+    }
+
     fun createShardingDataSource(
         dataSourcePrefixName: String,
         databaseAlgorithmClassName: String? = null,
@@ -241,7 +298,7 @@ class BkShardingDataSourceConfiguration {
                 AlgorithmConfiguration(CLASS_BASED, dbShardingAlgorithmProps)
         }
         // 生成table分片算法配置
-        if (!tableAlgorithmClassName.isNullOrBlank()) {
+        if (!tableAlgorithmClassName.isNullOrBlank() && !tableRuleConfigs.isNullOrEmpty()) {
             val tableShardingAlgorithmProps = Properties()
             tableShardingAlgorithmProps.setProperty(STRATEGY, STANDARD)
             tableShardingAlgorithmProps.setProperty(ALGORITHM_CLASS_NAME, tableAlgorithmClassName)
@@ -268,40 +325,47 @@ class BkShardingDataSourceConfiguration {
     fun getTableRuleConfiguration(
         dataSourcePrefixName: String,
         dataSourceSize: Int,
-        tableRuleConfig: TableRuleConfig
+        tableRuleConfig: TableRuleConfig,
+        logicTableSuffixName: String? = null
     ): ShardingTableRuleConfiguration? {
         // 生成实际节点规则
         val tableName = tableRuleConfig.name
         val databaseShardingStrategy = tableRuleConfig.databaseShardingStrategy
         val tableShardingStrategy = tableRuleConfig.tableShardingStrategy
         val lastDsIndex = dataSourceSize - 1
         val lastTableIndex = tableRuleConfig.shardingNum - 1
+        // 生成逻辑表名称
+        val logicTableName = if (logicTableSuffixName.isNullOrBlank()) {
+            tableName
+        } else {
+            "${tableName}_$logicTableSuffixName"
+        }
         val actualDataNodes = if (databaseShardingStrategy != null &&
             tableShardingStrategy == TableShardingStrategyEnum.SHARDING
         ) {
             // 生成分库分表场景下的节点规则
             if (databaseShardingStrategy == DatabaseShardingStrategyEnum.SPECIFY) {
-                "${dataSourcePrefixName}0.${tableName}_\${0..$lastTableIndex}"
+                "${dataSourcePrefixName}0.${logicTableName}_\${0..$lastTableIndex}"
             } else {
-                "$dataSourcePrefixName\${0..$lastDsIndex}.${tableName}_\${0..$lastTableIndex}"
+                "$dataSourcePrefixName\${0..$lastDsIndex}.${logicTableName}_\${0..$lastTableIndex}"
             }
         } else if (databaseShardingStrategy != null && tableShardingStrategy != TableShardingStrategyEnum.SHARDING) {
             // 生成分库场景下的节点规则
             if (databaseShardingStrategy == DatabaseShardingStrategyEnum.SPECIFY) {
-                "${dataSourcePrefixName}0.$tableName"
+                "${dataSourcePrefixName}0.$logicTableName"
             } else {
-                "$dataSourcePrefixName\${0..$lastDsIndex}.$tableName"
+                "$dataSourcePrefixName\${0..$lastDsIndex}.$logicTableName"
             }
         } else if (databaseShardingStrategy == null && tableShardingStrategy == TableShardingStrategyEnum.SHARDING) {
             // 生成分表场景下的节点规则
-            "${dataSourcePrefixName}0.${tableName}_\${0..$lastTableIndex}"
+            "${dataSourcePrefixName}0.${logicTableName}_\${0..$lastTableIndex}"
         } else {
-            "${dataSourcePrefixName}0.$tableName"
+            "${dataSourcePrefixName}0.$logicTableName"
         }
         val shardingTableRuleConfig = ShardingTableRuleConfiguration(tableName, actualDataNodes)
         logger.info(
             "BkShardingDataSourceConfiguration table:$tableName|databaseShardingStrategy: $databaseShardingStrategy|" +
-                    "tableShardingStrategy:$tableShardingStrategy|actualDataNodes:$actualDataNodes "
+                "tableShardingStrategy:$tableShardingStrategy|actualDataNodes:$actualDataNodes "
         )
         // 设置表的分库策略
         shardingTableRuleConfig.databaseShardingStrategy = if (databaseShardingStrategy != null) {

src/backend/ci/core/common/common-db-sharding/src/main/kotlin/com/tencent/devops/common/db/config/BkShardingJooqConfiguration.kt

@@ -27,6 +27,7 @@
 
 package com.tencent.devops.common.db.config
 
+import com.tencent.devops.common.db.pojo.ARCHIVE_SHARDING_DSL_CONTEXT
 import com.tencent.devops.common.db.pojo.MIGRATING_SHARDING_DSL_CONTEXT
 import org.jooq.DSLContext
 import org.jooq.ExecuteListenerProvider
@@ -42,6 +43,7 @@ import org.springframework.boot.autoconfigure.AutoConfigureAfter
 import org.springframework.context.annotation.Bean
 import org.springframework.context.annotation.Configuration
 import org.springframework.context.annotation.Import
+import org.springframework.context.annotation.Primary
 import javax.sql.DataSource
 
 @Configuration
@@ -50,6 +52,8 @@ import javax.sql.DataSource
 class BkShardingJooqConfiguration {
 
     @Bean
+    @Primary
+    @ConditionalOnProperty(prefix = "sharding", name = ["defaultFlag"], havingValue = "Y", matchIfMissing = true)
     fun shardingDslContext(
         @Qualifier("shardingDataSource")
         shardingDataSource: DataSource,
@@ -68,6 +72,16 @@ class BkShardingJooqConfiguration {
         return createDslContext(migratingShardingDataSource, executeListenerProviders)
     }
 
+    @Bean(name = [ARCHIVE_SHARDING_DSL_CONTEXT])
+    @ConditionalOnProperty(prefix = "sharding", name = ["archiveFlag"], havingValue = "Y")
+    fun archiveShardingDslContext(
+        @Qualifier("archiveShardingDataSource")
+        archiveShardingDataSource: DataSource,
+        executeListenerProviders: ObjectProvider<ExecuteListenerProvider>
+    ): DSLContext {
+        return createDslContext(archiveShardingDataSource, executeListenerProviders)
+    }
+
     private fun createDslContext(
         shardingDataSource: DataSource,
         executeListenerProviders: ObjectProvider<ExecuteListenerProvider>

src/backend/ci/core/common/common-db-sharding/src/main/kotlin/com/tencent/devops/common/db/pojo/DataSourceProperties.kt

@@ -38,5 +38,8 @@ data class DataSourceProperties(
     val bindingTableGroupConfigs: List<BindingTableGroupConfig>? = null, // 绑定表规则配置
     val migratingDataSourceConfigs: List<DataSourceConfig>? = null, // 迁移数据源配置
     val migratingTableRuleConfigs: List<TableRuleConfig>? = null, // 迁移数据库表规则配置
-    val migratingBindingTableGroupConfigs: List<BindingTableGroupConfig>? = null // 迁移绑定表规则配置
+    val migratingBindingTableGroupConfigs: List<BindingTableGroupConfig>? = null, // 迁移绑定表规则配置
+    val archiveDataSourceConfigs: List<DataSourceConfig>? = null, // 归档数据源配置
+    val archiveTableRuleConfigs: List<TableRuleConfig>? = null, // 归档数据库表规则配置
+    val archiveBindingTableGroupConfigs: List<BindingTableGroupConfig>? = null // 归档绑定表规则配置
 )

src/backend/ci/core/common/common-db-sharding/src/main/kotlin/com/tencent/devops/common/db/pojo/SharingDbConstants.kt

@@ -29,6 +29,9 @@ package com.tencent.devops.common.db.pojo
 
 const val DATA_SOURCE_NAME_PREFIX = "ds_"
 const val MIGRATING_DATA_SOURCE_NAME_PREFIX = "mig_ds_"
+const val ARCHIVE_DATA_SOURCE_NAME_PREFIX = "archive_ds_"
 const val DEFAULT_DATA_SOURCE_NAME = "ds_0"
 const val DEFAULT_MIGRATING_DATA_SOURCE_NAME = "mig_ds_0"
+const val DEFAULT_ARCHIVE_DATA_SOURCE_NAME = "archive_ds_0"
 const val MIGRATING_SHARDING_DSL_CONTEXT = "migratingShardingDslContext"
+const val ARCHIVE_SHARDING_DSL_CONTEXT = "archiveShardingDslContext"

src/backend/ci/core/common/common-db-sharding/src/main/kotlin/com/tencent/devops/common/db/pojo/TableRuleConfig.kt

@@ -30,8 +30,8 @@ package com.tencent.devops.common.db.pojo
 data class TableRuleConfig(
     val index: Int, // 序号
     val name: String, // 表名
-    val shardingNum: Int = 1, // 分表数量
+    var shardingNum: Int = 1, // 分表数量
     val broadcastFlag: Boolean? = null, // 是否为广播表
     val databaseShardingStrategy: DatabaseShardingStrategyEnum? = null, // 分库策略
-    val tableShardingStrategy: TableShardingStrategyEnum? = null // 分表策略
+    var tableShardingStrategy: TableShardingStrategyEnum? = null // 分表策略
 )

src/backend/ci/core/common/common-event/src/main/kotlin/com/tencent/devops/common/event/dispatcher/pipeline/mq/MQ.kt

@@ -43,6 +43,9 @@ object MQ {
     const val ROUTE_PIPELINE_RESTORE = "r.engine.pipeline.restore"
     const val QUEUE_PIPELINE_RESTORE = "q.engine.pipeline.restore"
 
+    const val ROUTE_PIPELINE_ARCHIVE = "r.engine.pipeline.archive"
+    const val QUEUE_PIPELINE_ARCHIVE = "q.engine.pipeline.archive"
+
     const val ROUTE_PIPELINE_TIMER = "r.engine.pipeline.timer"
     const val QUEUE_PIPELINE_TIMER = "q.engine.pipeline.timer"
 

src/backend/ci/core/common/common-event/src/main/kotlin/com/tencent/devops/common/event/enums/ActionType.kt

@@ -38,6 +38,7 @@ enum class ActionType {
     END, // 强制结束当前节点，会导致当前构建容器结束
     SKIP, // 跳过-不执行
     TERMINATE, // 终止
+    ARCHIVE, // 归档
     ;
 
     fun isStartOrRefresh() = isStart() || this == REFRESH

src/backend/ci/core/common/common-event/src/main/kotlin/com/tencent/devops/common/event/pojo/pipeline/PipelineArchiveEvent.kt

@@ -0,0 +1,48 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2019 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.devops.common.event.pojo.pipeline
+
+import com.tencent.devops.common.event.annotation.Event
+import com.tencent.devops.common.event.dispatcher.pipeline.mq.MQ
+import com.tencent.devops.common.event.enums.ActionType
+
+/**
+ * 归档流水线事件
+ *
+ * @version 1.0
+ */
+@Event(MQ.ENGINE_PROCESS_LISTENER_EXCHANGE, MQ.ROUTE_PIPELINE_ARCHIVE)
+data class PipelineArchiveEvent(
+    override val source: String,
+    override val projectId: String,
+    override val pipelineId: String,
+    override val userId: String,
+    override var actionType: ActionType = ActionType.ARCHIVE,
+    override var delayMills: Int = 0,
+    val cancelFlag: Boolean = false
+) : IPipelineEvent(actionType, source, projectId, pipelineId, userId, delayMills)