feat: openapi新增度量能力 #9638

src/backend/ci/core/common/common-es/build.gradle.kts

@@ -0,0 +1,33 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2019 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+dependencies {
+    api("org.elasticsearch:elasticsearch")
+    api("org.elasticsearch.client:elasticsearch-rest-client")
+    api("org.elasticsearch.client:elasticsearch-rest-high-level-client")
+    api(project(":core:common:common-web"))
+}

src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/es/ESAutoConfiguration.kt → src/backend/ci/core/common/common-es/src/main/kotlin/com/tencent/devops/common/es/ESAutoConfiguration.kt

@@ -25,21 +25,15 @@
  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 
-package com.tencent.devops.log.es
+package com.tencent.devops.common.es
 
-import com.tencent.devops.common.redis.RedisOperation
+import com.tencent.devops.common.es.client.LogClient
+import com.tencent.devops.common.es.client.impl.LogClientImpl
 import com.tencent.devops.common.web.WebAutoConfiguration
-import com.tencent.devops.log.client.LogClient
-import com.tencent.devops.log.client.impl.LogClientImpl
-import com.tencent.devops.log.jmx.CreateIndexBean
-import com.tencent.devops.log.jmx.LogStorageBean
-import com.tencent.devops.log.service.BuildLogPrintService
-import com.tencent.devops.log.service.IndexService
-import com.tencent.devops.log.service.LogService
-import com.tencent.devops.log.service.LogStatusService
-import com.tencent.devops.log.service.LogTagService
-import com.tencent.devops.log.service.impl.LogServiceESImpl
-import com.tencent.devops.log.util.ESConfigUtils
+import java.io.File
+import java.io.FileInputStream
+import java.security.KeyStore
+import javax.net.ssl.SSLContext
 import org.apache.http.auth.AuthScope
 import org.apache.http.auth.UsernamePasswordCredentials
 import org.apache.http.impl.client.BasicCredentialsProvider
@@ -57,10 +51,6 @@ import org.springframework.context.annotation.Bean
 import org.springframework.context.annotation.Configuration
 import org.springframework.context.annotation.Primary
 import org.springframework.core.Ordered
-import java.io.File
-import java.io.FileInputStream
-import java.security.KeyStore
-import javax.net.ssl.SSLContext
 
 @Suppress("ALL")
 @Configuration
@@ -71,6 +61,7 @@ import javax.net.ssl.SSLContext
 class ESAutoConfiguration : DisposableBean {
     @Value("\${log.elasticsearch.ip}")
     private val host: String? = null
+
     @Value("\${log.elasticsearch.port}")
     private val port: Int? = null
 
@@ -221,30 +212,6 @@ class ESAutoConfiguration : DisposableBean {
         )
     }
 
-    @Bean
-    fun esLogService(
-        @Autowired logESClient: LogClient,
-        @Autowired indexService: IndexService,
-        @Autowired logStatusService: LogStatusService,
-        @Autowired logTagService: LogTagService,
-        @Autowired defaultKeywords: List<String>,
-        @Autowired createIndexBean: CreateIndexBean,
-        @Autowired logStorageBean: LogStorageBean,
-        @Autowired redisOperation: RedisOperation,
-        @Autowired buildLogPrintService: BuildLogPrintService
-    ): LogService {
-        return LogServiceESImpl(
-            logClient = logESClient,
-            indexService = indexService,
-            logStatusService = logStatusService,
-            logTagService = logTagService,
-            logStorageBean = logStorageBean,
-            createIndexBean = createIndexBean,
-            buildLogPrintService = buildLogPrintService,
-            redisOperation = redisOperation
-        )
-    }
-
     @Bean
     @ConditionalOnMissingBean
     fun logClient(@Autowired transportClient: ESClient): LogClient =
@@ -264,8 +231,8 @@ class ESAutoConfiguration : DisposableBean {
 
     private fun hasCertificateConfig(): Boolean {
         return !keystoreFilePath.isNullOrBlank() ||
-            !truststoreFilePath.isNullOrBlank() ||
-            !keystorePassword.isNullOrBlank() ||
-            !truststorePassword.isNullOrBlank()
+                !truststoreFilePath.isNullOrBlank() ||
+                !keystorePassword.isNullOrBlank() ||
+                !truststorePassword.isNullOrBlank()
     }
 }

src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/es/ESClient.kt → src/backend/ci/core/common/common-es/src/main/kotlin/com/tencent/devops/common/es/ESClient.kt

@@ -25,7 +25,7 @@
  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 
-package com.tencent.devops.log.es
+package com.tencent.devops.common.es
 
 import org.elasticsearch.client.RestHighLevelClient
 

src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/util/ESConfigUtils.kt → src/backend/ci/core/common/common-es/src/main/kotlin/com/tencent/devops/common/es/ESConfigUtils.kt

@@ -25,14 +25,8 @@
  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 
-package com.tencent.devops.log.util
+package com.tencent.devops.common.es
 
-import com.tencent.devops.common.log.constant.LogMessageCode.FILE_NOT_FOUND_CHECK_PATH
-import com.tencent.devops.common.web.utils.I18nUtil
-import com.tencent.devops.log.es.NormalX509ExtendedTrustManager
-import java.io.File
-import java.io.FileInputStream
-import java.security.KeyStore
 import java.security.SecureRandom
 import javax.net.ssl.SSLContext
 import org.apache.http.HeaderElementIterator
@@ -42,12 +36,11 @@ import org.apache.http.client.CredentialsProvider
 import org.apache.http.message.BasicHeaderElementIterator
 import org.apache.http.protocol.HTTP
 import org.apache.http.protocol.HttpContext
-import org.apache.http.ssl.SSLContexts
 import org.elasticsearch.client.RestClient
 import org.elasticsearch.client.RestClientBuilder
 import org.slf4j.LoggerFactory
 
-@Suppress("LongParameterList", "MagicNumber")
+@Suppress("LongParameterList", "MagicNumber", "ComplexMethod")
 object ESConfigUtils {
 
     fun getClientBuilder(
@@ -110,39 +103,5 @@ object ESConfigUtils {
         }
     }
 
-    fun getSSLContext(
-        keystoreFilePath: String,
-        truststoreFilePath: String,
-        keystorePassword: String,
-        truststorePassword: String
-    ): SSLContext {
-        val keystoreFile = File(keystoreFilePath)
-        if (!keystoreFile.exists()) {
-            throw IllegalArgumentException(
-                I18nUtil.getCodeLanMessage(messageCode = FILE_NOT_FOUND_CHECK_PATH, params = arrayOf("keystore")) +
-                        keystoreFilePath
-            )
-        }
-        val truststoreFile = File(truststoreFilePath)
-        if (!truststoreFile.exists()) {
-            throw IllegalArgumentException(
-                I18nUtil.getCodeLanMessage(
-                    messageCode = FILE_NOT_FOUND_CHECK_PATH,
-                    params = arrayOf("truststore")
-                ) + truststoreFilePath
-            )
-        }
-        val keyStore = KeyStore.getInstance(KeyStore.getDefaultType())
-        val keystorePasswordCharArray = keystorePassword.toCharArray()
-        keyStore.load(FileInputStream(keystoreFile), keystorePasswordCharArray)
-        val truststore = KeyStore.getInstance(KeyStore.getDefaultType())
-        val truststorePasswordCharArray = truststorePassword.toCharArray()
-        truststore.load(FileInputStream(truststoreFile), truststorePasswordCharArray)
-        return SSLContexts.custom()
-            .loadTrustMaterial(truststore, null)
-            .loadKeyMaterial(keyStore, keystorePasswordCharArray)
-            .build()
-    }
-
     private val logger = LoggerFactory.getLogger(ESConfigUtils::class.java)
 }

src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/es/ESProperties.kt → src/backend/ci/core/common/common-es/src/main/kotlin/com/tencent/devops/common/es/ESProperties.kt

@@ -25,7 +25,7 @@
  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 
-package com.tencent.devops.log.es
+package com.tencent.devops.common.es
 
 import org.springframework.boot.context.properties.ConfigurationProperties
 import org.springframework.boot.context.properties.ConstructorBinding

src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/es/NormalX509ExtendedTrustManager.kt → src/backend/ci/core/common/common-es/src/main/kotlin/com/tencent/devops/common/es/NormalX509ExtendedTrustManager.kt

@@ -25,7 +25,7 @@
  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 
-package com.tencent.devops.log.es
+package com.tencent.devops.common.es
 
 import java.net.Socket
 import javax.net.ssl.SSLEngine

src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/client/LogClient.kt → src/backend/ci/core/common/common-es/src/main/kotlin/com/tencent/devops/common/es/client/LogClient.kt

@@ -25,9 +25,9 @@
  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 
-package com.tencent.devops.log.client
+package com.tencent.devops.common.es.client
 
-import com.tencent.devops.log.es.ESClient
+import com.tencent.devops.common.es.ESClient
 
 interface LogClient {
 

src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/client/impl/LogClientImpl.kt → src/backend/ci/core/common/common-es/src/main/kotlin/com/tencent/devops/common/es/client/impl/LogClientImpl.kt

@@ -25,10 +25,10 @@
  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 
-package com.tencent.devops.log.client.impl
+package com.tencent.devops.common.es.client.impl
 
-import com.tencent.devops.log.client.LogClient
-import com.tencent.devops.log.es.ESClient
+import com.tencent.devops.common.es.client.LogClient
+import com.tencent.devops.common.es.ESClient
 
 class LogClientImpl constructor(private val client: ESClient) : LogClient {
 

src/backend/ci/core/common/common-es/src/main/resources/META-INF/spring.factories

@@ -0,0 +1,2 @@
+org.springframework.boot.autoconfigure.EnableAutoConfiguration=\
+com.tencent.devops.common.es.ESAutoConfiguration

src/backend/ci/core/common/common-stream/src/main/kotlin/com/tencent/devops/common/stream/constants/StreamBinding.kt

@@ -39,4 +39,8 @@ object StreamBinding {
     // 日志构建状态事件
     const val BINDING_LOG_STATUS_EVENT_IN = "logStatusEventIn"
     const val BINDING_LOG_STATUS_EVENT_OUT = "logStatusEventOut"
+
+    // openapi审计日志预处理事件
+    const val BINDING_OPENAPI_LOG_EVENT_IN = "openapiLogOriginEventIn"
+    const val BINDING_OPENAPI_LOG_EVENT_OUT = "openapiLogOriginEventOut"
 }

src/backend/ci/core/log/biz-log/build.gradle.kts

@@ -33,11 +33,9 @@ dependencies {
     api(project(":core:common:common-auth:common-auth-api"))
     api(project(":core:common:common-db"))
     api(project(":core:common:common-stream"))
+    api(project(":core:common:common-es"))
     api(project(":core:auth:api-auth"))
 
-    api("org.elasticsearch:elasticsearch")
-    api("org.elasticsearch.client:elasticsearch-rest-client")
-    api("org.elasticsearch.client:elasticsearch-rest-high-level-client")
     api("org.apache.logging.log4j:log4j-core")
     api("org.apache.logging.log4j:log4j-api")
     api("com.github.ben-manes.caffeine:caffeine")

src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/cron/impl/IndexCleanJobESImpl.kt

@@ -27,12 +27,14 @@
 
 package com.tencent.devops.log.cron.impl
 
+import com.tencent.devops.common.es.client.LogClient
 import com.tencent.devops.common.redis.RedisLock
 import com.tencent.devops.common.redis.RedisOperation
-import com.tencent.devops.log.client.LogClient
 import com.tencent.devops.log.configuration.StorageProperties
 import com.tencent.devops.log.cron.IndexCleanJob
 import com.tencent.devops.log.util.IndexNameUtils.LOG_INDEX_PREFIX
+import java.time.LocalDateTime
+import java.time.temporal.ChronoUnit
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest
 import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest
 import org.elasticsearch.client.RequestOptions
@@ -44,8 +46,6 @@ import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty
 import org.springframework.scheduling.annotation.Scheduled
 import org.springframework.stereotype.Component
-import java.time.LocalDateTime
-import java.time.temporal.ChronoUnit
 
 @Suppress("MagicNumber")
 @Component

src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/es/LogESAutoConfiguration.kt

@@ -0,0 +1,81 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2019 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.devops.log.es
+
+import com.tencent.devops.common.es.ESAutoConfiguration
+import com.tencent.devops.common.es.ESProperties
+import com.tencent.devops.common.es.client.LogClient
+import com.tencent.devops.common.redis.RedisOperation
+import com.tencent.devops.log.jmx.CreateIndexBean
+import com.tencent.devops.log.jmx.LogStorageBean
+import com.tencent.devops.log.service.BuildLogPrintService
+import com.tencent.devops.log.service.IndexService
+import com.tencent.devops.log.service.LogService
+import com.tencent.devops.log.service.LogStatusService
+import com.tencent.devops.log.service.LogTagService
+import com.tencent.devops.log.service.impl.LogServiceESImpl
+import org.springframework.beans.factory.annotation.Autowired
+import org.springframework.boot.autoconfigure.AutoConfigureAfter
+import org.springframework.boot.autoconfigure.AutoConfigureOrder
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty
+import org.springframework.boot.context.properties.EnableConfigurationProperties
+import org.springframework.context.annotation.Bean
+import org.springframework.context.annotation.Configuration
+import org.springframework.core.Ordered
+
+@Suppress("ALL")
+@Configuration
+@ConditionalOnProperty(prefix = "log.storage", name = ["type"], havingValue = "elasticsearch")
+@AutoConfigureOrder(Ordered.HIGHEST_PRECEDENCE)
+@AutoConfigureAfter(ESAutoConfiguration::class)
+@EnableConfigurationProperties(ESProperties::class)
+class LogESAutoConfiguration {
+    @Bean
+    fun esLogService(
+        @Autowired logESClient: LogClient,
+        @Autowired indexService: IndexService,
+        @Autowired logStatusService: LogStatusService,
+        @Autowired logTagService: LogTagService,
+        @Autowired defaultKeywords: List<String>,
+        @Autowired createIndexBean: CreateIndexBean,
+        @Autowired logStorageBean: LogStorageBean,
+        @Autowired redisOperation: RedisOperation,
+        @Autowired buildLogPrintService: BuildLogPrintService
+    ): LogService {
+        return LogServiceESImpl(
+            logClient = logESClient,
+            indexService = indexService,
+            logStatusService = logStatusService,
+            logTagService = logTagService,
+            logStorageBean = logStorageBean,
+            createIndexBean = createIndexBean,
+            buildLogPrintService = buildLogPrintService,
+            redisOperation = redisOperation
+        )
+    }
+}