diff --git a/src/backend/ci/core/common/common-es/build.gradle.kts b/src/backend/ci/core/common/common-es/build.gradle.kts new file mode 100644 index 00000000000..79c301da8e3 --- /dev/null +++ b/src/backend/ci/core/common/common-es/build.gradle.kts @@ -0,0 +1,33 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +dependencies { + api("org.elasticsearch:elasticsearch") + api("org.elasticsearch.client:elasticsearch-rest-client") + api("org.elasticsearch.client:elasticsearch-rest-high-level-client") + api(project(":core:common:common-web")) +} diff --git a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/es/ESAutoConfiguration.kt b/src/backend/ci/core/common/common-es/src/main/kotlin/com/tencent/devops/common/es/ESAutoConfiguration.kt similarity index 84% rename from src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/es/ESAutoConfiguration.kt rename to src/backend/ci/core/common/common-es/src/main/kotlin/com/tencent/devops/common/es/ESAutoConfiguration.kt index ae6d4ea3812..4cdc5379301 100644 --- a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/es/ESAutoConfiguration.kt +++ b/src/backend/ci/core/common/common-es/src/main/kotlin/com/tencent/devops/common/es/ESAutoConfiguration.kt @@ -25,21 +25,15 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.devops.log.es +package com.tencent.devops.common.es -import com.tencent.devops.common.redis.RedisOperation +import com.tencent.devops.common.es.client.LogClient +import com.tencent.devops.common.es.client.impl.LogClientImpl import com.tencent.devops.common.web.WebAutoConfiguration -import com.tencent.devops.log.client.LogClient -import com.tencent.devops.log.client.impl.LogClientImpl -import com.tencent.devops.log.jmx.CreateIndexBean -import com.tencent.devops.log.jmx.LogStorageBean -import com.tencent.devops.log.service.BuildLogPrintService -import com.tencent.devops.log.service.IndexService -import com.tencent.devops.log.service.LogService -import com.tencent.devops.log.service.LogStatusService -import com.tencent.devops.log.service.LogTagService -import com.tencent.devops.log.service.impl.LogServiceESImpl -import com.tencent.devops.log.util.ESConfigUtils +import java.io.File +import java.io.FileInputStream +import java.security.KeyStore +import javax.net.ssl.SSLContext import org.apache.http.auth.AuthScope import org.apache.http.auth.UsernamePasswordCredentials import org.apache.http.impl.client.BasicCredentialsProvider @@ -57,10 +51,6 @@ import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration import org.springframework.context.annotation.Primary import org.springframework.core.Ordered -import java.io.File -import java.io.FileInputStream -import java.security.KeyStore -import javax.net.ssl.SSLContext @Suppress("ALL") @Configuration @@ -71,6 +61,7 @@ import javax.net.ssl.SSLContext class ESAutoConfiguration : DisposableBean { @Value("\${log.elasticsearch.ip}") private val host: String? = null + @Value("\${log.elasticsearch.port}") private val port: Int? = null @@ -221,30 +212,6 @@ class ESAutoConfiguration : DisposableBean { ) } - @Bean - fun esLogService( - @Autowired logESClient: LogClient, - @Autowired indexService: IndexService, - @Autowired logStatusService: LogStatusService, - @Autowired logTagService: LogTagService, - @Autowired defaultKeywords: List, - @Autowired createIndexBean: CreateIndexBean, - @Autowired logStorageBean: LogStorageBean, - @Autowired redisOperation: RedisOperation, - @Autowired buildLogPrintService: BuildLogPrintService - ): LogService { - return LogServiceESImpl( - logClient = logESClient, - indexService = indexService, - logStatusService = logStatusService, - logTagService = logTagService, - logStorageBean = logStorageBean, - createIndexBean = createIndexBean, - buildLogPrintService = buildLogPrintService, - redisOperation = redisOperation - ) - } - @Bean @ConditionalOnMissingBean fun logClient(@Autowired transportClient: ESClient): LogClient = @@ -264,8 +231,8 @@ class ESAutoConfiguration : DisposableBean { private fun hasCertificateConfig(): Boolean { return !keystoreFilePath.isNullOrBlank() || - !truststoreFilePath.isNullOrBlank() || - !keystorePassword.isNullOrBlank() || - !truststorePassword.isNullOrBlank() + !truststoreFilePath.isNullOrBlank() || + !keystorePassword.isNullOrBlank() || + !truststorePassword.isNullOrBlank() } } diff --git a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/es/ESClient.kt b/src/backend/ci/core/common/common-es/src/main/kotlin/com/tencent/devops/common/es/ESClient.kt similarity index 97% rename from src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/es/ESClient.kt rename to src/backend/ci/core/common/common-es/src/main/kotlin/com/tencent/devops/common/es/ESClient.kt index 8c9a0e41751..2c37d5fcd72 100644 --- a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/es/ESClient.kt +++ b/src/backend/ci/core/common/common-es/src/main/kotlin/com/tencent/devops/common/es/ESClient.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.devops.log.es +package com.tencent.devops.common.es import org.elasticsearch.client.RestHighLevelClient diff --git a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/util/ESConfigUtils.kt b/src/backend/ci/core/common/common-es/src/main/kotlin/com/tencent/devops/common/es/ESConfigUtils.kt similarity index 71% rename from src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/util/ESConfigUtils.kt rename to src/backend/ci/core/common/common-es/src/main/kotlin/com/tencent/devops/common/es/ESConfigUtils.kt index 73d9ca4a9dc..0b073536cf9 100644 --- a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/util/ESConfigUtils.kt +++ b/src/backend/ci/core/common/common-es/src/main/kotlin/com/tencent/devops/common/es/ESConfigUtils.kt @@ -25,14 +25,8 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.devops.log.util +package com.tencent.devops.common.es -import com.tencent.devops.common.log.constant.LogMessageCode.FILE_NOT_FOUND_CHECK_PATH -import com.tencent.devops.common.web.utils.I18nUtil -import com.tencent.devops.log.es.NormalX509ExtendedTrustManager -import java.io.File -import java.io.FileInputStream -import java.security.KeyStore import java.security.SecureRandom import javax.net.ssl.SSLContext import org.apache.http.HeaderElementIterator @@ -42,12 +36,11 @@ import org.apache.http.client.CredentialsProvider import org.apache.http.message.BasicHeaderElementIterator import org.apache.http.protocol.HTTP import org.apache.http.protocol.HttpContext -import org.apache.http.ssl.SSLContexts import org.elasticsearch.client.RestClient import org.elasticsearch.client.RestClientBuilder import org.slf4j.LoggerFactory -@Suppress("LongParameterList", "MagicNumber") +@Suppress("LongParameterList", "MagicNumber", "ComplexMethod") object ESConfigUtils { fun getClientBuilder( @@ -110,39 +103,5 @@ object ESConfigUtils { } } - fun getSSLContext( - keystoreFilePath: String, - truststoreFilePath: String, - keystorePassword: String, - truststorePassword: String - ): SSLContext { - val keystoreFile = File(keystoreFilePath) - if (!keystoreFile.exists()) { - throw IllegalArgumentException( - I18nUtil.getCodeLanMessage(messageCode = FILE_NOT_FOUND_CHECK_PATH, params = arrayOf("keystore")) + - keystoreFilePath - ) - } - val truststoreFile = File(truststoreFilePath) - if (!truststoreFile.exists()) { - throw IllegalArgumentException( - I18nUtil.getCodeLanMessage( - messageCode = FILE_NOT_FOUND_CHECK_PATH, - params = arrayOf("truststore") - ) + truststoreFilePath - ) - } - val keyStore = KeyStore.getInstance(KeyStore.getDefaultType()) - val keystorePasswordCharArray = keystorePassword.toCharArray() - keyStore.load(FileInputStream(keystoreFile), keystorePasswordCharArray) - val truststore = KeyStore.getInstance(KeyStore.getDefaultType()) - val truststorePasswordCharArray = truststorePassword.toCharArray() - truststore.load(FileInputStream(truststoreFile), truststorePasswordCharArray) - return SSLContexts.custom() - .loadTrustMaterial(truststore, null) - .loadKeyMaterial(keyStore, keystorePasswordCharArray) - .build() - } - private val logger = LoggerFactory.getLogger(ESConfigUtils::class.java) } diff --git a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/es/ESProperties.kt b/src/backend/ci/core/common/common-es/src/main/kotlin/com/tencent/devops/common/es/ESProperties.kt similarity index 98% rename from src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/es/ESProperties.kt rename to src/backend/ci/core/common/common-es/src/main/kotlin/com/tencent/devops/common/es/ESProperties.kt index cbc7039b168..861bc65d103 100644 --- a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/es/ESProperties.kt +++ b/src/backend/ci/core/common/common-es/src/main/kotlin/com/tencent/devops/common/es/ESProperties.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.devops.log.es +package com.tencent.devops.common.es import org.springframework.boot.context.properties.ConfigurationProperties import org.springframework.boot.context.properties.ConstructorBinding diff --git a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/es/NormalX509ExtendedTrustManager.kt b/src/backend/ci/core/common/common-es/src/main/kotlin/com/tencent/devops/common/es/NormalX509ExtendedTrustManager.kt similarity index 98% rename from src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/es/NormalX509ExtendedTrustManager.kt rename to src/backend/ci/core/common/common-es/src/main/kotlin/com/tencent/devops/common/es/NormalX509ExtendedTrustManager.kt index aad2f92312d..0b037ebb84c 100644 --- a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/es/NormalX509ExtendedTrustManager.kt +++ b/src/backend/ci/core/common/common-es/src/main/kotlin/com/tencent/devops/common/es/NormalX509ExtendedTrustManager.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.devops.log.es +package com.tencent.devops.common.es import java.net.Socket import javax.net.ssl.SSLEngine diff --git a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/client/LogClient.kt b/src/backend/ci/core/common/common-es/src/main/kotlin/com/tencent/devops/common/es/client/LogClient.kt similarity index 94% rename from src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/client/LogClient.kt rename to src/backend/ci/core/common/common-es/src/main/kotlin/com/tencent/devops/common/es/client/LogClient.kt index fb1de1be49b..ea6e8a422d5 100644 --- a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/client/LogClient.kt +++ b/src/backend/ci/core/common/common-es/src/main/kotlin/com/tencent/devops/common/es/client/LogClient.kt @@ -25,9 +25,9 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.devops.log.client +package com.tencent.devops.common.es.client -import com.tencent.devops.log.es.ESClient +import com.tencent.devops.common.es.ESClient interface LogClient { diff --git a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/client/impl/LogClientImpl.kt b/src/backend/ci/core/common/common-es/src/main/kotlin/com/tencent/devops/common/es/client/impl/LogClientImpl.kt similarity index 92% rename from src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/client/impl/LogClientImpl.kt rename to src/backend/ci/core/common/common-es/src/main/kotlin/com/tencent/devops/common/es/client/impl/LogClientImpl.kt index c4db85f7816..48ab81310a8 100644 --- a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/client/impl/LogClientImpl.kt +++ b/src/backend/ci/core/common/common-es/src/main/kotlin/com/tencent/devops/common/es/client/impl/LogClientImpl.kt @@ -25,10 +25,10 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.devops.log.client.impl +package com.tencent.devops.common.es.client.impl -import com.tencent.devops.log.client.LogClient -import com.tencent.devops.log.es.ESClient +import com.tencent.devops.common.es.client.LogClient +import com.tencent.devops.common.es.ESClient class LogClientImpl constructor(private val client: ESClient) : LogClient { diff --git a/src/backend/ci/core/common/common-es/src/main/resources/META-INF/spring.factories b/src/backend/ci/core/common/common-es/src/main/resources/META-INF/spring.factories new file mode 100644 index 00000000000..4d64e749f1d --- /dev/null +++ b/src/backend/ci/core/common/common-es/src/main/resources/META-INF/spring.factories @@ -0,0 +1,2 @@ +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ +com.tencent.devops.common.es.ESAutoConfiguration diff --git a/src/backend/ci/core/common/common-stream/src/main/kotlin/com/tencent/devops/common/stream/constants/StreamBinding.kt b/src/backend/ci/core/common/common-stream/src/main/kotlin/com/tencent/devops/common/stream/constants/StreamBinding.kt index ec76d5da014..46d78e55eb6 100644 --- a/src/backend/ci/core/common/common-stream/src/main/kotlin/com/tencent/devops/common/stream/constants/StreamBinding.kt +++ b/src/backend/ci/core/common/common-stream/src/main/kotlin/com/tencent/devops/common/stream/constants/StreamBinding.kt @@ -39,4 +39,8 @@ object StreamBinding { // 日志构建状态事件 const val BINDING_LOG_STATUS_EVENT_IN = "logStatusEventIn" const val BINDING_LOG_STATUS_EVENT_OUT = "logStatusEventOut" + + // openapi审计日志预处理事件 + const val BINDING_OPENAPI_LOG_EVENT_IN = "openapiLogOriginEventIn" + const val BINDING_OPENAPI_LOG_EVENT_OUT = "openapiLogOriginEventOut" } diff --git a/src/backend/ci/core/log/biz-log/build.gradle.kts b/src/backend/ci/core/log/biz-log/build.gradle.kts index e7277c4a259..927b388b331 100644 --- a/src/backend/ci/core/log/biz-log/build.gradle.kts +++ b/src/backend/ci/core/log/biz-log/build.gradle.kts @@ -33,11 +33,9 @@ dependencies { api(project(":core:common:common-auth:common-auth-api")) api(project(":core:common:common-db")) api(project(":core:common:common-stream")) + api(project(":core:common:common-es")) api(project(":core:auth:api-auth")) - api("org.elasticsearch:elasticsearch") - api("org.elasticsearch.client:elasticsearch-rest-client") - api("org.elasticsearch.client:elasticsearch-rest-high-level-client") api("org.apache.logging.log4j:log4j-core") api("org.apache.logging.log4j:log4j-api") api("com.github.ben-manes.caffeine:caffeine") diff --git a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/cron/impl/IndexCleanJobESImpl.kt b/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/cron/impl/IndexCleanJobESImpl.kt index 2507aeb8c39..28dfbb71c8d 100644 --- a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/cron/impl/IndexCleanJobESImpl.kt +++ b/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/cron/impl/IndexCleanJobESImpl.kt @@ -27,12 +27,14 @@ package com.tencent.devops.log.cron.impl +import com.tencent.devops.common.es.client.LogClient import com.tencent.devops.common.redis.RedisLock import com.tencent.devops.common.redis.RedisOperation -import com.tencent.devops.log.client.LogClient import com.tencent.devops.log.configuration.StorageProperties import com.tencent.devops.log.cron.IndexCleanJob import com.tencent.devops.log.util.IndexNameUtils.LOG_INDEX_PREFIX +import java.time.LocalDateTime +import java.time.temporal.ChronoUnit import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest import org.elasticsearch.client.RequestOptions @@ -44,8 +46,6 @@ import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty import org.springframework.scheduling.annotation.Scheduled import org.springframework.stereotype.Component -import java.time.LocalDateTime -import java.time.temporal.ChronoUnit @Suppress("MagicNumber") @Component diff --git a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/es/LogESAutoConfiguration.kt b/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/es/LogESAutoConfiguration.kt new file mode 100644 index 00000000000..4f896bb099f --- /dev/null +++ b/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/es/LogESAutoConfiguration.kt @@ -0,0 +1,81 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.log.es + +import com.tencent.devops.common.es.ESAutoConfiguration +import com.tencent.devops.common.es.ESProperties +import com.tencent.devops.common.es.client.LogClient +import com.tencent.devops.common.redis.RedisOperation +import com.tencent.devops.log.jmx.CreateIndexBean +import com.tencent.devops.log.jmx.LogStorageBean +import com.tencent.devops.log.service.BuildLogPrintService +import com.tencent.devops.log.service.IndexService +import com.tencent.devops.log.service.LogService +import com.tencent.devops.log.service.LogStatusService +import com.tencent.devops.log.service.LogTagService +import com.tencent.devops.log.service.impl.LogServiceESImpl +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.autoconfigure.AutoConfigureAfter +import org.springframework.boot.autoconfigure.AutoConfigureOrder +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty +import org.springframework.boot.context.properties.EnableConfigurationProperties +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Configuration +import org.springframework.core.Ordered + +@Suppress("ALL") +@Configuration +@ConditionalOnProperty(prefix = "log.storage", name = ["type"], havingValue = "elasticsearch") +@AutoConfigureOrder(Ordered.HIGHEST_PRECEDENCE) +@AutoConfigureAfter(ESAutoConfiguration::class) +@EnableConfigurationProperties(ESProperties::class) +class LogESAutoConfiguration { + @Bean + fun esLogService( + @Autowired logESClient: LogClient, + @Autowired indexService: IndexService, + @Autowired logStatusService: LogStatusService, + @Autowired logTagService: LogTagService, + @Autowired defaultKeywords: List, + @Autowired createIndexBean: CreateIndexBean, + @Autowired logStorageBean: LogStorageBean, + @Autowired redisOperation: RedisOperation, + @Autowired buildLogPrintService: BuildLogPrintService + ): LogService { + return LogServiceESImpl( + logClient = logESClient, + indexService = indexService, + logStatusService = logStatusService, + logTagService = logTagService, + logStorageBean = logStorageBean, + createIndexBean = createIndexBean, + buildLogPrintService = buildLogPrintService, + redisOperation = redisOperation + ) + } +} diff --git a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/service/impl/LogServiceESImpl.kt b/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/service/impl/LogServiceESImpl.kt index d20e3cc680f..a984c331bef 100644 --- a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/service/impl/LogServiceESImpl.kt +++ b/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/service/impl/LogServiceESImpl.kt @@ -30,6 +30,7 @@ package com.tencent.devops.log.service.impl import com.github.benmanes.caffeine.cache.Caffeine import com.tencent.devops.common.api.exception.ExecuteException import com.tencent.devops.common.api.pojo.Page +import com.tencent.devops.common.es.client.LogClient import com.tencent.devops.common.log.constant.LogMessageCode.LOG_INDEX_HAS_BEEN_CLEANED import com.tencent.devops.common.log.pojo.EndPageQueryLogs import com.tencent.devops.common.log.pojo.LogLine @@ -42,8 +43,7 @@ import com.tencent.devops.common.log.pojo.message.LogMessageWithLineNo import com.tencent.devops.common.redis.RedisLock import com.tencent.devops.common.redis.RedisOperation import com.tencent.devops.common.web.utils.I18nUtil -import com.tencent.devops.log.client.LogClient -import com.tencent.devops.log.es.ESClient +import com.tencent.devops.common.es.ESClient import com.tencent.devops.log.event.LogOriginEvent import com.tencent.devops.log.event.LogStatusEvent import com.tencent.devops.log.event.LogStorageEvent @@ -57,6 +57,14 @@ import com.tencent.devops.log.service.LogTagService import com.tencent.devops.log.util.Constants import com.tencent.devops.log.util.ESIndexUtils import com.tencent.devops.log.util.IndexNameUtils +import java.io.IOException +import java.sql.Date +import java.text.SimpleDateFormat +import java.util.concurrent.TimeUnit +import javax.ws.rs.core.MediaType +import javax.ws.rs.core.Response +import javax.ws.rs.core.StreamingOutput +import kotlin.math.ceil import org.elasticsearch.ElasticsearchStatusException import org.elasticsearch.action.admin.indices.open.OpenIndexRequest import org.elasticsearch.action.bulk.BulkRequest @@ -77,14 +85,6 @@ import org.elasticsearch.search.builder.SearchSourceBuilder import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder import org.elasticsearch.search.sort.SortOrder import org.slf4j.LoggerFactory -import java.io.IOException -import java.sql.Date -import java.text.SimpleDateFormat -import java.util.concurrent.TimeUnit -import javax.ws.rs.core.MediaType -import javax.ws.rs.core.Response -import javax.ws.rs.core.StreamingOutput -import kotlin.math.ceil @Suppress( "LongParameterList", @@ -143,7 +143,7 @@ class LogServiceESImpl constructor( if (doAddMultiLines(buf, event.buildId) == 0) { throw ExecuteException( "None of lines is inserted successfully to ES " + - "[${event.buildId}|${event.retryTime}]" + "[${event.buildId}|${event.retryTime}]" ) } else { buf.clear() @@ -684,7 +684,7 @@ class LogServiceESImpl constructor( ) logger.info( "[$index|$buildId|$tag|$subTag|$jobId|$executeCount] " + - "doQueryInitLogs get the query builder: $boolQueryBuilder" + "doQueryInitLogs get the query builder: $boolQueryBuilder" ) val searchRequest = SearchRequest(index) @@ -753,7 +753,7 @@ class LogServiceESImpl constructor( logger.info( "[$index|$buildId|$tag|$subTag|$jobId|$executeCount] " + - "doQueryLogsAfterLine get the query builder: $boolQueryBuilder" + "doQueryLogsAfterLine get the query builder: $boolQueryBuilder" ) val searchRequest = SearchRequest(index) .source( @@ -840,9 +840,11 @@ class LogServiceESImpl constructor( end >= size -> { end - size } + logSize >= size -> { logSize - size } + else -> { 0 } @@ -859,7 +861,7 @@ class LogServiceESImpl constructor( .must(QueryBuilders.rangeQuery("lineNo").lte(end)) logger.info( "[$index|$buildId|$tag|$subTag|$jobId|$executeCount] " + - "doQueryLogsBeforeLine get the query builder: $boolQueryBuilder" + "doQueryLogsBeforeLine get the query builder: $boolQueryBuilder" ) val searchRequest = SearchRequest(index) .source( @@ -1136,7 +1138,7 @@ class LogServiceESImpl constructor( return try { logger.info( "[${createClient.clusterName}][$index]|createIndex|: shards[${createClient.shards}]" + - " replicas[${createClient.replicas}] shardsPerNode[${createClient.shardsPerNode}]" + " replicas[${createClient.replicas}] shardsPerNode[${createClient.shardsPerNode}]" ) val request = CreateIndexRequest(index) .settings( diff --git a/src/backend/ci/core/log/biz-log/src/main/resources/META-INF/spring.factories b/src/backend/ci/core/log/biz-log/src/main/resources/META-INF/spring.factories index fc65d3b15bb..c5a4f1375f7 100644 --- a/src/backend/ci/core/log/biz-log/src/main/resources/META-INF/spring.factories +++ b/src/backend/ci/core/log/biz-log/src/main/resources/META-INF/spring.factories @@ -1,3 +1,3 @@ org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ -com.tencent.devops.log.es.ESAutoConfiguration,\ -com.tencent.devops.log.lucene.LuceneAutoConfiguration \ No newline at end of file +com.tencent.devops.log.es.LogESAutoConfiguration,\ +com.tencent.devops.log.lucene.LuceneAutoConfiguration diff --git a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/pojo/MetricsApiData.kt b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/pojo/MetricsApiData.kt new file mode 100644 index 00000000000..59e6ceb7983 --- /dev/null +++ b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/pojo/MetricsApiData.kt @@ -0,0 +1,50 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.tencent.devops.openapi.pojo + +import io.swagger.annotations.ApiModel +import io.swagger.annotations.ApiModelProperty + +@ApiModel("api度量数据") +data class MetricsApiData( + @ApiModelProperty("接口代码") + val api: String, + @ApiModelProperty("请求app code/ user id") + val key: String, + @ApiModelProperty("秒级并发量") + var secondLevelConcurrency: Int? = null, + @ApiModelProperty("峰值并发量") + var peakConcurrency: Int? = null, + @ApiModelProperty("5min调用量") + var call5m: Int? = null, + @ApiModelProperty("1h调用量") + var call1h: Int? = null, + @ApiModelProperty("24h调用量") + var call24h: Int? = null, + @ApiModelProperty("7d调用量") + var call7d: Int? = null +) diff --git a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/pojo/MetricsProjectData.kt b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/pojo/MetricsProjectData.kt new file mode 100644 index 00000000000..64cf3d26851 --- /dev/null +++ b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/pojo/MetricsProjectData.kt @@ -0,0 +1,42 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.tencent.devops.openapi.pojo + +import io.swagger.annotations.ApiModel +import io.swagger.annotations.ApiModelProperty + +@ApiModel("api度量数据") +data class MetricsProjectData( + @ApiModelProperty("接口代码") + val api: String, + @ApiModelProperty("请求app code/ user id") + val key: String, + @ApiModelProperty("项目id") + var projectId: String? = null, + @ApiModelProperty("调用量") + var callHistory: Int? = null +) diff --git a/src/backend/ci/core/openapi/biz-openapi/build.gradle.kts b/src/backend/ci/core/openapi/biz-openapi/build.gradle.kts index 2c058459137..3a54077df65 100644 --- a/src/backend/ci/core/openapi/biz-openapi/build.gradle.kts +++ b/src/backend/ci/core/openapi/biz-openapi/build.gradle.kts @@ -30,6 +30,8 @@ dependencies { api(project(":core:openapi:api-openapi")) api(project(":core:common:common-db")) api(project(":core:common:common-pipeline")) + api(project(":core:common:common-es")) + api(project(":core:common:common-stream")) api(project(":core:repository:api-repository")) api(project(":core:environment:api-environment")) api(project(":core:artifactory:api-artifactory")) diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/aspect/ApiAspect.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/aspect/ApiAspect.kt index 4f2dc590645..a1e6d3ca51e 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/aspect/ApiAspect.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/aspect/ApiAspect.kt @@ -26,6 +26,7 @@ */ package com.tencent.devops.openapi.aspect +import com.github.benmanes.caffeine.cache.Caffeine import com.tencent.devops.common.api.constant.HTTP_500 import com.tencent.devops.common.api.exception.CustomException import com.tencent.devops.common.api.exception.ParamBlankException @@ -37,9 +38,14 @@ import com.tencent.devops.common.service.BkTag import com.tencent.devops.common.web.utils.I18nUtil import com.tencent.devops.openapi.IgnoreProjectId import com.tencent.devops.openapi.constant.OpenAPIMessageCode.PARAM_VERIFY_FAIL +import com.tencent.devops.openapi.es.ESMessage +import com.tencent.devops.openapi.es.IESService import com.tencent.devops.openapi.service.OpenapiPermissionService import com.tencent.devops.openapi.service.op.AppCodeService import com.tencent.devops.openapi.utils.ApiGatewayUtil +import io.swagger.annotations.ApiOperation +import javax.ws.rs.core.Response +import kotlin.reflect.jvm.kotlinFunction import org.aspectj.lang.JoinPoint import org.aspectj.lang.ProceedingJoinPoint import org.aspectj.lang.annotation.Around @@ -48,8 +54,8 @@ import org.aspectj.lang.reflect.MethodSignature import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Value import org.springframework.stereotype.Component -import javax.ws.rs.core.Response -import kotlin.reflect.jvm.kotlinFunction +import org.springframework.web.context.request.RequestContextHolder +import org.springframework.web.context.request.ServletRequestAttributes @Aspect @Component @@ -58,7 +64,8 @@ class ApiAspect( private val apiGatewayUtil: ApiGatewayUtil, private val redisOperation: RedisOperation, private val bkTag: BkTag, - private val permissionService: OpenapiPermissionService + private val permissionService: OpenapiPermissionService, + private val esService: IESService ) { companion object { @@ -68,6 +75,10 @@ class ApiAspect( @Value("\${openapi.verify.project: #{null}}") val verifyProjectFlag: String = "false" + private val apiTagCache = Caffeine.newBuilder() + .maximumSize(500) + .build() + /** * 前置增强：目标方法执行之前执行 * @@ -100,6 +111,28 @@ class ApiAspect( } } + kotlin.runCatching { + if (esService.esReady()) { + val request = (RequestContextHolder.currentRequestAttributes() as ServletRequestAttributes).request + val apiType = apigwType?.split("-")?.getOrNull(1) ?: "" + esService.addMessage( + ESMessage( + api = getApiTag(jp = jp, apiType = apiType), + key = when { + apiType.isBlank() -> "null" + apiType.contains("user") -> "user:$userId" + else -> "app:$appCode" + }, + projectId = projectId ?: "", + path = request.requestURI, + timestamp = System.currentTimeMillis() + ) + ) + } + }.onFailure { + logger.error("es add message error ${it.message}", it) + } + if (logger.isDebugEnabled) { val methodName: String = jp.signature.name @@ -223,4 +256,22 @@ class ApiAspect( // 删除线程ThreadLocal数据,防止线程池复用。导致流量指向被污染 bkTag.removeGatewayTag() } + + private fun getApiTag(jp: JoinPoint, apiType: String): String { + val method = (jp.signature as MethodSignature).method + val methodName = method.declaringClass.typeName + "." + method.name + return apiTagCache.get(methodName) { + jp.target + ?.javaClass + ?.interfaces + ?.first() + ?.getDeclaredMethod( + method.name, *method.parameterTypes + ) + ?.getAnnotation(ApiOperation::class.java) + ?.tags + ?.first() + ?.replace(Regex("app|user"), apiType) ?: methodName + } ?: methodName + } } diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/dao/MetricsForApiDao.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/dao/MetricsForApiDao.kt new file mode 100644 index 00000000000..6afc0def3dc --- /dev/null +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/dao/MetricsForApiDao.kt @@ -0,0 +1,113 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.openapi.dao + +import com.tencent.devops.common.db.utils.skipCheck +import com.tencent.devops.model.openapi.tables.TOpenapiMetricsForApi +import com.tencent.devops.openapi.pojo.MetricsApiData +import org.jooq.DSLContext +import org.springframework.stereotype.Repository + +@Repository +class MetricsForApiDao { + + companion object { + const val batchGetLimit = Short.MAX_VALUE + } + + @Suppress("ComplexMethod") + fun createOrUpdate( + dslContext: DSLContext, + metricsApis: List, + perHour: Boolean, + perDay: Boolean + ): Int { + return with(TOpenapiMetricsForApi.T_OPENAPI_METRICS_FOR_API) { + dslContext.batch( + metricsApis.mapNotNull { metricsApi -> + dslContext.insertInto( + this, + API, + KEY, + SECOND_LEVEL_CONCURRENCY, + PEAK_CONCURRENCY, + CALL_5M, + CALL_1H, + CALL_24H, + CALL_7D + ).values( + metricsApi.api, + metricsApi.key, + metricsApi.secondLevelConcurrency ?: 0, + metricsApi.peakConcurrency ?: 0, + metricsApi.call5m ?: 0, + metricsApi.call1h ?: 0, + metricsApi.call24h ?: 0, + metricsApi.call7d ?: 0 + ).onDuplicateKeyUpdate() + .let { u -> + metricsApi.secondLevelConcurrency?.let { i -> + u.set(SECOND_LEVEL_CONCURRENCY, i) + } ?: u.set(SECOND_LEVEL_CONCURRENCY, 0) + } + .let { u -> + metricsApi.peakConcurrency?.let { i -> + u.set(PEAK_CONCURRENCY, i) + } ?: u.set(PEAK_CONCURRENCY, 0) + } + .let { u -> + metricsApi.call5m?.let { i -> u.set(CALL_5M, i) } ?: u.set(CALL_5M, 0) + } + .let { u -> + metricsApi.call1h?.let { i -> u.set(CALL_1H, i) } ?: if (perHour) u.set(CALL_1H, 0) else u + } + .let { u -> + metricsApi.call24h?.let { i -> u.set(CALL_24H, i) } ?: if (perDay) u.set(CALL_24H, 0) else u + } + .let { u -> + metricsApi.call7d?.let { i -> u.set(CALL_7D, i) } ?: if (perDay) u.set(CALL_7D, 0) else u + } + } + ).execute().sum() + } + } + + fun batchGet( + dslContext: DSLContext + ): List { + return with(TOpenapiMetricsForApi.T_OPENAPI_METRICS_FOR_API) { + dslContext.select(API, KEY, PEAK_CONCURRENCY).from(this).limit(batchGetLimit).skipCheck().fetch { + MetricsApiData( + api = it.value1(), + key = it.value2(), + peakConcurrency = it.value3() + ) + } + } + } +} diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/dao/MetricsForProjectDao.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/dao/MetricsForProjectDao.kt new file mode 100644 index 00000000000..6459c454b51 --- /dev/null +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/dao/MetricsForProjectDao.kt @@ -0,0 +1,65 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.openapi.dao + +import com.tencent.devops.model.openapi.tables.TOpenapiMetricsForProject +import com.tencent.devops.openapi.pojo.MetricsProjectData +import org.jooq.DSLContext +import org.jooq.InsertOnDuplicateSetMoreStep +import org.springframework.stereotype.Repository + +@Repository +class MetricsForProjectDao { + fun createOrUpdate( + dslContext: DSLContext, + metricsApis: List + ): Int { + return with(TOpenapiMetricsForProject.T_OPENAPI_METRICS_FOR_PROJECT) { + dslContext.batch( + metricsApis.mapNotNull { metricsApi -> + dslContext.insertInto( + this, + PROJECT, + API, + KEY, + CALL_HISTORY + ).values( + metricsApi.projectId ?: "", + metricsApi.api, + metricsApi.key, + metricsApi.callHistory ?: 0 + ).onDuplicateKeyUpdate() + .let { u -> + metricsApi.callHistory?.let { i -> u.set(CALL_HISTORY, CALL_HISTORY + i) } ?: u + } + .let { if (it is InsertOnDuplicateSetMoreStep) it else return@mapNotNull null } + } + ).execute().sum() + } + } +} diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESIndexUtils.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESIndexUtils.kt new file mode 100644 index 00000000000..6425a8ed1d9 --- /dev/null +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESIndexUtils.kt @@ -0,0 +1,70 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.openapi.es + +import org.elasticsearch.common.settings.Settings +import org.elasticsearch.xcontent.XContentBuilder +import org.elasticsearch.xcontent.XContentFactory + +object ESIndexUtils { + + fun getIndexSettings(shards: Int, replicas: Int, shardsPerNode: Int): Settings.Builder { + return Settings.builder() + .put("index.number_of_shards", shards) + .put("index.number_of_replicas", replicas) + .put("index.refresh_interval", "3s") + .put("index.queries.cache.enabled", false) + .put("index.routing.allocation.total_shards_per_node", shardsPerNode) + } + + fun getTypeMappings(): XContentBuilder { + return XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(ESMessage::api.name).field("type", "keyword").endObject() + .startObject(ESMessage::timestamp.name).field("type", "date").endObject() + .startObject(ESMessage::key.name).field("type", "keyword").endObject() + .startObject(ESMessage::path.name).field("type", "text").endObject() + .startObject(ESMessage::projectId.name).field("type", "keyword").endObject() + .endObject() + .endObject() + } + + fun getDocumentObject( + logMessage: ESMessage + ): XContentBuilder { + return XContentFactory.jsonBuilder() + .startObject() + .field(ESMessage::api.name, logMessage.api) + .field(ESMessage::key.name, logMessage.key) + .field(ESMessage::projectId.name, logMessage.projectId) + .field(ESMessage::path.name, logMessage.path) + .field(ESMessage::timestamp.name, logMessage.timestamp) + .endObject() + } +} diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESMessage.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESMessage.kt new file mode 100644 index 00000000000..3fb5a47a04d --- /dev/null +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESMessage.kt @@ -0,0 +1,40 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.openapi.es + +/** + * + * Powered By Tencent + */ +data class ESMessage( + val api: String, + val key: String = "", + val projectId: String = "", + val path: String = "", + var timestamp: Long = 0 +) diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/IESService.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/IESService.kt new file mode 100644 index 00000000000..590164e8f62 --- /dev/null +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/IESService.kt @@ -0,0 +1,38 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.openapi.es + +import com.tencent.devops.openapi.es.mq.ESEvent + +interface IESService { + + fun addMessage(message: ESMessage) + fun esAddMessage(event: ESEvent) + + fun esReady() = false +} diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/IndexNameUtils.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/IndexNameUtils.kt new file mode 100644 index 00000000000..a2f1a294ca1 --- /dev/null +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/IndexNameUtils.kt @@ -0,0 +1,51 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.openapi.es + +import java.time.LocalDateTime +import java.time.format.DateTimeFormatter + +object IndexNameUtils { + + fun getIndexName(): String { + val formatter = DateTimeFormatter.ofPattern(LOG_INDEX_DATE_FORMAT) + return LOG_INDEX_PREFIX + formatter.format(LocalDateTime.now()) + } + + fun getIndexNamePrefix(): String { + return "$LOG_INDEX_PREFIX*" + } + + fun getNextIndexName(): String { + val formatter = DateTimeFormatter.ofPattern(LOG_INDEX_DATE_FORMAT) + return LOG_INDEX_PREFIX + formatter.format(LocalDateTime.now().plusDays(1)) + } + + private const val LOG_INDEX_PREFIX = "openapi-metric-log-" + private const val LOG_INDEX_DATE_FORMAT = "yyyy-MM-dd" +} diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/MetricsService.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/MetricsService.kt new file mode 100644 index 00000000000..f2a2d86173a --- /dev/null +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/MetricsService.kt @@ -0,0 +1,120 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.tencent.devops.openapi.es + +import com.tencent.devops.common.redis.RedisLock +import com.tencent.devops.common.redis.RedisOperation +import com.tencent.devops.openapi.dao.MetricsForApiDao +import com.tencent.devops.openapi.dao.MetricsForProjectDao +import com.tencent.devops.openapi.es.impl.ESServiceImpl +import com.tencent.devops.openapi.pojo.MetricsApiData +import com.tencent.devops.openapi.pojo.MetricsProjectData +import java.time.Duration +import java.time.LocalTime +import org.jooq.DSLContext +import org.slf4j.LoggerFactory + +class MetricsService( + private val dslContext: DSLContext, + private val apiDao: MetricsForApiDao, + private val projectDao: MetricsForProjectDao, + private val esServiceImpl: ESServiceImpl, + private val redisOperation: RedisOperation +) { + companion object { + private val logger = LoggerFactory.getLogger(MetricsService::class.java) + private const val ES_INDEX_CLOSE_JOB_KEY = "openapi:es:index:close:job:lock:key" + private const val EACH_HOUR = 12 + private const val MOD_HOUR = 4 + private const val EACH_DAY = 288 + private const val MOD_DAY = 10 + } + + /** + * every 5m + */ +// @Scheduled(cron = "0 0/5 * * * ?") + fun job() { + logger.info("Start to openapi metrics job") + RedisLock(redisOperation, ES_INDEX_CLOSE_JOB_KEY, 60).run { + if (!this.tryLock()) { + return + } + val begin = System.currentTimeMillis() + val keyMap = mutableMapOf() + apiDao.batchGet(dslContext).associateByTo(keyMap) { "${it.api}@${it.key}" } + val between = Duration.between(LocalTime.of(0, 0), LocalTime.now()).toMinutes().toInt() / 5 + val hourMod = between % EACH_HOUR + val dayMod = between % EACH_DAY + logger.info("openapi metrics job dayMod=$dayMod") + esServiceImpl.executeElasticsearchQueryS(keyMap = keyMap, newDay = dayMod == MOD_DAY) + // 每小时 + if (hourMod == MOD_HOUR) { + logger.info("Start to openapi metrics job for 1H") + jobFor1H(keyMap) + } + // 每天 + if (dayMod == MOD_DAY) { + logger.info("Start to openapi metrics job for 1D") + jobFor24H(keyMap) + jobFor7D(keyMap) + } + apiDao.createOrUpdate( + dslContext = dslContext, + metricsApis = keyMap.values.toList(), + perHour = hourMod == MOD_HOUR, + perDay = dayMod == MOD_DAY + ) + + logger.info("execution time ${System.currentTimeMillis() - begin} millisecond") + } + } + + private fun jobFor1H(keyMap: MutableMap) { + esServiceImpl.executeElasticsearchQueryM(keyMap, "1h") { count, data -> + data.call1h = count + return@executeElasticsearchQueryM data + } + val projects = mutableListOf() + esServiceImpl.executeElasticsearchQueryP(projects) + projectDao.createOrUpdate(dslContext, projects) + } + + private fun jobFor24H(keyMap: MutableMap) { + esServiceImpl.executeElasticsearchQueryM(keyMap, "1d") { count, data -> + data.call24h = count + return@executeElasticsearchQueryM data + } + } + + private fun jobFor7D(keyMap: MutableMap) { + esServiceImpl.executeElasticsearchQueryM(keyMap, "7d") { count, data -> + data.call7d = count + return@executeElasticsearchQueryM data + } + } +} diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/config/ESAutoConfiguration.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/config/ESAutoConfiguration.kt new file mode 100644 index 00000000000..b8f84d0d622 --- /dev/null +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/config/ESAutoConfiguration.kt @@ -0,0 +1,101 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.openapi.es.config + +import com.tencent.devops.common.es.ESAutoConfiguration +import com.tencent.devops.common.es.ESProperties +import com.tencent.devops.common.es.client.LogClient +import com.tencent.devops.common.redis.RedisOperation +import com.tencent.devops.openapi.dao.MetricsForApiDao +import com.tencent.devops.openapi.dao.MetricsForProjectDao +import com.tencent.devops.openapi.es.IESService +import com.tencent.devops.openapi.es.MetricsService +import com.tencent.devops.openapi.es.impl.DefaultESServiceImpl +import com.tencent.devops.openapi.es.impl.ESServiceImpl +import com.tencent.devops.openapi.es.mq.MQDispatcher +import org.jooq.DSLContext +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.beans.factory.annotation.Value +import org.springframework.boot.autoconfigure.AutoConfigureAfter +import org.springframework.boot.autoconfigure.AutoConfigureOrder +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty +import org.springframework.boot.context.properties.EnableConfigurationProperties +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Configuration +import org.springframework.core.Ordered + +@Suppress("ALL") +@Configuration +@AutoConfigureOrder(Ordered.HIGHEST_PRECEDENCE) +@AutoConfigureAfter(ESAutoConfiguration::class) +@EnableConfigurationProperties(ESProperties::class) +class ESAutoConfiguration { + + @Value("\${log.elasticsearch.consumerCount:1}") + val consumerCount: Int = 1 + + @Bean + @ConditionalOnProperty(prefix = "log.storage", name = ["type"], havingValue = "elasticsearch") + fun esLogService( + @Autowired logESClient: LogClient, + @Autowired redisOperation: RedisOperation, + @Autowired openapiMQDispatcher: MQDispatcher + ): ESServiceImpl { + return ESServiceImpl( + logClient = logESClient, + redisOperation = redisOperation, + dispatcher = openapiMQDispatcher, + configuration = this + ) + } + + @Bean + @ConditionalOnProperty(prefix = "log.storage", name = ["type"], havingValue = "elasticsearch") + fun metricsService( + @Autowired dslContext: DSLContext, + @Autowired apiDao: MetricsForApiDao, + @Autowired projectDao: MetricsForProjectDao, + @Autowired esServiceImpl: ESServiceImpl, + @Autowired redisOperation: RedisOperation + ): MetricsService { + return MetricsService( + dslContext = dslContext, + apiDao = apiDao, + projectDao = projectDao, + esServiceImpl = esServiceImpl, + redisOperation = redisOperation + ) + } + + @Bean + @ConditionalOnMissingBean(IESService::class) + fun defaultLogService(): DefaultESServiceImpl { + return DefaultESServiceImpl() + } +} diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/config/MQConfiguration.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/config/MQConfiguration.kt new file mode 100644 index 00000000000..1acebcb0d87 --- /dev/null +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/config/MQConfiguration.kt @@ -0,0 +1,71 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.openapi.es.config + +import com.tencent.devops.common.stream.constants.StreamBinding +import com.tencent.devops.openapi.es.IESService +import com.tencent.devops.openapi.es.mq.ESEvent +import com.tencent.devops.openapi.es.mq.MQDispatcher +import com.tencent.devops.openapi.es.mq.MQListenerService +import java.util.function.Consumer +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.autoconfigure.AutoConfigureOrder +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty +import org.springframework.boot.autoconfigure.condition.ConditionalOnWebApplication +import org.springframework.cloud.stream.function.StreamBridge +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Configuration +import org.springframework.core.Ordered +import org.springframework.messaging.Message + +@Configuration +@ConditionalOnWebApplication +@ConditionalOnProperty(prefix = "log.storage", name = ["type"], havingValue = "elasticsearch") +@AutoConfigureOrder(Ordered.LOWEST_PRECEDENCE) +class MQConfiguration @Autowired constructor() { + + @Bean + fun openapiMQDispatcher( + @Autowired streamBridge: StreamBridge + ) = MQDispatcher(streamBridge) + + @Bean(StreamBinding.BINDING_OPENAPI_LOG_EVENT_IN) + fun openapiLogEventIn( + @Autowired listenerService: MQListenerService + ): Consumer> { + return Consumer { event: Message -> + listenerService.handleEvent(event.payload) + } + } + + @Bean + fun mqListenerService( + @Autowired logService: IESService, + @Autowired dispatcher: MQDispatcher + ) = MQListenerService(logService, dispatcher) +} diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/impl/DefaultESServiceImpl.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/impl/DefaultESServiceImpl.kt new file mode 100644 index 00000000000..61579f688ea --- /dev/null +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/impl/DefaultESServiceImpl.kt @@ -0,0 +1,38 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.openapi.es.impl + +import com.tencent.devops.openapi.es.ESMessage +import com.tencent.devops.openapi.es.IESService +import com.tencent.devops.openapi.es.mq.ESEvent + +class DefaultESServiceImpl : IESService { + + override fun addMessage(message: ESMessage) = Unit + override fun esAddMessage(event: ESEvent) = Unit +} diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/impl/ESServiceImpl.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/impl/ESServiceImpl.kt new file mode 100644 index 00000000000..74719d65fde --- /dev/null +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/impl/ESServiceImpl.kt @@ -0,0 +1,521 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.openapi.es.impl + +import com.github.benmanes.caffeine.cache.Caffeine +import com.tencent.devops.common.api.exception.ExecuteException +import com.tencent.devops.common.es.ESClient +import com.tencent.devops.common.es.client.LogClient +import com.tencent.devops.common.redis.RedisLock +import com.tencent.devops.common.redis.RedisOperation +import com.tencent.devops.openapi.es.ESIndexUtils +import com.tencent.devops.openapi.es.ESMessage +import com.tencent.devops.openapi.es.IESService +import com.tencent.devops.openapi.es.IndexNameUtils +import com.tencent.devops.openapi.es.config.ESAutoConfiguration +import com.tencent.devops.openapi.es.mq.ESEvent +import com.tencent.devops.openapi.es.mq.MQDispatcher +import com.tencent.devops.openapi.pojo.MetricsApiData +import com.tencent.devops.openapi.pojo.MetricsProjectData +import java.io.IOException +import java.util.concurrent.ArrayBlockingQueue +import java.util.concurrent.BlockingQueue +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit +import org.elasticsearch.action.bulk.BulkRequest +import org.elasticsearch.action.index.IndexRequest +import org.elasticsearch.action.search.SearchRequest +import org.elasticsearch.action.search.SearchResponse +import org.elasticsearch.client.HttpAsyncResponseConsumerFactory.HeapBufferedResponseConsumerFactory +import org.elasticsearch.client.RequestOptions +import org.elasticsearch.client.indices.CreateIndexRequest +import org.elasticsearch.client.indices.GetIndexRequest +import org.elasticsearch.core.TimeValue +import org.elasticsearch.index.query.QueryBuilders +import org.elasticsearch.search.aggregations.AggregationBuilders +import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval +import org.elasticsearch.search.aggregations.bucket.terms.Terms +import org.elasticsearch.search.aggregations.pipeline.AvgBucketPipelineAggregationBuilder +import org.elasticsearch.search.aggregations.pipeline.BucketMetricValue +import org.elasticsearch.search.aggregations.pipeline.MaxBucketPipelineAggregationBuilder +import org.elasticsearch.search.aggregations.pipeline.SimpleValue +import org.elasticsearch.search.builder.SearchSourceBuilder +import org.slf4j.LoggerFactory + +@Suppress("NestedBlockDepth", "ComplexMethod") +class ESServiceImpl constructor( + private val logClient: LogClient, + private val redisOperation: RedisOperation, + private val dispatcher: MQDispatcher, + private val configuration: ESAutoConfiguration +) : IESService { + + companion object { + private val logger = LoggerFactory.getLogger(ESServiceImpl::class.java) + private const val SEARCH_TIMEOUT_SECONDS = 60L + private const val INDEX_CACHE_MAX_SIZE = 10L + private const val INDEX_CACHE_EXPIRE_DAY = 1L + private const val INDEX_LOCK_EXPIRE_SECONDS = 10L + private const val INDEX_STORAGE_WARN_MILLIS = 1000 + private const val BULK_BUFFER_SIZE = 100 + private const val maxQueueSize = 9000 // 队列溢出的最大大小 + private const val RESPONSE_ENTITY_MAX_SIZE = 1024 * 1024 * 1024 + } + + private val queue: BlockingQueue = ArrayBlockingQueue(10000) // 创建一个容量为10000的阻塞队列 + + private val indexCache = Caffeine.newBuilder() + .maximumSize(INDEX_CACHE_MAX_SIZE) + .expireAfterAccess(INDEX_CACHE_EXPIRE_DAY, TimeUnit.DAYS) + .build() + + private val executor = Executors.newCachedThreadPool() + + init { + repeat(configuration.consumerCount) { + executor.submit(BulkSend()) + } + } + + override fun addMessage(message: ESMessage) { + if (queue.size >= maxQueueSize) { + dispatcher.dispatchEvent(ESEvent(message)) // 将消息推送到es + } else { + queue.put(message) // 将消息放入队列，如果队列已满则阻塞等待 + } + } + + override fun esReady() = true + + override fun esAddMessage(event: ESEvent) { + queue.put(event.logs) + } + + private inner class BulkSend : Runnable { + val buf = mutableListOf() + override fun run() { + while (true) { + val message = queue.take() ?: continue + buf.add(message) + if (buf.size == BULK_BUFFER_SIZE) { + val currentEpoch = System.currentTimeMillis() + try { + prepareIndex() + if (doAddMultiLines(buf) == 0) { + throw ExecuteException( + "None of lines is inserted successfully to ES " + ) + } else { + buf.clear() + } + } finally { + val elapse = System.currentTimeMillis() - currentEpoch + // #4265 当日志消息处理时间过长时打印消息内容 + if (elapse >= INDEX_STORAGE_WARN_MILLIS) logger.warn( + " addBatchLogEvent spent too much time($elapse)" + ) + } + } + } + } + } + + private fun doAddMultiLines(logMessages: List): Int { + val currentEpoch = System.currentTimeMillis() + val index = IndexNameUtils.getIndexName() + val bulkClient = logClient.hashClient(index) + var lines = 0 + var bulkLines = 0 + val bulkRequest = BulkRequest() + .timeout(TimeValue.timeValueMillis(bulkClient.requestTimeout)) + for (i in logMessages.indices) { + val logMessage = logMessages[i] + + val indexRequest = genIndexRequest( + logMessage = logMessage, + index = index + ) + if (indexRequest != null) { + bulkRequest.add(indexRequest) + lines++ + } + } + try { + val bulkResponse = bulkClient.restClient.bulk(bulkRequest, RequestOptions.DEFAULT) + bulkLines = bulkResponse.count() + return if (bulkResponse.hasFailures()) { + throw ExecuteException(bulkResponse.buildFailureMessage()) + } else { + bulkLines + } + } catch (ignore: Exception) { + val exString = ignore.toString() + if (exString.contains("circuit_breaking_exception")) { + logger.warn( + "Add bulk lines failed|$exString, attempting to add index. [$logMessages]", + ignore + ) + val bulkResponse = bulkClient.restClient.bulk( + bulkRequest.timeout(TimeValue.timeValueSeconds(SEARCH_TIMEOUT_SECONDS)), + genLargeSearchOptions() + ) + bulkLines = bulkResponse.count() + return if (bulkResponse.hasFailures()) { + logger.error(bulkResponse.buildFailureMessage()) + 0 + } else { + bulkLines + } + } else { + logger.warn("Add bulk lines failed because of unknown Exception. [$logMessages]", ignore) + throw ignore + } + } finally { + if (bulkLines != lines) { + logger.warn("Part of bulk lines failed, lines:$lines, bulkLines:$bulkLines") + } + val elapse = System.currentTimeMillis() - currentEpoch + + // #4265 当日志消息处理时间过长时打印消息内容 + if (elapse >= INDEX_STORAGE_WARN_MILLIS && logMessages.isNotEmpty()) logger.warn( + "doAddMultiLines spent too much time($elapse) with tag=${logMessages.first()}" + ) + } + } + + private fun genIndexRequest( + logMessage: ESMessage, + index: String + ): IndexRequest? { + val builder = ESIndexUtils.getDocumentObject(logMessage) + return try { + IndexRequest(index).source(builder) + } catch (e: IOException) { + logger.error("Convert logMessage to es document failure", e) + null + } finally { + builder.close() + } + } + + private fun prepareIndex(): Boolean { + val index = IndexNameUtils.getIndexName() + return if (!checkIndexCreate(index)) { + createIndex(index) + indexCache.put(index, true) + true + } else { + false + } + } + + private fun checkIndexCreate(index: String): Boolean { + if (indexCache.getIfPresent(index) == true) { + return true + } + val redisLock = RedisLock(redisOperation, "LOG:index:create:lock:key:$index", INDEX_LOCK_EXPIRE_SECONDS) + try { + redisLock.lock() + if (indexCache.getIfPresent(index) == true) { + return true + } + + // Check from ES + if (isExistIndex(index)) { + logger.info("[$index] the index is already created") + indexCache.put(index, true) + return true + } + return false + } finally { + redisLock.unlock() + } + } + + private fun createIndex(index: String): Boolean { + val createClient = logClient.hashClient(index) + // 提前创建第二天的索引备用 + createESIndex(createClient, IndexNameUtils.getNextIndexName()) + return createESIndex(createClient, index) + } + + private fun createESIndex(createClient: ESClient, index: String): Boolean { + logger.info("[$index] Create index") + return try { + logger.info( + "[${createClient.clusterName}][$index]|createIndex|: shards[${createClient.shards}]" + + " replicas[${createClient.replicas}] shardsPerNode[${createClient.shardsPerNode}]" + ) + val request = CreateIndexRequest(index) + .settings( + ESIndexUtils.getIndexSettings( + shards = createClient.shards, + replicas = createClient.replicas, + shardsPerNode = createClient.shardsPerNode + ) + ) + .mapping(ESIndexUtils.getTypeMappings()) + request.setTimeout(TimeValue.timeValueSeconds(SEARCH_TIMEOUT_SECONDS)) + val response = createClient.restClient.indices() + .create(request, RequestOptions.DEFAULT) + response.isShardsAcknowledged + } catch (e: IOException) { + logger.error("BKSystemErrorMonitor|[${createClient.clusterName}] Create index $index failure", e) + return false + } + } + + private fun isExistIndex(index: String): Boolean { + val request = GetIndexRequest(index) + request.setTimeout(TimeValue.timeValueSeconds(SEARCH_TIMEOUT_SECONDS)) + return logClient.hashClient(index).restClient.indices() + .exists(request, RequestOptions.DEFAULT) + } + + private fun genLargeSearchOptions(): RequestOptions { + val builder = RequestOptions.DEFAULT.toBuilder() + builder.setHttpAsyncResponseConsumerFactory( + HeapBufferedResponseConsumerFactory(RESPONSE_ENTITY_MAX_SIZE) + ) + return builder.build() + } + + /* + * 每5分钟执行一次 + * 根据"api"字段进行分组，然后在每个分组内根据"key"字段进行进一步的分组。 + * 在每个"key"分组内，使用"date_histogram"聚合，按照"timestamp"字段的时间间隔（1秒）进行分桶。 + * 最后，在每个时间桶内，使用"value_count"聚合计算"timestamp"值的数量，即并发请求数量。 + */ + fun executeElasticsearchQueryS(keyMap: MutableMap, newDay: Boolean) { + val indexName = IndexNameUtils.getIndexName() + + val searchRequest = SearchRequest(indexName) + .source( + SearchSourceBuilder() + .query( + // 设置查询条件 + QueryBuilders.rangeQuery(ESMessage::timestamp.name) + .from("now-5m") + .to("now") + ) + .aggregation( + // 设置聚合 + AggregationBuilders.terms(ESMessage::api.name).field(ESMessage::api.name) + .subAggregation( + AggregationBuilders.terms(ESMessage::key.name).field(ESMessage::key.name) + .subAggregation( + AggregationBuilders.dateHistogram("concurrency") + .field(ESMessage::timestamp.name) + .calendarInterval(DateHistogramInterval.SECOND) + .minDocCount(1) + .subAggregation( + AggregationBuilders.count("count").field(ESMessage::timestamp.name) + ) + ) + .subAggregation( + MaxBucketPipelineAggregationBuilder( + "max_concurrency", + "concurrency>count" + ) + ) + .subAggregation( + AvgBucketPipelineAggregationBuilder( + "avg_concurrency", + "concurrency>count" + ) + ) + ) + ) + // 不返回任何文档，只返回聚合结果。 + .size(0) + ) + + val response: SearchResponse = logClient.hashClient(indexName) + .restClient + .search(searchRequest, RequestOptions.DEFAULT) + + // 处理聚合结果 + val apiAggregation: Terms = response.aggregations.get(ESMessage::api.name) + + for (apiBucket in apiAggregation.buckets) { + val apiName = apiBucket.keyAsString + val keyAggregation: Terms = apiBucket.aggregations.get(ESMessage::key.name) + + for (keyBucket in keyAggregation.buckets) { + val keyName: String = keyBucket.keyAsString + val max = keyBucket.aggregations.get("max_concurrency").value().toInt() + val avg = keyBucket.aggregations.get("avg_concurrency").value().toInt() + val count = keyBucket.docCount.toInt() + keyMap["$apiName@$keyName"]?.apply { + secondLevelConcurrency = avg + peakConcurrency = if (newDay) max else max.coerceAtLeast(peakConcurrency ?: 0) + call5m = count + } ?: kotlin.run { + keyMap["$apiName@$keyName"] = MetricsApiData( + api = apiName, + key = keyName, + secondLevelConcurrency = avg, + peakConcurrency = max, + call5m = count + ) + } + } + } + } + + /* + * 查询时间间隔为time(1h、1d、7d) + * 根据"api"字段进行分组，然后在每个分组内根据"key"字段进行进一步的分组。 + * 得到每组内的计数 + */ + fun executeElasticsearchQueryM( + keyMap: MutableMap, + time: String, + f: (count: Int, data: MetricsApiData) -> MetricsApiData + ) { + val indexName = IndexNameUtils.getIndexNamePrefix() + + val searchRequest = SearchRequest(indexName) + .source( + SearchSourceBuilder() + .query( + // 设置查询条件 + QueryBuilders.rangeQuery(ESMessage::timestamp.name) + .from("now-$time") + .to("now") + ) + .aggregation( + // 计数 + AggregationBuilders.terms(ESMessage::api.name).field(ESMessage::api.name) + .subAggregation( + AggregationBuilders.terms(ESMessage::key.name).field(ESMessage::key.name) + .subAggregation( + AggregationBuilders.count("count").field(ESMessage::timestamp.name) + ) + ) + ) + // 不返回任何文档，只返回聚合结果。 + .size(0) + ) + + val response: SearchResponse = logClient.hashClient(indexName) + .restClient + .search(searchRequest, RequestOptions.DEFAULT) + + // 处理聚合结果 + val apiAggregation: Terms = response.aggregations.get(ESMessage::api.name) + + for (apiBucket in apiAggregation.buckets) { + val apiName = apiBucket.keyAsString + val keyAggregation: Terms = apiBucket.aggregations.get(ESMessage::key.name) + + for (keyBucket in keyAggregation.buckets) { + val keyName: String = keyBucket.keyAsString + val count = keyBucket.docCount.toInt() + keyMap["$apiName@$keyName"]?.let { + f(count, it) + } ?: kotlin.run { + keyMap["$apiName@$keyName"] = f( + count, MetricsApiData( + api = apiName, + key = keyName + ) + ) + } + } + } + } + + /* + * 查询时间间隔为1h + * 查询api-key-project三重分组 + * 得到每组内的计数 + */ + fun executeElasticsearchQueryP( + keyList: MutableList + ) { + val indexName = IndexNameUtils.getIndexName() + + val searchRequest = SearchRequest(indexName) + .source( + SearchSourceBuilder() + .query( + // 设置查询条件 + QueryBuilders.rangeQuery(ESMessage::timestamp.name) + .from("now-1h") + .to("now") + ) + .aggregation( + // 计数 + AggregationBuilders.terms(ESMessage::api.name).field(ESMessage::api.name) + .subAggregation( + AggregationBuilders.terms(ESMessage::key.name).field(ESMessage::key.name) + .subAggregation( + AggregationBuilders.terms(ESMessage::projectId.name) + .field(ESMessage::projectId.name) + .subAggregation( + AggregationBuilders.count("count").field(ESMessage::timestamp.name) + ) + ) + ) + ) + // 不返回任何文档，只返回聚合结果。 + .size(0) + ) + + val response: SearchResponse = logClient.hashClient(indexName) + .restClient + .search(searchRequest, RequestOptions.DEFAULT) + + // 处理聚合结果 + val apiAggregation: Terms = response.aggregations.get(ESMessage::api.name) + + for (apiBucket in apiAggregation.buckets) { + val apiName = apiBucket.keyAsString + val keyAggregation: Terms = apiBucket.aggregations.get(ESMessage::key.name) + + for (keyBucket in keyAggregation.buckets) { + val keyName: String = keyBucket.keyAsString + val projectAggregation: Terms = keyBucket.aggregations.get(ESMessage::projectId.name) + + for (projectBucket in projectAggregation.buckets) { + val projectName: String = projectBucket.keyAsString + val count = projectBucket.docCount.toInt() + keyList.add( + MetricsProjectData( + api = apiName, + key = keyName, + projectId = projectName, + callHistory = count + ) + ) + } + } + } + } +} diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/mq/ESEvent.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/mq/ESEvent.kt new file mode 100644 index 00000000000..76d37cacfe7 --- /dev/null +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/mq/ESEvent.kt @@ -0,0 +1,48 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.openapi.es.mq + +import com.tencent.devops.common.stream.annotation.StreamEvent +import com.tencent.devops.common.stream.constants.StreamBinding +import com.tencent.devops.common.stream.pojo.IEvent +import com.tencent.devops.openapi.es.ESMessage + +@StreamEvent(StreamBinding.BINDING_OPENAPI_LOG_EVENT_OUT) +data class ESEvent( + val logs: ESMessage, + override var retryTime: Int = 2, + override var delayMills: Int = 0 +) : IEvent(delayMills, retryTime) { + companion object { + private const val DELAY_DURATION_MILLS = 3 * 1000 + } + + fun getNextDelayMills(retryTime: Int): Int { + return DELAY_DURATION_MILLS * (3 - retryTime) + } +} diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/mq/MQDispatcher.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/mq/MQDispatcher.kt new file mode 100644 index 00000000000..b1c7a1b419d --- /dev/null +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/mq/MQDispatcher.kt @@ -0,0 +1,45 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.openapi.es.mq + +import org.slf4j.LoggerFactory +import org.springframework.cloud.stream.function.StreamBridge + +@Suppress("MagicNumber") +class MQDispatcher constructor( + private val streamBridge: StreamBridge +) { + + fun dispatchEvent(event: ESEvent) { + event.sendTo(streamBridge) + } + + companion object { + private val logger = LoggerFactory.getLogger(MQDispatcher::class.java) + } +} diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/mq/MQListenerService.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/mq/MQListenerService.kt new file mode 100644 index 00000000000..57aeabed9e6 --- /dev/null +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/mq/MQListenerService.kt @@ -0,0 +1,64 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.openapi.es.mq + +import com.tencent.devops.openapi.es.IESService +import org.slf4j.LoggerFactory + +class MQListenerService constructor( + private val logService: IESService, + private val dispatcher: MQDispatcher +) { + + fun handleEvent(event: ESEvent) { + var result = false + try { + logService.esAddMessage(event) + result = true + } catch (ignored: Throwable) { + logger.warn("Fail to add the log batch event [${event.logs}|${event.retryTime}]", ignored) + } finally { + if (!result && event.retryTime >= 0) { + logger.warn("Retry to add log batch event [${event.logs}|${event.retryTime}]") + with(event) { + dispatcher.dispatchEvent( + ESEvent( + logs = logs, + retryTime = retryTime - 1, + delayMills = getNextDelayMills(retryTime) + ) + ) + } + } + } + } + + companion object { + private val logger = LoggerFactory.getLogger(MQListenerService::class.java) + } +} diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/resources/META-INF/spring.factories b/src/backend/ci/core/openapi/biz-openapi/src/main/resources/META-INF/spring.factories new file mode 100644 index 00000000000..af0bc69488c --- /dev/null +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/resources/META-INF/spring.factories @@ -0,0 +1,2 @@ +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ +com.tencent.devops.openapi.es.config.ESAutoConfiguration diff --git a/src/backend/ci/settings.gradle.kts b/src/backend/ci/settings.gradle.kts index bbcf02fd0e9..32cb1bef226 100644 --- a/src/backend/ci/settings.gradle.kts +++ b/src/backend/ci/settings.gradle.kts @@ -63,6 +63,7 @@ include(":core:common:common-db-sharding") include(":core:common:common-client") include(":core:common:common-redis") include(":core:common:common-kafka") +include(":core:common:common-es") include(":core:common:common-scm") include(":core:common:common-archive") include(":core:common:common-quality") diff --git a/support-files/sql/1001_ci_openapi_ddl_mysql.sql b/support-files/sql/1001_ci_openapi_ddl_mysql.sql index d50fe0555a8..0937428db21 100644 --- a/support-files/sql/1001_ci_openapi_ddl_mysql.sql +++ b/support-files/sql/1001_ci_openapi_ddl_mysql.sql @@ -40,7 +40,7 @@ CREATE TABLE IF NOT EXISTS `T_APP_CODE_PROJECT` ( -- ---------------------------- -- Table structure for T_APP_USER_INFO -- ---------------------------- -CREATE TABLE IF NOT EXISTS T_APP_USER_INFO( +CREATE TABLE IF NOT EXISTS `T_APP_USER_INFO`( `ID` INT(11) NOT NULL AUTO_INCREMENT COMMENT '主键ID', `APP_CODE` VARCHAR(64) NOT NULL COMMENT 'APP编码', `MANAGER_ID` VARCHAR(64) NOT NULL COMMENT 'APP管理员ID', @@ -52,4 +52,33 @@ CREATE TABLE IF NOT EXISTS T_APP_USER_INFO( UNIQUE INDEX `IDX_APP_USER` (`APP_CODE`, `MANAGER_ID`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='app_code对应的管理员'; +-- ---------------------------- +-- Table structure for T_OPENAPI_METRICS_FOR_API +-- ---------------------------- +CREATE TABLE IF NOT EXISTS `T_OPENAPI_METRICS_FOR_API`( + `API` VARCHAR(64) NOT NULL COMMENT 'api接口代码', + `KEY` VARCHAR(64) NOT NULL COMMENT 'APP编码/api请求用户', + `SECOND_LEVEL_CONCURRENCY` int(11) NOT NULL COMMENT '秒级并发量', + `PEAK_CONCURRENCY` int(11) NOT NULL COMMENT '峰值并发量', + `CALL_5M` int(11) NOT NULL COMMENT '5min调用量', + `CALL_1H` int(11) NOT NULL COMMENT '1h调用量', + `CALL_24H` int(11) NOT NULL COMMENT '24h调用量', + `CALL_7D` int(11) NOT NULL COMMENT '7d调用量', + INDEX `IDX_KEY` (`KEY`), + UNIQUE INDEX `IDX_API_KEY` (`API`, `KEY`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='接口维度度量表'; + +-- ---------------------------- +-- Table structure for T_OPENAPI_METRICS_FOR_PROJECT +-- ---------------------------- +CREATE TABLE IF NOT EXISTS `T_OPENAPI_METRICS_FOR_PROJECT`( + `PROJECT` VARCHAR(64) NOT NULL COMMENT '项目id', + `API` VARCHAR(64) NOT NULL COMMENT 'api接口代码', + `KEY` VARCHAR(64) NOT NULL COMMENT 'APP编码/api请求用户', + `CALL_HISTORY` int(11) NOT NULL COMMENT '历史累计调用', + INDEX `IDX_API` (`API`), + INDEX `IDX_KEY` (`KEY`), + UNIQUE INDEX `IDX_APP_USER` (`PROJECT`, `API`, `KEY`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='项目维度度量表'; + SET FOREIGN_KEY_CHECKS = 1;