From 0c9db438b7fe6f41347ef7f8c4d5bae4142a4442 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?yongyiduan=28=E6=AE=B5=E6=B0=B8=E5=84=84=29?= Date: Mon, 6 Nov 2023 09:43:05 +0800 Subject: [PATCH 01/31] =?UTF-8?q?feat:=20openapi=E6=96=B0=E5=A2=9E?= =?UTF-8?q?=E5=BA=A6=E9=87=8F=E8=83=BD=E5=8A=9B=20#9638?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ci/core/common/common-es/build.gradle.kts | 33 ++ .../devops/common}/es/ESAutoConfiguration.kt | 55 +--- .../com/tencent/devops/common}/es/ESClient.kt | 2 +- .../devops/common/es}/ESConfigUtils.kt | 43 +-- .../tencent/devops/common}/es/ESProperties.kt | 2 +- .../es/NormalX509ExtendedTrustManager.kt | 2 +- .../devops/common/es}/client/LogClient.kt | 4 +- .../common/es}/client/impl/LogClientImpl.kt | 6 +- .../main/resources/META-INF/spring.factories | 2 + .../common/stream/constants/StreamBinding.kt | 4 + .../ci/core/log/biz-log/build.gradle.kts | 4 +- .../log/cron/impl/IndexCleanJobESImpl.kt | 6 +- .../devops/log/es/LogESAutoConfiguration.kt | 81 +++++ .../log/service/impl/LogServiceESImpl.kt | 32 +- .../main/resources/META-INF/spring.factories | 4 +- .../core/openapi/biz-openapi/build.gradle.kts | 2 + .../devops/openapi/aspect/ApiAspect.kt | 44 ++- .../tencent/devops/openapi/es/ESIndexUtils.kt | 74 +++++ .../tencent/devops/openapi/es/ESMessage.kt | 41 +++ .../devops/openapi/es/ESServiceImpl.kt | 286 ++++++++++++++++++ .../devops/openapi/es/IndexNameUtils.kt | 47 +++ .../openapi/es/config/ESAutoConfiguration.kt | 64 ++++ .../openapi/es/config/MQConfiguration.kt | 62 ++++ .../tencent/devops/openapi/es/mq/ESEvent.kt | 48 +++ .../devops/openapi/es/mq/MQDispatcher.kt | 45 +++ .../devops/openapi/es/mq/MQListenerService.kt | 68 +++++ .../main/resources/META-INF/spring.factories | 2 + src/backend/ci/settings.gradle.kts | 1 + .../sql/1001_ci_openapi_ddl_mysql.sql | 35 ++- 29 files changed, 978 insertions(+), 121 deletions(-) create mode 100644 src/backend/ci/core/common/common-es/build.gradle.kts rename src/backend/ci/core/{log/biz-log/src/main/kotlin/com/tencent/devops/log => common/common-es/src/main/kotlin/com/tencent/devops/common}/es/ESAutoConfiguration.kt (84%) rename src/backend/ci/core/{log/biz-log/src/main/kotlin/com/tencent/devops/log => common/common-es/src/main/kotlin/com/tencent/devops/common}/es/ESClient.kt (97%) rename src/backend/ci/core/{log/biz-log/src/main/kotlin/com/tencent/devops/log/util => common/common-es/src/main/kotlin/com/tencent/devops/common/es}/ESConfigUtils.kt (72%) rename src/backend/ci/core/{log/biz-log/src/main/kotlin/com/tencent/devops/log => common/common-es/src/main/kotlin/com/tencent/devops/common}/es/ESProperties.kt (98%) rename src/backend/ci/core/{log/biz-log/src/main/kotlin/com/tencent/devops/log => common/common-es/src/main/kotlin/com/tencent/devops/common}/es/NormalX509ExtendedTrustManager.kt (98%) rename src/backend/ci/core/{log/biz-log/src/main/kotlin/com/tencent/devops/log => common/common-es/src/main/kotlin/com/tencent/devops/common/es}/client/LogClient.kt (94%) rename src/backend/ci/core/{log/biz-log/src/main/kotlin/com/tencent/devops/log => common/common-es/src/main/kotlin/com/tencent/devops/common/es}/client/impl/LogClientImpl.kt (92%) create mode 100644 src/backend/ci/core/common/common-es/src/main/resources/META-INF/spring.factories create mode 100644 src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/es/LogESAutoConfiguration.kt create mode 100644 src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESIndexUtils.kt create mode 100644 src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESMessage.kt create mode 100644 src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESServiceImpl.kt create mode 100644 src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/IndexNameUtils.kt create mode 100644 src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/config/ESAutoConfiguration.kt create mode 100644 src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/config/MQConfiguration.kt create mode 100644 src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/mq/ESEvent.kt create mode 100644 src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/mq/MQDispatcher.kt create mode 100644 src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/mq/MQListenerService.kt create mode 100644 src/backend/ci/core/openapi/biz-openapi/src/main/resources/META-INF/spring.factories diff --git a/src/backend/ci/core/common/common-es/build.gradle.kts b/src/backend/ci/core/common/common-es/build.gradle.kts new file mode 100644 index 00000000000..79c301da8e3 --- /dev/null +++ b/src/backend/ci/core/common/common-es/build.gradle.kts @@ -0,0 +1,33 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +dependencies { + api("org.elasticsearch:elasticsearch") + api("org.elasticsearch.client:elasticsearch-rest-client") + api("org.elasticsearch.client:elasticsearch-rest-high-level-client") + api(project(":core:common:common-web")) +} diff --git a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/es/ESAutoConfiguration.kt b/src/backend/ci/core/common/common-es/src/main/kotlin/com/tencent/devops/common/es/ESAutoConfiguration.kt similarity index 84% rename from src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/es/ESAutoConfiguration.kt rename to src/backend/ci/core/common/common-es/src/main/kotlin/com/tencent/devops/common/es/ESAutoConfiguration.kt index ae6d4ea3812..4cdc5379301 100644 --- a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/es/ESAutoConfiguration.kt +++ b/src/backend/ci/core/common/common-es/src/main/kotlin/com/tencent/devops/common/es/ESAutoConfiguration.kt @@ -25,21 +25,15 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.devops.log.es +package com.tencent.devops.common.es -import com.tencent.devops.common.redis.RedisOperation +import com.tencent.devops.common.es.client.LogClient +import com.tencent.devops.common.es.client.impl.LogClientImpl import com.tencent.devops.common.web.WebAutoConfiguration -import com.tencent.devops.log.client.LogClient -import com.tencent.devops.log.client.impl.LogClientImpl -import com.tencent.devops.log.jmx.CreateIndexBean -import com.tencent.devops.log.jmx.LogStorageBean -import com.tencent.devops.log.service.BuildLogPrintService -import com.tencent.devops.log.service.IndexService -import com.tencent.devops.log.service.LogService -import com.tencent.devops.log.service.LogStatusService -import com.tencent.devops.log.service.LogTagService -import com.tencent.devops.log.service.impl.LogServiceESImpl -import com.tencent.devops.log.util.ESConfigUtils +import java.io.File +import java.io.FileInputStream +import java.security.KeyStore +import javax.net.ssl.SSLContext import org.apache.http.auth.AuthScope import org.apache.http.auth.UsernamePasswordCredentials import org.apache.http.impl.client.BasicCredentialsProvider @@ -57,10 +51,6 @@ import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration import org.springframework.context.annotation.Primary import org.springframework.core.Ordered -import java.io.File -import java.io.FileInputStream -import java.security.KeyStore -import javax.net.ssl.SSLContext @Suppress("ALL") @Configuration @@ -71,6 +61,7 @@ import javax.net.ssl.SSLContext class ESAutoConfiguration : DisposableBean { @Value("\${log.elasticsearch.ip}") private val host: String? = null + @Value("\${log.elasticsearch.port}") private val port: Int? = null @@ -221,30 +212,6 @@ class ESAutoConfiguration : DisposableBean { ) } - @Bean - fun esLogService( - @Autowired logESClient: LogClient, - @Autowired indexService: IndexService, - @Autowired logStatusService: LogStatusService, - @Autowired logTagService: LogTagService, - @Autowired defaultKeywords: List, - @Autowired createIndexBean: CreateIndexBean, - @Autowired logStorageBean: LogStorageBean, - @Autowired redisOperation: RedisOperation, - @Autowired buildLogPrintService: BuildLogPrintService - ): LogService { - return LogServiceESImpl( - logClient = logESClient, - indexService = indexService, - logStatusService = logStatusService, - logTagService = logTagService, - logStorageBean = logStorageBean, - createIndexBean = createIndexBean, - buildLogPrintService = buildLogPrintService, - redisOperation = redisOperation - ) - } - @Bean @ConditionalOnMissingBean fun logClient(@Autowired transportClient: ESClient): LogClient = @@ -264,8 +231,8 @@ class ESAutoConfiguration : DisposableBean { private fun hasCertificateConfig(): Boolean { return !keystoreFilePath.isNullOrBlank() || - !truststoreFilePath.isNullOrBlank() || - !keystorePassword.isNullOrBlank() || - !truststorePassword.isNullOrBlank() + !truststoreFilePath.isNullOrBlank() || + !keystorePassword.isNullOrBlank() || + !truststorePassword.isNullOrBlank() } } diff --git a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/es/ESClient.kt b/src/backend/ci/core/common/common-es/src/main/kotlin/com/tencent/devops/common/es/ESClient.kt similarity index 97% rename from src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/es/ESClient.kt rename to src/backend/ci/core/common/common-es/src/main/kotlin/com/tencent/devops/common/es/ESClient.kt index 8c9a0e41751..2c37d5fcd72 100644 --- a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/es/ESClient.kt +++ b/src/backend/ci/core/common/common-es/src/main/kotlin/com/tencent/devops/common/es/ESClient.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.devops.log.es +package com.tencent.devops.common.es import org.elasticsearch.client.RestHighLevelClient diff --git a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/util/ESConfigUtils.kt b/src/backend/ci/core/common/common-es/src/main/kotlin/com/tencent/devops/common/es/ESConfigUtils.kt similarity index 72% rename from src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/util/ESConfigUtils.kt rename to src/backend/ci/core/common/common-es/src/main/kotlin/com/tencent/devops/common/es/ESConfigUtils.kt index 73d9ca4a9dc..2f441cd7393 100644 --- a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/util/ESConfigUtils.kt +++ b/src/backend/ci/core/common/common-es/src/main/kotlin/com/tencent/devops/common/es/ESConfigUtils.kt @@ -25,14 +25,8 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.devops.log.util +package com.tencent.devops.common.es -import com.tencent.devops.common.log.constant.LogMessageCode.FILE_NOT_FOUND_CHECK_PATH -import com.tencent.devops.common.web.utils.I18nUtil -import com.tencent.devops.log.es.NormalX509ExtendedTrustManager -import java.io.File -import java.io.FileInputStream -import java.security.KeyStore import java.security.SecureRandom import javax.net.ssl.SSLContext import org.apache.http.HeaderElementIterator @@ -42,7 +36,6 @@ import org.apache.http.client.CredentialsProvider import org.apache.http.message.BasicHeaderElementIterator import org.apache.http.protocol.HTTP import org.apache.http.protocol.HttpContext -import org.apache.http.ssl.SSLContexts import org.elasticsearch.client.RestClient import org.elasticsearch.client.RestClientBuilder import org.slf4j.LoggerFactory @@ -110,39 +103,5 @@ object ESConfigUtils { } } - fun getSSLContext( - keystoreFilePath: String, - truststoreFilePath: String, - keystorePassword: String, - truststorePassword: String - ): SSLContext { - val keystoreFile = File(keystoreFilePath) - if (!keystoreFile.exists()) { - throw IllegalArgumentException( - I18nUtil.getCodeLanMessage(messageCode = FILE_NOT_FOUND_CHECK_PATH, params = arrayOf("keystore")) + - keystoreFilePath - ) - } - val truststoreFile = File(truststoreFilePath) - if (!truststoreFile.exists()) { - throw IllegalArgumentException( - I18nUtil.getCodeLanMessage( - messageCode = FILE_NOT_FOUND_CHECK_PATH, - params = arrayOf("truststore") - ) + truststoreFilePath - ) - } - val keyStore = KeyStore.getInstance(KeyStore.getDefaultType()) - val keystorePasswordCharArray = keystorePassword.toCharArray() - keyStore.load(FileInputStream(keystoreFile), keystorePasswordCharArray) - val truststore = KeyStore.getInstance(KeyStore.getDefaultType()) - val truststorePasswordCharArray = truststorePassword.toCharArray() - truststore.load(FileInputStream(truststoreFile), truststorePasswordCharArray) - return SSLContexts.custom() - .loadTrustMaterial(truststore, null) - .loadKeyMaterial(keyStore, keystorePasswordCharArray) - .build() - } - private val logger = LoggerFactory.getLogger(ESConfigUtils::class.java) } diff --git a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/es/ESProperties.kt b/src/backend/ci/core/common/common-es/src/main/kotlin/com/tencent/devops/common/es/ESProperties.kt similarity index 98% rename from src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/es/ESProperties.kt rename to src/backend/ci/core/common/common-es/src/main/kotlin/com/tencent/devops/common/es/ESProperties.kt index cbc7039b168..861bc65d103 100644 --- a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/es/ESProperties.kt +++ b/src/backend/ci/core/common/common-es/src/main/kotlin/com/tencent/devops/common/es/ESProperties.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.devops.log.es +package com.tencent.devops.common.es import org.springframework.boot.context.properties.ConfigurationProperties import org.springframework.boot.context.properties.ConstructorBinding diff --git a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/es/NormalX509ExtendedTrustManager.kt b/src/backend/ci/core/common/common-es/src/main/kotlin/com/tencent/devops/common/es/NormalX509ExtendedTrustManager.kt similarity index 98% rename from src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/es/NormalX509ExtendedTrustManager.kt rename to src/backend/ci/core/common/common-es/src/main/kotlin/com/tencent/devops/common/es/NormalX509ExtendedTrustManager.kt index aad2f92312d..0b037ebb84c 100644 --- a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/es/NormalX509ExtendedTrustManager.kt +++ b/src/backend/ci/core/common/common-es/src/main/kotlin/com/tencent/devops/common/es/NormalX509ExtendedTrustManager.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.devops.log.es +package com.tencent.devops.common.es import java.net.Socket import javax.net.ssl.SSLEngine diff --git a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/client/LogClient.kt b/src/backend/ci/core/common/common-es/src/main/kotlin/com/tencent/devops/common/es/client/LogClient.kt similarity index 94% rename from src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/client/LogClient.kt rename to src/backend/ci/core/common/common-es/src/main/kotlin/com/tencent/devops/common/es/client/LogClient.kt index fb1de1be49b..ea6e8a422d5 100644 --- a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/client/LogClient.kt +++ b/src/backend/ci/core/common/common-es/src/main/kotlin/com/tencent/devops/common/es/client/LogClient.kt @@ -25,9 +25,9 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.devops.log.client +package com.tencent.devops.common.es.client -import com.tencent.devops.log.es.ESClient +import com.tencent.devops.common.es.ESClient interface LogClient { diff --git a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/client/impl/LogClientImpl.kt b/src/backend/ci/core/common/common-es/src/main/kotlin/com/tencent/devops/common/es/client/impl/LogClientImpl.kt similarity index 92% rename from src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/client/impl/LogClientImpl.kt rename to src/backend/ci/core/common/common-es/src/main/kotlin/com/tencent/devops/common/es/client/impl/LogClientImpl.kt index c4db85f7816..48ab81310a8 100644 --- a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/client/impl/LogClientImpl.kt +++ b/src/backend/ci/core/common/common-es/src/main/kotlin/com/tencent/devops/common/es/client/impl/LogClientImpl.kt @@ -25,10 +25,10 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.devops.log.client.impl +package com.tencent.devops.common.es.client.impl -import com.tencent.devops.log.client.LogClient -import com.tencent.devops.log.es.ESClient +import com.tencent.devops.common.es.client.LogClient +import com.tencent.devops.common.es.ESClient class LogClientImpl constructor(private val client: ESClient) : LogClient { diff --git a/src/backend/ci/core/common/common-es/src/main/resources/META-INF/spring.factories b/src/backend/ci/core/common/common-es/src/main/resources/META-INF/spring.factories new file mode 100644 index 00000000000..4d64e749f1d --- /dev/null +++ b/src/backend/ci/core/common/common-es/src/main/resources/META-INF/spring.factories @@ -0,0 +1,2 @@ +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ +com.tencent.devops.common.es.ESAutoConfiguration diff --git a/src/backend/ci/core/common/common-stream/src/main/kotlin/com/tencent/devops/common/stream/constants/StreamBinding.kt b/src/backend/ci/core/common/common-stream/src/main/kotlin/com/tencent/devops/common/stream/constants/StreamBinding.kt index ec76d5da014..9ccd6d76617 100644 --- a/src/backend/ci/core/common/common-stream/src/main/kotlin/com/tencent/devops/common/stream/constants/StreamBinding.kt +++ b/src/backend/ci/core/common/common-stream/src/main/kotlin/com/tencent/devops/common/stream/constants/StreamBinding.kt @@ -39,4 +39,8 @@ object StreamBinding { // 日志构建状态事件 const val BINDING_LOG_STATUS_EVENT_IN = "logStatusEventIn" const val BINDING_LOG_STATUS_EVENT_OUT = "logStatusEventOut" + + // openapi审计日志预处理事件 + const val BINDING_OPENAPI_LOG_EVENT_IN = "logOriginEventIn" + const val BINDING_OPENAPI_LOG_EVENT_OUT = "logOriginEventOut" } diff --git a/src/backend/ci/core/log/biz-log/build.gradle.kts b/src/backend/ci/core/log/biz-log/build.gradle.kts index e7277c4a259..927b388b331 100644 --- a/src/backend/ci/core/log/biz-log/build.gradle.kts +++ b/src/backend/ci/core/log/biz-log/build.gradle.kts @@ -33,11 +33,9 @@ dependencies { api(project(":core:common:common-auth:common-auth-api")) api(project(":core:common:common-db")) api(project(":core:common:common-stream")) + api(project(":core:common:common-es")) api(project(":core:auth:api-auth")) - api("org.elasticsearch:elasticsearch") - api("org.elasticsearch.client:elasticsearch-rest-client") - api("org.elasticsearch.client:elasticsearch-rest-high-level-client") api("org.apache.logging.log4j:log4j-core") api("org.apache.logging.log4j:log4j-api") api("com.github.ben-manes.caffeine:caffeine") diff --git a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/cron/impl/IndexCleanJobESImpl.kt b/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/cron/impl/IndexCleanJobESImpl.kt index 2507aeb8c39..28dfbb71c8d 100644 --- a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/cron/impl/IndexCleanJobESImpl.kt +++ b/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/cron/impl/IndexCleanJobESImpl.kt @@ -27,12 +27,14 @@ package com.tencent.devops.log.cron.impl +import com.tencent.devops.common.es.client.LogClient import com.tencent.devops.common.redis.RedisLock import com.tencent.devops.common.redis.RedisOperation -import com.tencent.devops.log.client.LogClient import com.tencent.devops.log.configuration.StorageProperties import com.tencent.devops.log.cron.IndexCleanJob import com.tencent.devops.log.util.IndexNameUtils.LOG_INDEX_PREFIX +import java.time.LocalDateTime +import java.time.temporal.ChronoUnit import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest import org.elasticsearch.client.RequestOptions @@ -44,8 +46,6 @@ import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty import org.springframework.scheduling.annotation.Scheduled import org.springframework.stereotype.Component -import java.time.LocalDateTime -import java.time.temporal.ChronoUnit @Suppress("MagicNumber") @Component diff --git a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/es/LogESAutoConfiguration.kt b/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/es/LogESAutoConfiguration.kt new file mode 100644 index 00000000000..4f896bb099f --- /dev/null +++ b/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/es/LogESAutoConfiguration.kt @@ -0,0 +1,81 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.log.es + +import com.tencent.devops.common.es.ESAutoConfiguration +import com.tencent.devops.common.es.ESProperties +import com.tencent.devops.common.es.client.LogClient +import com.tencent.devops.common.redis.RedisOperation +import com.tencent.devops.log.jmx.CreateIndexBean +import com.tencent.devops.log.jmx.LogStorageBean +import com.tencent.devops.log.service.BuildLogPrintService +import com.tencent.devops.log.service.IndexService +import com.tencent.devops.log.service.LogService +import com.tencent.devops.log.service.LogStatusService +import com.tencent.devops.log.service.LogTagService +import com.tencent.devops.log.service.impl.LogServiceESImpl +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.autoconfigure.AutoConfigureAfter +import org.springframework.boot.autoconfigure.AutoConfigureOrder +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty +import org.springframework.boot.context.properties.EnableConfigurationProperties +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Configuration +import org.springframework.core.Ordered + +@Suppress("ALL") +@Configuration +@ConditionalOnProperty(prefix = "log.storage", name = ["type"], havingValue = "elasticsearch") +@AutoConfigureOrder(Ordered.HIGHEST_PRECEDENCE) +@AutoConfigureAfter(ESAutoConfiguration::class) +@EnableConfigurationProperties(ESProperties::class) +class LogESAutoConfiguration { + @Bean + fun esLogService( + @Autowired logESClient: LogClient, + @Autowired indexService: IndexService, + @Autowired logStatusService: LogStatusService, + @Autowired logTagService: LogTagService, + @Autowired defaultKeywords: List, + @Autowired createIndexBean: CreateIndexBean, + @Autowired logStorageBean: LogStorageBean, + @Autowired redisOperation: RedisOperation, + @Autowired buildLogPrintService: BuildLogPrintService + ): LogService { + return LogServiceESImpl( + logClient = logESClient, + indexService = indexService, + logStatusService = logStatusService, + logTagService = logTagService, + logStorageBean = logStorageBean, + createIndexBean = createIndexBean, + buildLogPrintService = buildLogPrintService, + redisOperation = redisOperation + ) + } +} diff --git a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/service/impl/LogServiceESImpl.kt b/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/service/impl/LogServiceESImpl.kt index d20e3cc680f..a984c331bef 100644 --- a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/service/impl/LogServiceESImpl.kt +++ b/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/service/impl/LogServiceESImpl.kt @@ -30,6 +30,7 @@ package com.tencent.devops.log.service.impl import com.github.benmanes.caffeine.cache.Caffeine import com.tencent.devops.common.api.exception.ExecuteException import com.tencent.devops.common.api.pojo.Page +import com.tencent.devops.common.es.client.LogClient import com.tencent.devops.common.log.constant.LogMessageCode.LOG_INDEX_HAS_BEEN_CLEANED import com.tencent.devops.common.log.pojo.EndPageQueryLogs import com.tencent.devops.common.log.pojo.LogLine @@ -42,8 +43,7 @@ import com.tencent.devops.common.log.pojo.message.LogMessageWithLineNo import com.tencent.devops.common.redis.RedisLock import com.tencent.devops.common.redis.RedisOperation import com.tencent.devops.common.web.utils.I18nUtil -import com.tencent.devops.log.client.LogClient -import com.tencent.devops.log.es.ESClient +import com.tencent.devops.common.es.ESClient import com.tencent.devops.log.event.LogOriginEvent import com.tencent.devops.log.event.LogStatusEvent import com.tencent.devops.log.event.LogStorageEvent @@ -57,6 +57,14 @@ import com.tencent.devops.log.service.LogTagService import com.tencent.devops.log.util.Constants import com.tencent.devops.log.util.ESIndexUtils import com.tencent.devops.log.util.IndexNameUtils +import java.io.IOException +import java.sql.Date +import java.text.SimpleDateFormat +import java.util.concurrent.TimeUnit +import javax.ws.rs.core.MediaType +import javax.ws.rs.core.Response +import javax.ws.rs.core.StreamingOutput +import kotlin.math.ceil import org.elasticsearch.ElasticsearchStatusException import org.elasticsearch.action.admin.indices.open.OpenIndexRequest import org.elasticsearch.action.bulk.BulkRequest @@ -77,14 +85,6 @@ import org.elasticsearch.search.builder.SearchSourceBuilder import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder import org.elasticsearch.search.sort.SortOrder import org.slf4j.LoggerFactory -import java.io.IOException -import java.sql.Date -import java.text.SimpleDateFormat -import java.util.concurrent.TimeUnit -import javax.ws.rs.core.MediaType -import javax.ws.rs.core.Response -import javax.ws.rs.core.StreamingOutput -import kotlin.math.ceil @Suppress( "LongParameterList", @@ -143,7 +143,7 @@ class LogServiceESImpl constructor( if (doAddMultiLines(buf, event.buildId) == 0) { throw ExecuteException( "None of lines is inserted successfully to ES " + - "[${event.buildId}|${event.retryTime}]" + "[${event.buildId}|${event.retryTime}]" ) } else { buf.clear() @@ -684,7 +684,7 @@ class LogServiceESImpl constructor( ) logger.info( "[$index|$buildId|$tag|$subTag|$jobId|$executeCount] " + - "doQueryInitLogs get the query builder: $boolQueryBuilder" + "doQueryInitLogs get the query builder: $boolQueryBuilder" ) val searchRequest = SearchRequest(index) @@ -753,7 +753,7 @@ class LogServiceESImpl constructor( logger.info( "[$index|$buildId|$tag|$subTag|$jobId|$executeCount] " + - "doQueryLogsAfterLine get the query builder: $boolQueryBuilder" + "doQueryLogsAfterLine get the query builder: $boolQueryBuilder" ) val searchRequest = SearchRequest(index) .source( @@ -840,9 +840,11 @@ class LogServiceESImpl constructor( end >= size -> { end - size } + logSize >= size -> { logSize - size } + else -> { 0 } @@ -859,7 +861,7 @@ class LogServiceESImpl constructor( .must(QueryBuilders.rangeQuery("lineNo").lte(end)) logger.info( "[$index|$buildId|$tag|$subTag|$jobId|$executeCount] " + - "doQueryLogsBeforeLine get the query builder: $boolQueryBuilder" + "doQueryLogsBeforeLine get the query builder: $boolQueryBuilder" ) val searchRequest = SearchRequest(index) .source( @@ -1136,7 +1138,7 @@ class LogServiceESImpl constructor( return try { logger.info( "[${createClient.clusterName}][$index]|createIndex|: shards[${createClient.shards}]" + - " replicas[${createClient.replicas}] shardsPerNode[${createClient.shardsPerNode}]" + " replicas[${createClient.replicas}] shardsPerNode[${createClient.shardsPerNode}]" ) val request = CreateIndexRequest(index) .settings( diff --git a/src/backend/ci/core/log/biz-log/src/main/resources/META-INF/spring.factories b/src/backend/ci/core/log/biz-log/src/main/resources/META-INF/spring.factories index fc65d3b15bb..c5a4f1375f7 100644 --- a/src/backend/ci/core/log/biz-log/src/main/resources/META-INF/spring.factories +++ b/src/backend/ci/core/log/biz-log/src/main/resources/META-INF/spring.factories @@ -1,3 +1,3 @@ org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ -com.tencent.devops.log.es.ESAutoConfiguration,\ -com.tencent.devops.log.lucene.LuceneAutoConfiguration \ No newline at end of file +com.tencent.devops.log.es.LogESAutoConfiguration,\ +com.tencent.devops.log.lucene.LuceneAutoConfiguration diff --git a/src/backend/ci/core/openapi/biz-openapi/build.gradle.kts b/src/backend/ci/core/openapi/biz-openapi/build.gradle.kts index 2c058459137..3a54077df65 100644 --- a/src/backend/ci/core/openapi/biz-openapi/build.gradle.kts +++ b/src/backend/ci/core/openapi/biz-openapi/build.gradle.kts @@ -30,6 +30,8 @@ dependencies { api(project(":core:openapi:api-openapi")) api(project(":core:common:common-db")) api(project(":core:common:common-pipeline")) + api(project(":core:common:common-es")) + api(project(":core:common:common-stream")) api(project(":core:repository:api-repository")) api(project(":core:environment:api-environment")) api(project(":core:artifactory:api-artifactory")) diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/aspect/ApiAspect.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/aspect/ApiAspect.kt index 4f2dc590645..ca0d8f6a432 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/aspect/ApiAspect.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/aspect/ApiAspect.kt @@ -26,6 +26,7 @@ */ package com.tencent.devops.openapi.aspect +import com.github.benmanes.caffeine.cache.Caffeine import com.tencent.devops.common.api.constant.HTTP_500 import com.tencent.devops.common.api.exception.CustomException import com.tencent.devops.common.api.exception.ParamBlankException @@ -37,9 +38,14 @@ import com.tencent.devops.common.service.BkTag import com.tencent.devops.common.web.utils.I18nUtil import com.tencent.devops.openapi.IgnoreProjectId import com.tencent.devops.openapi.constant.OpenAPIMessageCode.PARAM_VERIFY_FAIL +import com.tencent.devops.openapi.es.ESMessage +import com.tencent.devops.openapi.es.ESServiceImpl import com.tencent.devops.openapi.service.OpenapiPermissionService import com.tencent.devops.openapi.service.op.AppCodeService import com.tencent.devops.openapi.utils.ApiGatewayUtil +import io.swagger.annotations.ApiOperation +import javax.ws.rs.core.Response +import kotlin.reflect.jvm.kotlinFunction import org.aspectj.lang.JoinPoint import org.aspectj.lang.ProceedingJoinPoint import org.aspectj.lang.annotation.Around @@ -48,8 +54,6 @@ import org.aspectj.lang.reflect.MethodSignature import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Value import org.springframework.stereotype.Component -import javax.ws.rs.core.Response -import kotlin.reflect.jvm.kotlinFunction @Aspect @Component @@ -58,7 +62,8 @@ class ApiAspect( private val apiGatewayUtil: ApiGatewayUtil, private val redisOperation: RedisOperation, private val bkTag: BkTag, - private val permissionService: OpenapiPermissionService + private val permissionService: OpenapiPermissionService, + private val esServiceImpl: ESServiceImpl ) { companion object { @@ -68,6 +73,10 @@ class ApiAspect( @Value("\${openapi.verify.project: #{null}}") val verifyProjectFlag: String = "false" + private val apiTagCache = Caffeine.newBuilder() + .maximumSize(500) + .build() + /** * 前置增强：目标方法执行之前执行 * @@ -100,6 +109,17 @@ class ApiAspect( } } + esServiceImpl.addMessage( + ESMessage( + api = getApiTag(jp = jp, apiType = apigwType?.split("-")?.getOrNull(1) ?: ""), + apiType = apigwType ?: "", + appCode = appCode ?: "", + userId = userId ?: "", + projectId = projectId ?: "", + timestamp = System.currentTimeMillis() + ) + ) + if (logger.isDebugEnabled) { val methodName: String = jp.signature.name @@ -223,4 +243,22 @@ class ApiAspect( // 删除线程ThreadLocal数据,防止线程池复用。导致流量指向被污染 bkTag.removeGatewayTag() } + + private fun getApiTag(jp: JoinPoint, apiType: String): String { + val method = (jp.signature as MethodSignature).method + val methodName = method.declaringClass.typeName + "." + method.name + return apiTagCache.get(methodName) { + jp.target + ?.javaClass + ?.interfaces + ?.first() + ?.getDeclaredMethod( + method.name + ) + ?.getAnnotation(ApiOperation::class.java) + ?.tags + ?.first() + ?.replace(Regex("app|user"), apiType) ?: methodName + } ?: methodName + } } diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESIndexUtils.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESIndexUtils.kt new file mode 100644 index 00000000000..c7a6a30a896 --- /dev/null +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESIndexUtils.kt @@ -0,0 +1,74 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.openapi.es + +import org.elasticsearch.common.settings.Settings +import org.elasticsearch.xcontent.XContentBuilder +import org.elasticsearch.xcontent.XContentFactory + +object ESIndexUtils { + + fun getIndexSettings(shards: Int, replicas: Int, shardsPerNode: Int): Settings.Builder { + return Settings.builder() + .put("index.number_of_shards", shards) + .put("index.number_of_replicas", replicas) + .put("index.refresh_interval", "3s") + .put("index.queries.cache.enabled", false) + .put("index.routing.allocation.total_shards_per_node", shardsPerNode) + } + + fun getTypeMappings(): XContentBuilder { + return XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject("api").field("type", "keyword").endObject() + .startObject("apiType").field("type", "keyword").endObject() + .startObject("timestamp").field("type", "long").endObject() + .startObject("appCode").field("type", "keyword").endObject() + .startObject("userId").field("type", "keyword").endObject() + .startObject("projectId").field("type", "keyword").endObject() + .field("analyzer", "standard") + .endObject() + .endObject() + .endObject() + } + + fun getDocumentObject( + logMessage: ESMessage + ): XContentBuilder { + return XContentFactory.jsonBuilder() + .startObject() + .field("api", logMessage.api) + .field("apiType", logMessage.apiType) + .field("appCode", logMessage.appCode) + .field("userId", logMessage.userId) + .field("projectId", logMessage.projectId) + .field("timestamp", logMessage.timestamp) + .endObject() + } +} diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESMessage.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESMessage.kt new file mode 100644 index 00000000000..41110b922bd --- /dev/null +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESMessage.kt @@ -0,0 +1,41 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.openapi.es + +/** + * + * Powered By Tencent + */ +data class ESMessage( + val api: String, + val apiType: String = "", + val appCode: String = "", + val userId: String = "", + val projectId: String = "", + var timestamp: Long = 0 +) diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESServiceImpl.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESServiceImpl.kt new file mode 100644 index 00000000000..5c9de82b851 --- /dev/null +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESServiceImpl.kt @@ -0,0 +1,286 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.openapi.es + +import com.github.benmanes.caffeine.cache.Caffeine +import com.tencent.devops.common.api.exception.ExecuteException +import com.tencent.devops.common.es.ESClient +import com.tencent.devops.common.es.client.LogClient +import com.tencent.devops.common.redis.RedisLock +import com.tencent.devops.common.redis.RedisOperation +import com.tencent.devops.openapi.es.mq.ESEvent +import com.tencent.devops.openapi.es.mq.MQDispatcher +import java.io.IOException +import java.util.concurrent.ArrayBlockingQueue +import java.util.concurrent.BlockingQueue +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit +import org.elasticsearch.action.bulk.BulkRequest +import org.elasticsearch.action.index.IndexRequest +import org.elasticsearch.client.HttpAsyncResponseConsumerFactory.HeapBufferedResponseConsumerFactory +import org.elasticsearch.client.RequestOptions +import org.elasticsearch.client.indices.CreateIndexRequest +import org.elasticsearch.client.indices.GetIndexRequest +import org.elasticsearch.core.TimeValue +import org.slf4j.LoggerFactory + +class ESServiceImpl constructor( + private val logClient: LogClient, + private val redisOperation: RedisOperation, + private val dispatcher: MQDispatcher +) { + + companion object { + private val logger = LoggerFactory.getLogger(ESServiceImpl::class.java) + private const val LONG_SEARCH_TIME: Long = 64000 + private const val SHORT_SEARCH_TIME: Long = 32000 + private const val SEARCH_TIMEOUT_SECONDS = 60L + private const val SEARCH_FRAGMENT_SIZE = 100000 + private const val INDEX_CACHE_MAX_SIZE = 10L + private const val INDEX_CACHE_EXPIRE_DAY = 1L + private const val INDEX_LOCK_EXPIRE_SECONDS = 10L + private const val INDEX_STORAGE_WARN_MILLIS = 1000 + private const val BULK_BUFFER_SIZE = 100 + private const val maxQueueSize = 9000 // 队列溢出的最大大小 + private const val RESPONSE_ENTITY_MAX_SIZE = 1024 * 1024 * 1024 + } + + private val queue: BlockingQueue = ArrayBlockingQueue(10000) // 创建一个容量为10000的阻塞队列 + + private val indexCache = Caffeine.newBuilder() + .maximumSize(INDEX_CACHE_MAX_SIZE) + .expireAfterAccess(INDEX_CACHE_EXPIRE_DAY, TimeUnit.DAYS) + .build() + + private val executor = Executors.newCachedThreadPool() + + init { + executor.submit(BulkSend()) + } + + fun addMessage(message: ESMessage) { + if (queue.size >= maxQueueSize) { + dispatcher.dispatchEvent(ESEvent(message)) // 将消息推送到es + } else { + queue.put(message) // 将消息放入队列，如果队列已满则阻塞等待 + } + } + + fun esAddMessage(event: ESEvent) { + queue.put(event.logs) + } + + private inner class BulkSend : Runnable { + val buf = mutableListOf() + override fun run() { + while (true) { + val message = queue.poll() ?: continue + buf.add(message) + if (buf.size == BULK_BUFFER_SIZE) { + val currentEpoch = System.currentTimeMillis() + try { + prepareIndex() + if (doAddMultiLines(buf) == 0) { + throw ExecuteException( + "None of lines is inserted successfully to ES " + ) + } else { + buf.clear() + } + } finally { + val elapse = System.currentTimeMillis() - currentEpoch + // #4265 当日志消息处理时间过长时打印消息内容 + if (elapse >= INDEX_STORAGE_WARN_MILLIS) logger.warn( + " addBatchLogEvent spent too much time($elapse)" + ) + } + } + } + } + } + + private fun doAddMultiLines(logMessages: List): Int { + val currentEpoch = System.currentTimeMillis() + val index = IndexNameUtils.getIndexName() + val bulkClient = logClient.hashClient(index) + var lines = 0 + var bulkLines = 0 + val bulkRequest = BulkRequest() + .timeout(TimeValue.timeValueMillis(bulkClient.requestTimeout)) + for (i in logMessages.indices) { + val logMessage = logMessages[i] + + val indexRequest = genIndexRequest( + logMessage = logMessage, + index = index + ) + if (indexRequest != null) { + bulkRequest.add(indexRequest) + lines++ + } + } + try { + val bulkResponse = bulkClient.restClient.bulk(bulkRequest, RequestOptions.DEFAULT) + bulkLines = bulkResponse.count() + return if (bulkResponse.hasFailures()) { + throw ExecuteException(bulkResponse.buildFailureMessage()) + } else { + bulkLines + } + } catch (ignore: Exception) { + val exString = ignore.toString() + if (exString.contains("circuit_breaking_exception")) { + logger.warn( + "Add bulk lines failed|$exString, attempting to add index. [$logMessages]", + ignore + ) + val bulkResponse = bulkClient.restClient.bulk( + bulkRequest.timeout(TimeValue.timeValueSeconds(SEARCH_TIMEOUT_SECONDS)), + genLargeSearchOptions() + ) + bulkLines = bulkResponse.count() + return if (bulkResponse.hasFailures()) { + logger.error(bulkResponse.buildFailureMessage()) + 0 + } else { + bulkLines + } + } else { + logger.warn("Add bulk lines failed because of unknown Exception. [$logMessages]", ignore) + throw ignore + } + } finally { + if (bulkLines != lines) { + logger.warn("Part of bulk lines failed, lines:$lines, bulkLines:$bulkLines") + } + val elapse = System.currentTimeMillis() - currentEpoch + + // #4265 当日志消息处理时间过长时打印消息内容 + if (elapse >= INDEX_STORAGE_WARN_MILLIS && logMessages.isNotEmpty()) logger.warn( + "doAddMultiLines spent too much time($elapse) with tag=${logMessages.first()}" + ) + } + } + + private fun genIndexRequest( + logMessage: ESMessage, + index: String + ): IndexRequest? { + val builder = ESIndexUtils.getDocumentObject(logMessage) + return try { + IndexRequest(index).source(builder) + } catch (e: IOException) { + logger.error("Convert logMessage to es document failure", e) + null + } finally { + builder.close() + } + } + + private fun prepareIndex(): Boolean { + val index = IndexNameUtils.getIndexName() + return if (!checkIndexCreate(index)) { + createIndex(index) + indexCache.put(index, true) + true + } else { + false + } + } + + private fun checkIndexCreate(index: String): Boolean { + if (indexCache.getIfPresent(index) == true) { + return true + } + val redisLock = RedisLock(redisOperation, "LOG:index:create:lock:key:$index", INDEX_LOCK_EXPIRE_SECONDS) + try { + redisLock.lock() + if (indexCache.getIfPresent(index) == true) { + return true + } + + // Check from ES + if (isExistIndex(index)) { + logger.info("[$index] the index is already created") + indexCache.put(index, true) + return true + } + return false + } finally { + redisLock.unlock() + } + } + + private fun createIndex(index: String): Boolean { + val createClient = logClient.hashClient(index) + // 提前创建第二天的索引备用 + createESIndex(createClient, IndexNameUtils.getNextIndexName()) + return createESIndex(createClient, index) + } + + private fun createESIndex(createClient: ESClient, index: String): Boolean { + logger.info("[$index] Create index") + return try { + logger.info( + "[${createClient.clusterName}][$index]|createIndex|: shards[${createClient.shards}]" + + " replicas[${createClient.replicas}] shardsPerNode[${createClient.shardsPerNode}]" + ) + val request = CreateIndexRequest(index) + .settings( + ESIndexUtils.getIndexSettings( + shards = createClient.shards, + replicas = createClient.replicas, + shardsPerNode = createClient.shardsPerNode + ) + ) + .mapping(ESIndexUtils.getTypeMappings()) + request.setTimeout(TimeValue.timeValueSeconds(SEARCH_TIMEOUT_SECONDS)) + val response = createClient.restClient.indices() + .create(request, RequestOptions.DEFAULT) + response.isShardsAcknowledged + } catch (e: IOException) { + logger.error("BKSystemErrorMonitor|[${createClient.clusterName}] Create index $index failure", e) + return false + } + } + + private fun isExistIndex(index: String): Boolean { + val request = GetIndexRequest(index) + request.setTimeout(TimeValue.timeValueSeconds(SEARCH_TIMEOUT_SECONDS)) + return logClient.hashClient(index).restClient.indices() + .exists(request, RequestOptions.DEFAULT) + } + + private fun genLargeSearchOptions(): RequestOptions { + val builder = RequestOptions.DEFAULT.toBuilder() + builder.setHttpAsyncResponseConsumerFactory( + HeapBufferedResponseConsumerFactory(RESPONSE_ENTITY_MAX_SIZE) + ) + return builder.build() + } +} diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/IndexNameUtils.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/IndexNameUtils.kt new file mode 100644 index 00000000000..cc86cccf5e5 --- /dev/null +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/IndexNameUtils.kt @@ -0,0 +1,47 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.openapi.es + +import java.time.LocalDateTime +import java.time.format.DateTimeFormatter + +object IndexNameUtils { + + fun getIndexName(): String { + val formatter = DateTimeFormatter.ofPattern(LOG_INDEX_DATE_FORMAT) + return LOG_INDEX_PREFIX + formatter.format(LocalDateTime.now()) + } + + fun getNextIndexName(): String { + val formatter = DateTimeFormatter.ofPattern(LOG_INDEX_DATE_FORMAT) + return LOG_INDEX_PREFIX + formatter.format(LocalDateTime.now().plusDays(1)) + } + + private const val LOG_INDEX_PREFIX = "openapi-metric-log-" + private const val LOG_INDEX_DATE_FORMAT = "yyyy-MM-dd" +} diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/config/ESAutoConfiguration.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/config/ESAutoConfiguration.kt new file mode 100644 index 00000000000..fe77eaf9493 --- /dev/null +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/config/ESAutoConfiguration.kt @@ -0,0 +1,64 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.openapi.es.config + +import com.tencent.devops.common.es.ESAutoConfiguration +import com.tencent.devops.common.es.ESProperties +import com.tencent.devops.common.es.client.LogClient +import com.tencent.devops.common.redis.RedisOperation +import com.tencent.devops.openapi.es.ESServiceImpl +import com.tencent.devops.openapi.es.mq.MQDispatcher +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.autoconfigure.AutoConfigureAfter +import org.springframework.boot.autoconfigure.AutoConfigureOrder +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty +import org.springframework.boot.context.properties.EnableConfigurationProperties +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Configuration +import org.springframework.core.Ordered + +@Suppress("ALL") +@Configuration +@ConditionalOnProperty(prefix = "log.storage", name = ["type"], havingValue = "elasticsearch") +@AutoConfigureOrder(Ordered.HIGHEST_PRECEDENCE) +@AutoConfigureAfter(ESAutoConfiguration::class) +@EnableConfigurationProperties(ESProperties::class) +class ESAutoConfiguration { + @Bean + fun esLogService( + @Autowired logESClient: LogClient, + @Autowired redisOperation: RedisOperation, + @Autowired openapiMQDispatcher: MQDispatcher + ): ESServiceImpl { + return ESServiceImpl( + logClient = logESClient, + redisOperation = redisOperation, + dispatcher = openapiMQDispatcher + ) + } +} diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/config/MQConfiguration.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/config/MQConfiguration.kt new file mode 100644 index 00000000000..01933c5d149 --- /dev/null +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/config/MQConfiguration.kt @@ -0,0 +1,62 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.openapi.es.config + +import com.tencent.devops.common.stream.constants.StreamBinding +import com.tencent.devops.openapi.es.mq.ESEvent +import com.tencent.devops.openapi.es.mq.MQDispatcher +import com.tencent.devops.openapi.es.mq.MQListenerService +import java.util.function.Consumer +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.autoconfigure.AutoConfigureOrder +import org.springframework.boot.autoconfigure.condition.ConditionalOnWebApplication +import org.springframework.cloud.stream.function.StreamBridge +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Configuration +import org.springframework.core.Ordered +import org.springframework.messaging.Message + +@Configuration +@ConditionalOnWebApplication +@AutoConfigureOrder(Ordered.LOWEST_PRECEDENCE) +class MQConfiguration @Autowired constructor() { + + @Bean + fun openapiMQDispatcher( + streamBridge: StreamBridge + ) = MQDispatcher(streamBridge) + + @Bean(StreamBinding.BINDING_OPENAPI_LOG_EVENT_IN) + fun openapiLogEventIn( + listenerService: MQListenerService + ): Consumer> { + return Consumer { event: Message -> + listenerService.handleEvent(event.payload) + } + } +} diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/mq/ESEvent.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/mq/ESEvent.kt new file mode 100644 index 00000000000..76d37cacfe7 --- /dev/null +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/mq/ESEvent.kt @@ -0,0 +1,48 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.openapi.es.mq + +import com.tencent.devops.common.stream.annotation.StreamEvent +import com.tencent.devops.common.stream.constants.StreamBinding +import com.tencent.devops.common.stream.pojo.IEvent +import com.tencent.devops.openapi.es.ESMessage + +@StreamEvent(StreamBinding.BINDING_OPENAPI_LOG_EVENT_OUT) +data class ESEvent( + val logs: ESMessage, + override var retryTime: Int = 2, + override var delayMills: Int = 0 +) : IEvent(delayMills, retryTime) { + companion object { + private const val DELAY_DURATION_MILLS = 3 * 1000 + } + + fun getNextDelayMills(retryTime: Int): Int { + return DELAY_DURATION_MILLS * (3 - retryTime) + } +} diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/mq/MQDispatcher.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/mq/MQDispatcher.kt new file mode 100644 index 00000000000..b1c7a1b419d --- /dev/null +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/mq/MQDispatcher.kt @@ -0,0 +1,45 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.openapi.es.mq + +import org.slf4j.LoggerFactory +import org.springframework.cloud.stream.function.StreamBridge + +@Suppress("MagicNumber") +class MQDispatcher constructor( + private val streamBridge: StreamBridge +) { + + fun dispatchEvent(event: ESEvent) { + event.sendTo(streamBridge) + } + + companion object { + private val logger = LoggerFactory.getLogger(MQDispatcher::class.java) + } +} diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/mq/MQListenerService.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/mq/MQListenerService.kt new file mode 100644 index 00000000000..27c27240660 --- /dev/null +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/mq/MQListenerService.kt @@ -0,0 +1,68 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.openapi.es.mq + +import com.tencent.devops.openapi.es.ESServiceImpl +import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Component + +@Component +class MQListenerService @Autowired constructor( + private val logService: ESServiceImpl, + private val dispatcher: MQDispatcher +) { + + fun handleEvent(event: ESEvent) { + var result = false + try { + logService.esAddMessage(event) + result = true + } catch (ignored: Throwable) { + logger.warn("Fail to add the log batch event [${event.logs}|${event.retryTime}]", ignored) + } finally { + if (!result && event.retryTime >= 0) { + logger.warn("Retry to add log batch event [${event.logs}|${event.retryTime}]") + with(event) { + dispatcher.dispatchEvent( + ESEvent( + logs = logs, + retryTime = retryTime - 1, + delayMills = getNextDelayMills(retryTime) + ) + ) + } + } + } + } + + + companion object { + private val logger = LoggerFactory.getLogger(MQListenerService::class.java) + } +} diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/resources/META-INF/spring.factories b/src/backend/ci/core/openapi/biz-openapi/src/main/resources/META-INF/spring.factories new file mode 100644 index 00000000000..af0bc69488c --- /dev/null +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/resources/META-INF/spring.factories @@ -0,0 +1,2 @@ +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ +com.tencent.devops.openapi.es.config.ESAutoConfiguration diff --git a/src/backend/ci/settings.gradle.kts b/src/backend/ci/settings.gradle.kts index bbcf02fd0e9..32cb1bef226 100644 --- a/src/backend/ci/settings.gradle.kts +++ b/src/backend/ci/settings.gradle.kts @@ -63,6 +63,7 @@ include(":core:common:common-db-sharding") include(":core:common:common-client") include(":core:common:common-redis") include(":core:common:common-kafka") +include(":core:common:common-es") include(":core:common:common-scm") include(":core:common:common-archive") include(":core:common:common-quality") diff --git a/support-files/sql/1001_ci_openapi_ddl_mysql.sql b/support-files/sql/1001_ci_openapi_ddl_mysql.sql index d50fe0555a8..ee00e0de9be 100644 --- a/support-files/sql/1001_ci_openapi_ddl_mysql.sql +++ b/support-files/sql/1001_ci_openapi_ddl_mysql.sql @@ -40,7 +40,7 @@ CREATE TABLE IF NOT EXISTS `T_APP_CODE_PROJECT` ( -- ---------------------------- -- Table structure for T_APP_USER_INFO -- ---------------------------- -CREATE TABLE IF NOT EXISTS T_APP_USER_INFO( +CREATE TABLE IF NOT EXISTS `T_APP_USER_INFO`( `ID` INT(11) NOT NULL AUTO_INCREMENT COMMENT '主键ID', `APP_CODE` VARCHAR(64) NOT NULL COMMENT 'APP编码', `MANAGER_ID` VARCHAR(64) NOT NULL COMMENT 'APP管理员ID', @@ -52,4 +52,37 @@ CREATE TABLE IF NOT EXISTS T_APP_USER_INFO( UNIQUE INDEX `IDX_APP_USER` (`APP_CODE`, `MANAGER_ID`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='app_code对应的管理员'; +-- ---------------------------- +-- Table structure for T_OPENAPI_METRICS_FOR_API +-- ---------------------------- +CREATE TABLE IF NOT EXISTS `T_OPENAPI_METRICS_FOR_API`( + `API` VARCHAR(64) NOT NULL COMMENT 'api接口代码', + `APP_CODE` VARCHAR(64) NOT NULL COMMENT 'APP编码', + `USER_ID` VARCHAR(64) NOT NULL COMMENT 'api请求用户', + `SECOND_LEVEL_CONCURRENCY` int(11) NOT NULL COMMENT '秒级并发量', + `PEAK_CONCURRENCY` int(11) NOT NULL COMMENT '峰值并发量', + `CALL_5M` int(11) NOT NULL COMMENT '5min调用量', + `CALL_1H` int(11) NOT NULL COMMENT '1h调用量', + `CALL_24H` int(11) NOT NULL COMMENT '24h调用量', + `CALL_7D` int(11) NOT NULL COMMENT '7d调用量', + INDEX `IDX_APP` (`APP_CODE`), + INDEX `IDX_USER_ID` (`USER_ID`), + UNIQUE INDEX `IDX_APP_USER` (`API`, `APP_CODE`, `USER_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='接口维度度量表'; + +-- ---------------------------- +-- Table structure for T_OPENAPI_METRICS_FOR_PROJECT +-- ---------------------------- +CREATE TABLE IF NOT EXISTS `T_OPENAPI_METRICS_FOR_PROJECT`( + `PROJECT` VARCHAR(64) NOT NULL COMMENT '项目id', + `API` VARCHAR(64) NOT NULL COMMENT 'api接口代码', + `APP_CODE` VARCHAR(64) NOT NULL COMMENT 'APP编码', + `USER_ID` VARCHAR(64) NOT NULL COMMENT 'api请求用户', + `CALL_LAST_DAY` int(11) NOT NULL COMMENT '昨日', + INDEX `IDX_API` (`API`), + INDEX `IDX_APP` (`APP_CODE`), + INDEX `IDX_USER_ID` (`USER_ID`), + UNIQUE INDEX `IDX_APP_USER` (`PROJECT`, `API`, `APP_CODE`, `USER_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='项目维度度量表'; + SET FOREIGN_KEY_CHECKS = 1; From d5251fadff1a2b0c9f45e96876b6d02c53132457 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?yongyiduan=28=E6=AE=B5=E6=B0=B8=E5=84=84=29?= Date: Mon, 6 Nov 2023 10:27:04 +0800 Subject: [PATCH 02/31] =?UTF-8?q?feat:=20openapi=E6=96=B0=E5=A2=9E?= =?UTF-8?q?=E5=BA=A6=E9=87=8F=E8=83=BD=E5=8A=9B=20#9638?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/kotlin/com/tencent/devops/openapi/aspect/ApiAspect.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/aspect/ApiAspect.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/aspect/ApiAspect.kt index ca0d8f6a432..5fc8af68ec4 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/aspect/ApiAspect.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/aspect/ApiAspect.kt @@ -253,7 +253,7 @@ class ApiAspect( ?.interfaces ?.first() ?.getDeclaredMethod( - method.name + method.name, *method.parameterTypes ) ?.getAnnotation(ApiOperation::class.java) ?.tags From 007aaec25d294e31fe5616b8cbce926f4d19b000 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?yongyiduan=28=E6=AE=B5=E6=B0=B8=E5=84=84=29?= Date: Mon, 6 Nov 2023 11:17:28 +0800 Subject: [PATCH 03/31] =?UTF-8?q?feat:=20openapi=E6=96=B0=E5=A2=9E?= =?UTF-8?q?=E5=BA=A6=E9=87=8F=E8=83=BD=E5=8A=9B=20#9638?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tencent/devops/common/stream/constants/StreamBinding.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/ci/core/common/common-stream/src/main/kotlin/com/tencent/devops/common/stream/constants/StreamBinding.kt b/src/backend/ci/core/common/common-stream/src/main/kotlin/com/tencent/devops/common/stream/constants/StreamBinding.kt index 9ccd6d76617..46d78e55eb6 100644 --- a/src/backend/ci/core/common/common-stream/src/main/kotlin/com/tencent/devops/common/stream/constants/StreamBinding.kt +++ b/src/backend/ci/core/common/common-stream/src/main/kotlin/com/tencent/devops/common/stream/constants/StreamBinding.kt @@ -41,6 +41,6 @@ object StreamBinding { const val BINDING_LOG_STATUS_EVENT_OUT = "logStatusEventOut" // openapi审计日志预处理事件 - const val BINDING_OPENAPI_LOG_EVENT_IN = "logOriginEventIn" - const val BINDING_OPENAPI_LOG_EVENT_OUT = "logOriginEventOut" + const val BINDING_OPENAPI_LOG_EVENT_IN = "openapiLogOriginEventIn" + const val BINDING_OPENAPI_LOG_EVENT_OUT = "openapiLogOriginEventOut" } From cdb0e4b6b786511a369abf60546e088506891b30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?yongyiduan=28=E6=AE=B5=E6=B0=B8=E5=84=84=29?= Date: Mon, 6 Nov 2023 11:27:13 +0800 Subject: [PATCH 04/31] =?UTF-8?q?feat:=20openapi=E6=96=B0=E5=A2=9E?= =?UTF-8?q?=E5=BA=A6=E9=87=8F=E8=83=BD=E5=8A=9B=20#9638?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/kotlin/com/tencent/devops/openapi/es/ESIndexUtils.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESIndexUtils.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESIndexUtils.kt index c7a6a30a896..4af89a18b43 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESIndexUtils.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESIndexUtils.kt @@ -51,7 +51,7 @@ object ESIndexUtils { .startObject("timestamp").field("type", "long").endObject() .startObject("appCode").field("type", "keyword").endObject() .startObject("userId").field("type", "keyword").endObject() - .startObject("projectId").field("type", "keyword").endObject() + .startObject("projectId").field("type", "keyword") .field("analyzer", "standard") .endObject() .endObject() From 7367fcb70330cce65bedcf075e37e93d1dad1d9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?yongyiduan=28=E6=AE=B5=E6=B0=B8=E5=84=84=29?= Date: Mon, 6 Nov 2023 14:53:45 +0800 Subject: [PATCH 05/31] =?UTF-8?q?feat:=20openapi=E6=96=B0=E5=A2=9E?= =?UTF-8?q?=E5=BA=A6=E9=87=8F=E8=83=BD=E5=8A=9B=20#9638?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kotlin/com/tencent/devops/openapi/aspect/ApiAspect.kt | 3 ++- .../main/kotlin/com/tencent/devops/openapi/es/ESIndexUtils.kt | 4 ++-- .../main/kotlin/com/tencent/devops/openapi/es/ESMessage.kt | 4 +++- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/aspect/ApiAspect.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/aspect/ApiAspect.kt index 5fc8af68ec4..9542a0cd07b 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/aspect/ApiAspect.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/aspect/ApiAspect.kt @@ -44,6 +44,7 @@ import com.tencent.devops.openapi.service.OpenapiPermissionService import com.tencent.devops.openapi.service.op.AppCodeService import com.tencent.devops.openapi.utils.ApiGatewayUtil import io.swagger.annotations.ApiOperation +import java.util.Date import javax.ws.rs.core.Response import kotlin.reflect.jvm.kotlinFunction import org.aspectj.lang.JoinPoint @@ -116,7 +117,7 @@ class ApiAspect( appCode = appCode ?: "", userId = userId ?: "", projectId = projectId ?: "", - timestamp = System.currentTimeMillis() + date = Date() ) ) diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESIndexUtils.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESIndexUtils.kt index 4af89a18b43..643f7387f23 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESIndexUtils.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESIndexUtils.kt @@ -48,7 +48,7 @@ object ESIndexUtils { .startObject("properties") .startObject("api").field("type", "keyword").endObject() .startObject("apiType").field("type", "keyword").endObject() - .startObject("timestamp").field("type", "long").endObject() + .startObject("date").field("type", "date").endObject() .startObject("appCode").field("type", "keyword").endObject() .startObject("userId").field("type", "keyword").endObject() .startObject("projectId").field("type", "keyword") @@ -68,7 +68,7 @@ object ESIndexUtils { .field("appCode", logMessage.appCode) .field("userId", logMessage.userId) .field("projectId", logMessage.projectId) - .field("timestamp", logMessage.timestamp) + .field("date", logMessage.date) .endObject() } } diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESMessage.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESMessage.kt index 41110b922bd..f8a0412e228 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESMessage.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESMessage.kt @@ -27,6 +27,8 @@ package com.tencent.devops.openapi.es +import java.util.Date + /** * * Powered By Tencent @@ -37,5 +39,5 @@ data class ESMessage( val appCode: String = "", val userId: String = "", val projectId: String = "", - var timestamp: Long = 0 + var date: Date ) From 239c75f3dca3f288ceceb6761569dd48bfd5236b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?yongyiduan=28=E6=AE=B5=E6=B0=B8=E5=84=84=29?= Date: Mon, 6 Nov 2023 15:02:51 +0800 Subject: [PATCH 06/31] =?UTF-8?q?Revert=20"feat:=20openapi=E6=96=B0?= =?UTF-8?q?=E5=A2=9E=E5=BA=A6=E9=87=8F=E8=83=BD=E5=8A=9B=20#9638"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 7367fcb70330cce65bedcf075e37e93d1dad1d9d. --- .../kotlin/com/tencent/devops/openapi/aspect/ApiAspect.kt | 3 +-- .../main/kotlin/com/tencent/devops/openapi/es/ESIndexUtils.kt | 4 ++-- .../main/kotlin/com/tencent/devops/openapi/es/ESMessage.kt | 4 +--- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/aspect/ApiAspect.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/aspect/ApiAspect.kt index 9542a0cd07b..5fc8af68ec4 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/aspect/ApiAspect.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/aspect/ApiAspect.kt @@ -44,7 +44,6 @@ import com.tencent.devops.openapi.service.OpenapiPermissionService import com.tencent.devops.openapi.service.op.AppCodeService import com.tencent.devops.openapi.utils.ApiGatewayUtil import io.swagger.annotations.ApiOperation -import java.util.Date import javax.ws.rs.core.Response import kotlin.reflect.jvm.kotlinFunction import org.aspectj.lang.JoinPoint @@ -117,7 +116,7 @@ class ApiAspect( appCode = appCode ?: "", userId = userId ?: "", projectId = projectId ?: "", - date = Date() + timestamp = System.currentTimeMillis() ) ) diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESIndexUtils.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESIndexUtils.kt index 643f7387f23..4af89a18b43 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESIndexUtils.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESIndexUtils.kt @@ -48,7 +48,7 @@ object ESIndexUtils { .startObject("properties") .startObject("api").field("type", "keyword").endObject() .startObject("apiType").field("type", "keyword").endObject() - .startObject("date").field("type", "date").endObject() + .startObject("timestamp").field("type", "long").endObject() .startObject("appCode").field("type", "keyword").endObject() .startObject("userId").field("type", "keyword").endObject() .startObject("projectId").field("type", "keyword") @@ -68,7 +68,7 @@ object ESIndexUtils { .field("appCode", logMessage.appCode) .field("userId", logMessage.userId) .field("projectId", logMessage.projectId) - .field("date", logMessage.date) + .field("timestamp", logMessage.timestamp) .endObject() } } diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESMessage.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESMessage.kt index f8a0412e228..41110b922bd 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESMessage.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESMessage.kt @@ -27,8 +27,6 @@ package com.tencent.devops.openapi.es -import java.util.Date - /** * * Powered By Tencent @@ -39,5 +37,5 @@ data class ESMessage( val appCode: String = "", val userId: String = "", val projectId: String = "", - var date: Date + var timestamp: Long = 0 ) From 9f7be43806f24159909be36bcc47fc560e4ce7fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?yongyiduan=28=E6=AE=B5=E6=B0=B8=E5=84=84=29?= Date: Mon, 6 Nov 2023 15:03:32 +0800 Subject: [PATCH 07/31] =?UTF-8?q?feat:=20openapi=E6=96=B0=E5=A2=9E?= =?UTF-8?q?=E5=BA=A6=E9=87=8F=E8=83=BD=E5=8A=9B=20#9638?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/kotlin/com/tencent/devops/openapi/es/ESIndexUtils.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESIndexUtils.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESIndexUtils.kt index 4af89a18b43..5906070900d 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESIndexUtils.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESIndexUtils.kt @@ -48,7 +48,7 @@ object ESIndexUtils { .startObject("properties") .startObject("api").field("type", "keyword").endObject() .startObject("apiType").field("type", "keyword").endObject() - .startObject("timestamp").field("type", "long").endObject() + .startObject("timestamp").field("type", "date").endObject() .startObject("appCode").field("type", "keyword").endObject() .startObject("userId").field("type", "keyword").endObject() .startObject("projectId").field("type", "keyword") From 13354edf224834331c32f5360dcc35d5a505c18e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?yongyiduan=28=E6=AE=B5=E6=B0=B8=E5=84=84=29?= Date: Mon, 6 Nov 2023 15:20:58 +0800 Subject: [PATCH 08/31] =?UTF-8?q?feat:=20openapi=E6=96=B0=E5=A2=9E?= =?UTF-8?q?=E5=BA=A6=E9=87=8F=E8=83=BD=E5=8A=9B=20#9638?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/kotlin/com/tencent/devops/openapi/es/ESIndexUtils.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESIndexUtils.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESIndexUtils.kt index 5906070900d..50e07e71a0d 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESIndexUtils.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESIndexUtils.kt @@ -52,7 +52,6 @@ object ESIndexUtils { .startObject("appCode").field("type", "keyword").endObject() .startObject("userId").field("type", "keyword").endObject() .startObject("projectId").field("type", "keyword") - .field("analyzer", "standard") .endObject() .endObject() .endObject() From 6a9b02b6d8908dcd2222b22cfb3725ee2974ae4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?yongyiduan=28=E6=AE=B5=E6=B0=B8=E5=84=84=29?= Date: Mon, 6 Nov 2023 16:07:00 +0800 Subject: [PATCH 09/31] =?UTF-8?q?feat:=20openapi=E6=96=B0=E5=A2=9E?= =?UTF-8?q?=E5=BA=A6=E9=87=8F=E8=83=BD=E5=8A=9B=20#9638?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/tencent/devops/openapi/es/ESIndexUtils.kt | 13 +++++++------ .../com/tencent/devops/openapi/es/ESServiceImpl.kt | 2 ++ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESIndexUtils.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESIndexUtils.kt index 50e07e71a0d..e478ef31dde 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESIndexUtils.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESIndexUtils.kt @@ -47,10 +47,8 @@ object ESIndexUtils { .startObject() .startObject("properties") .startObject("api").field("type", "keyword").endObject() - .startObject("apiType").field("type", "keyword").endObject() .startObject("timestamp").field("type", "date").endObject() - .startObject("appCode").field("type", "keyword").endObject() - .startObject("userId").field("type", "keyword").endObject() + .startObject("key").field("type", "keyword").endObject() .startObject("projectId").field("type", "keyword") .endObject() .endObject() @@ -63,9 +61,12 @@ object ESIndexUtils { return XContentFactory.jsonBuilder() .startObject() .field("api", logMessage.api) - .field("apiType", logMessage.apiType) - .field("appCode", logMessage.appCode) - .field("userId", logMessage.userId) + .let { + when { + logMessage.apiType.contains("user") -> it.field("key", "user:" + logMessage.userId) + else -> it.field("key", "app:" + logMessage.appCode) + } + } .field("projectId", logMessage.projectId) .field("timestamp", logMessage.timestamp) .endObject() diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESServiceImpl.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESServiceImpl.kt index 5c9de82b851..785f22f357d 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESServiceImpl.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESServiceImpl.kt @@ -283,4 +283,6 @@ class ESServiceImpl constructor( ) return builder.build() } + + fun } From e9363ed0da0c274f5f1530961bf24bae2cc1673d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?yongyiduan=28=E6=AE=B5=E6=B0=B8=E5=84=84=29?= Date: Mon, 6 Nov 2023 16:07:45 +0800 Subject: [PATCH 10/31] =?UTF-8?q?feat:=20openapi=E6=96=B0=E5=A2=9E?= =?UTF-8?q?=E5=BA=A6=E9=87=8F=E8=83=BD=E5=8A=9B=20#9638?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/kotlin/com/tencent/devops/openapi/es/ESServiceImpl.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESServiceImpl.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESServiceImpl.kt index 785f22f357d..35a598e1e53 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESServiceImpl.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESServiceImpl.kt @@ -284,5 +284,5 @@ class ESServiceImpl constructor( return builder.build() } - fun +// fun } From 8b79c9c37a49ade376d03c4549a94419fbbc386b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?yongyiduan=28=E6=AE=B5=E6=B0=B8=E5=84=84=29?= Date: Mon, 6 Nov 2023 16:48:34 +0800 Subject: [PATCH 11/31] =?UTF-8?q?feat:=20openapi=E6=96=B0=E5=A2=9E?= =?UTF-8?q?=E5=BA=A6=E9=87=8F=E8=83=BD=E5=8A=9B=20#9638?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kotlin/com/tencent/devops/openapi/aspect/ApiAspect.kt | 5 ++++- .../kotlin/com/tencent/devops/openapi/es/ESIndexUtils.kt | 5 +++-- .../main/kotlin/com/tencent/devops/openapi/es/ESMessage.kt | 1 + 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/aspect/ApiAspect.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/aspect/ApiAspect.kt index 5fc8af68ec4..30890c8a4cd 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/aspect/ApiAspect.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/aspect/ApiAspect.kt @@ -54,6 +54,8 @@ import org.aspectj.lang.reflect.MethodSignature import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Value import org.springframework.stereotype.Component +import org.springframework.web.context.request.RequestContextHolder +import org.springframework.web.context.request.ServletRequestAttributes @Aspect @Component @@ -108,7 +110,7 @@ class ApiAspect( else -> Unit } } - + val request = (RequestContextHolder.currentRequestAttributes() as ServletRequestAttributes).request esServiceImpl.addMessage( ESMessage( api = getApiTag(jp = jp, apiType = apigwType?.split("-")?.getOrNull(1) ?: ""), @@ -116,6 +118,7 @@ class ApiAspect( appCode = appCode ?: "", userId = userId ?: "", projectId = projectId ?: "", + path = request.requestURI, timestamp = System.currentTimeMillis() ) ) diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESIndexUtils.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESIndexUtils.kt index e478ef31dde..1e710ebc309 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESIndexUtils.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESIndexUtils.kt @@ -49,8 +49,8 @@ object ESIndexUtils { .startObject("api").field("type", "keyword").endObject() .startObject("timestamp").field("type", "date").endObject() .startObject("key").field("type", "keyword").endObject() - .startObject("projectId").field("type", "keyword") - .endObject() + .startObject("path").field("type", "text").endObject() + .startObject("projectId").field("type", "keyword").endObject() .endObject() .endObject() } @@ -68,6 +68,7 @@ object ESIndexUtils { } } .field("projectId", logMessage.projectId) + .field("path", logMessage.path) .field("timestamp", logMessage.timestamp) .endObject() } diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESMessage.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESMessage.kt index 41110b922bd..f3418a6dfcd 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESMessage.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESMessage.kt @@ -37,5 +37,6 @@ data class ESMessage( val appCode: String = "", val userId: String = "", val projectId: String = "", + val path: String = "", var timestamp: Long = 0 ) From c52e14098a0016e6ab14a3dbccbb56e157ae2a0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?yongyiduan=28=E6=AE=B5=E6=B0=B8=E5=84=84=29?= Date: Tue, 7 Nov 2023 15:43:36 +0800 Subject: [PATCH 12/31] =?UTF-8?q?feat:=20openapi=E6=96=B0=E5=A2=9E?= =?UTF-8?q?=E5=BA=A6=E9=87=8F=E8=83=BD=E5=8A=9B=20#9638?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../devops/openapi/pojo/MetricsApiData.kt | 50 ++++ .../devops/openapi/pojo/MetricsProjectData.kt | 42 ++++ .../devops/openapi/aspect/ApiAspect.kt | 11 +- .../devops/openapi/dao/MetricsForApiDao.kt | 103 +++++++++ .../openapi/dao/MetricsForProjectDao.kt | 65 ++++++ .../tencent/devops/openapi/es/ESIndexUtils.kt | 25 +- .../tencent/devops/openapi/es/ESMessage.kt | 4 +- .../devops/openapi/es/ESServiceImpl.kt | 216 +++++++++++++++++- .../devops/openapi/service/MetricsService.kt | 113 +++++++++ .../sql/1001_ci_openapi_ddl_mysql.sql | 18 +- 10 files changed, 613 insertions(+), 34 deletions(-) create mode 100644 src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/pojo/MetricsApiData.kt create mode 100644 src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/pojo/MetricsProjectData.kt create mode 100644 src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/dao/MetricsForApiDao.kt create mode 100644 src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/dao/MetricsForProjectDao.kt create mode 100644 src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/service/MetricsService.kt diff --git a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/pojo/MetricsApiData.kt b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/pojo/MetricsApiData.kt new file mode 100644 index 00000000000..59e6ceb7983 --- /dev/null +++ b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/pojo/MetricsApiData.kt @@ -0,0 +1,50 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.tencent.devops.openapi.pojo + +import io.swagger.annotations.ApiModel +import io.swagger.annotations.ApiModelProperty + +@ApiModel("api度量数据") +data class MetricsApiData( + @ApiModelProperty("接口代码") + val api: String, + @ApiModelProperty("请求app code/ user id") + val key: String, + @ApiModelProperty("秒级并发量") + var secondLevelConcurrency: Int? = null, + @ApiModelProperty("峰值并发量") + var peakConcurrency: Int? = null, + @ApiModelProperty("5min调用量") + var call5m: Int? = null, + @ApiModelProperty("1h调用量") + var call1h: Int? = null, + @ApiModelProperty("24h调用量") + var call24h: Int? = null, + @ApiModelProperty("7d调用量") + var call7d: Int? = null +) diff --git a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/pojo/MetricsProjectData.kt b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/pojo/MetricsProjectData.kt new file mode 100644 index 00000000000..64cf3d26851 --- /dev/null +++ b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/pojo/MetricsProjectData.kt @@ -0,0 +1,42 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.tencent.devops.openapi.pojo + +import io.swagger.annotations.ApiModel +import io.swagger.annotations.ApiModelProperty + +@ApiModel("api度量数据") +data class MetricsProjectData( + @ApiModelProperty("接口代码") + val api: String, + @ApiModelProperty("请求app code/ user id") + val key: String, + @ApiModelProperty("项目id") + var projectId: String? = null, + @ApiModelProperty("调用量") + var callHistory: Int? = null +) diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/aspect/ApiAspect.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/aspect/ApiAspect.kt index 30890c8a4cd..ad694ccbdb6 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/aspect/ApiAspect.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/aspect/ApiAspect.kt @@ -111,12 +111,15 @@ class ApiAspect( } } val request = (RequestContextHolder.currentRequestAttributes() as ServletRequestAttributes).request + val apiType = apigwType?.split("-")?.getOrNull(1) ?: "" esServiceImpl.addMessage( ESMessage( - api = getApiTag(jp = jp, apiType = apigwType?.split("-")?.getOrNull(1) ?: ""), - apiType = apigwType ?: "", - appCode = appCode ?: "", - userId = userId ?: "", + api = getApiTag(jp = jp, apiType = apiType), + key = when { + apiType.isBlank() -> "null" + apiType.contains("user") -> "user:$userId" + else -> "app:$appCode" + }, projectId = projectId ?: "", path = request.requestURI, timestamp = System.currentTimeMillis() diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/dao/MetricsForApiDao.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/dao/MetricsForApiDao.kt new file mode 100644 index 00000000000..59e6ac76575 --- /dev/null +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/dao/MetricsForApiDao.kt @@ -0,0 +1,103 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.openapi.dao + +import com.tencent.devops.common.db.utils.skipCheck +import com.tencent.devops.model.openapi.tables.TOpenapiMetricsForApi +import com.tencent.devops.openapi.pojo.MetricsApiData +import org.jooq.DSLContext +import org.jooq.InsertOnDuplicateSetMoreStep +import org.springframework.stereotype.Repository + +@Repository +class MetricsForApiDao { + fun createOrUpdate( + dslContext: DSLContext, + metricsApis: List, + ): Int { + return with(TOpenapiMetricsForApi.T_OPENAPI_METRICS_FOR_API) { + dslContext.batch( + metricsApis.mapNotNull { metricsApi -> + dslContext.insertInto( + this, + API, + KEY, + SECOND_LEVEL_CONCURRENCY, + PEAK_CONCURRENCY, + CALL_5M, + CALL_1H, + CALL_24H, + CALL_7D + ).values( + metricsApi.api, + metricsApi.key, + metricsApi.secondLevelConcurrency ?: 0, + metricsApi.peakConcurrency ?: 0, + metricsApi.call5m ?: 0, + metricsApi.call1h ?: 0, + metricsApi.call24h ?: 0, + metricsApi.call7d ?: 0 + ).onDuplicateKeyUpdate() + .let { u -> + metricsApi.secondLevelConcurrency?.let { i -> u.set(SECOND_LEVEL_CONCURRENCY, i) } ?: u + } + .let { u -> + metricsApi.peakConcurrency?.let { i -> u.set(PEAK_CONCURRENCY, i) } ?: u + } + .let { u -> + metricsApi.call5m?.let { i -> u.set(CALL_5M, i) } ?: u + } + .let { u -> + metricsApi.call1h?.let { i -> u.set(CALL_1H, i) } ?: u + } + .let { u -> + metricsApi.call24h?.let { i -> u.set(CALL_24H, i) } ?: u + } + .let { u -> + metricsApi.call7d?.let { i -> u.set(CALL_7D, i) } ?: u + } + .let { if (it is InsertOnDuplicateSetMoreStep) it else return@mapNotNull null } + } + ).execute().sum() + } + } + + fun batchGet( + dslContext: DSLContext + ): List { + return with(TOpenapiMetricsForApi.T_OPENAPI_METRICS_FOR_API) { + dslContext.select(API, KEY, PEAK_CONCURRENCY).from(this).skipCheck().fetch { + MetricsApiData( + api = it.value1(), + key = it.value2(), + peakConcurrency = it.value3() + ) + } + } + } +} diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/dao/MetricsForProjectDao.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/dao/MetricsForProjectDao.kt new file mode 100644 index 00000000000..18bbd5ad303 --- /dev/null +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/dao/MetricsForProjectDao.kt @@ -0,0 +1,65 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.openapi.dao + +import com.tencent.devops.model.openapi.tables.TOpenapiMetricsForProject +import com.tencent.devops.openapi.pojo.MetricsProjectData +import org.jooq.DSLContext +import org.jooq.InsertOnDuplicateSetMoreStep +import org.springframework.stereotype.Repository + +@Repository +class MetricsForProjectDao { + fun createOrUpdate( + dslContext: DSLContext, + metricsApis: List, + ): Int { + return with(TOpenapiMetricsForProject.T_OPENAPI_METRICS_FOR_PROJECT) { + dslContext.batch( + metricsApis.mapNotNull { metricsApi -> + dslContext.insertInto( + this, + PROJECT, + API, + KEY, + CALL_HISTORY + ).values( + metricsApi.projectId ?: "", + metricsApi.api, + metricsApi.key, + metricsApi.callHistory ?: 0 + ).onDuplicateKeyUpdate() + .let { u -> + metricsApi.callHistory?.let { i -> u.set(CALL_HISTORY, CALL_HISTORY + i) } ?: u + } + .let { if (it is InsertOnDuplicateSetMoreStep) it else return@mapNotNull null } + } + ).execute().sum() + } + } +} diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESIndexUtils.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESIndexUtils.kt index 1e710ebc309..6425a8ed1d9 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESIndexUtils.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESIndexUtils.kt @@ -46,11 +46,11 @@ object ESIndexUtils { return XContentFactory.jsonBuilder() .startObject() .startObject("properties") - .startObject("api").field("type", "keyword").endObject() - .startObject("timestamp").field("type", "date").endObject() - .startObject("key").field("type", "keyword").endObject() - .startObject("path").field("type", "text").endObject() - .startObject("projectId").field("type", "keyword").endObject() + .startObject(ESMessage::api.name).field("type", "keyword").endObject() + .startObject(ESMessage::timestamp.name).field("type", "date").endObject() + .startObject(ESMessage::key.name).field("type", "keyword").endObject() + .startObject(ESMessage::path.name).field("type", "text").endObject() + .startObject(ESMessage::projectId.name).field("type", "keyword").endObject() .endObject() .endObject() } @@ -60,16 +60,11 @@ object ESIndexUtils { ): XContentBuilder { return XContentFactory.jsonBuilder() .startObject() - .field("api", logMessage.api) - .let { - when { - logMessage.apiType.contains("user") -> it.field("key", "user:" + logMessage.userId) - else -> it.field("key", "app:" + logMessage.appCode) - } - } - .field("projectId", logMessage.projectId) - .field("path", logMessage.path) - .field("timestamp", logMessage.timestamp) + .field(ESMessage::api.name, logMessage.api) + .field(ESMessage::key.name, logMessage.key) + .field(ESMessage::projectId.name, logMessage.projectId) + .field(ESMessage::path.name, logMessage.path) + .field(ESMessage::timestamp.name, logMessage.timestamp) .endObject() } } diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESMessage.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESMessage.kt index f3418a6dfcd..3fb5a47a04d 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESMessage.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESMessage.kt @@ -33,9 +33,7 @@ package com.tencent.devops.openapi.es */ data class ESMessage( val api: String, - val apiType: String = "", - val appCode: String = "", - val userId: String = "", + val key: String = "", val projectId: String = "", val path: String = "", var timestamp: Long = 0 diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESServiceImpl.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESServiceImpl.kt index 35a598e1e53..fd4c5a3753c 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESServiceImpl.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESServiceImpl.kt @@ -35,6 +35,8 @@ import com.tencent.devops.common.redis.RedisLock import com.tencent.devops.common.redis.RedisOperation import com.tencent.devops.openapi.es.mq.ESEvent import com.tencent.devops.openapi.es.mq.MQDispatcher +import com.tencent.devops.openapi.pojo.MetricsApiData +import com.tencent.devops.openapi.pojo.MetricsProjectData import java.io.IOException import java.util.concurrent.ArrayBlockingQueue import java.util.concurrent.BlockingQueue @@ -42,11 +44,20 @@ import java.util.concurrent.Executors import java.util.concurrent.TimeUnit import org.elasticsearch.action.bulk.BulkRequest import org.elasticsearch.action.index.IndexRequest +import org.elasticsearch.action.search.SearchRequest +import org.elasticsearch.action.search.SearchResponse import org.elasticsearch.client.HttpAsyncResponseConsumerFactory.HeapBufferedResponseConsumerFactory import org.elasticsearch.client.RequestOptions import org.elasticsearch.client.indices.CreateIndexRequest import org.elasticsearch.client.indices.GetIndexRequest import org.elasticsearch.core.TimeValue +import org.elasticsearch.index.query.QueryBuilders +import org.elasticsearch.search.aggregations.AggregationBuilders +import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval +import org.elasticsearch.search.aggregations.bucket.terms.Terms +import org.elasticsearch.search.aggregations.metrics.Avg +import org.elasticsearch.search.aggregations.metrics.Max +import org.elasticsearch.search.builder.SearchSourceBuilder import org.slf4j.LoggerFactory class ESServiceImpl constructor( @@ -284,5 +295,208 @@ class ESServiceImpl constructor( return builder.build() } -// fun + + /* + * 每5分钟执行一次 + * 根据"api"字段进行分组，然后在每个分组内根据"key"字段进行进一步的分组。 + * 在每个"key"分组内，使用"date_histogram"聚合，按照"timestamp"字段的时间间隔（1秒）进行分桶。 + * 最后，在每个时间桶内，使用"value_count"聚合计算"timestamp"值的数量，即并发请求数量。 + */ + fun executeElasticsearchQueryS(keyMap: MutableMap, newDay: Boolean) { + val indexName = IndexNameUtils.getIndexName() + + val searchRequest = SearchRequest(indexName) + .source( + SearchSourceBuilder() + .query( + // 设置查询条件 + QueryBuilders.rangeQuery(ESMessage::timestamp.name) + .from("now-5m") + .to("now") + ) + .aggregation( + // 设置聚合 + AggregationBuilders.terms(ESMessage::api.name).field(ESMessage::api.name) + .subAggregation( + AggregationBuilders.terms(ESMessage::key.name).field(ESMessage::key.name) + .subAggregation( + AggregationBuilders.dateHistogram("concurrency") + .field(ESMessage::timestamp.name) + .calendarInterval(DateHistogramInterval.SECOND) + .minDocCount(1) + .subAggregation( + AggregationBuilders.count("count").field(ESMessage::timestamp.name) + ) + ) + .subAggregation(AggregationBuilders.max("max_concurrency").field("count")) + .subAggregation(AggregationBuilders.avg("avg_concurrency").field("count")) + ) + ) + // 不返回任何文档，只返回聚合结果。 + .size(0) + ) + + val response: SearchResponse = logClient.hashClient(indexName) + .restClient + .search(searchRequest, RequestOptions.DEFAULT) + + // 处理聚合结果 + val apiAggregation: Terms = response.aggregations.get(ESMessage::api.name) + + for (apiBucket in apiAggregation.buckets) { + val apiName = apiBucket.keyAsString + val keyAggregation: Terms = apiBucket.aggregations.get(ESMessage::key.name) + + for (keyBucket in keyAggregation.buckets) { + val keyName: String = keyBucket.keyAsString + val max = keyBucket.aggregations.get("max_concurrency").value.toInt() + val avg = keyBucket.aggregations.get("avg_concurrency").value.toInt() + val count = keyBucket.docCount.toInt() + keyMap["$apiName@$keyName"]?.apply { + secondLevelConcurrency = avg + peakConcurrency = if (newDay) max else max.coerceAtLeast(peakConcurrency ?: 0) + call5m = count + } ?: kotlin.run { + keyMap["$apiName@$keyName"] = MetricsApiData( + api = apiName, + key = keyName, + secondLevelConcurrency = avg, + peakConcurrency = max, + call5m = count + ) + } + } + } + } + + /* + * 查询时间间隔为time(1h、1d、7d) + * 根据"api"字段进行分组，然后在每个分组内根据"key"字段进行进一步的分组。 + * 得到每组内的计数 + */ + fun executeElasticsearchQueryM( + keyMap: MutableMap, + time: String, + f: (count: Int, data: MetricsApiData) -> MetricsApiData + ) { + val indexName = IndexNameUtils.getIndexName() + + val searchRequest = SearchRequest(indexName) + .source( + SearchSourceBuilder() + .query( + // 设置查询条件 + QueryBuilders.rangeQuery(ESMessage::timestamp.name) + .from("now-$time") + .to("now") + ) + .aggregation( + // 计数 + AggregationBuilders.terms(ESMessage::api.name).field(ESMessage::api.name) + .subAggregation( + AggregationBuilders.terms(ESMessage::key.name).field(ESMessage::key.name) + .subAggregation( + AggregationBuilders.count("count").field(ESMessage::timestamp.name) + ) + ) + ) + // 不返回任何文档，只返回聚合结果。 + .size(0) + ) + + val response: SearchResponse = logClient.hashClient(indexName) + .restClient + .search(searchRequest, RequestOptions.DEFAULT) + + // 处理聚合结果 + val apiAggregation: Terms = response.aggregations.get(ESMessage::api.name) + + for (apiBucket in apiAggregation.buckets) { + val apiName = apiBucket.keyAsString + val keyAggregation: Terms = apiBucket.aggregations.get(ESMessage::key.name) + + for (keyBucket in keyAggregation.buckets) { + val keyName: String = keyBucket.keyAsString + val count = keyBucket.docCount.toInt() + keyMap["$apiName@$keyName"]?.let { + f(count, it) + } ?: kotlin.run { + keyMap["$apiName@$keyName"] = f( + count, MetricsApiData( + api = apiName, + key = keyName + ) + ) + } + } + } + } + + /* + * 查询时间间隔为1h + * 查询api-key-project三重分组 + * 得到每组内的计数 + */ + fun executeElasticsearchQueryP( + keyList: MutableList + ) { + val indexName = IndexNameUtils.getIndexName() + + val searchRequest = SearchRequest(indexName) + .source( + SearchSourceBuilder() + .query( + // 设置查询条件 + QueryBuilders.rangeQuery(ESMessage::timestamp.name) + .from("now-1h") + .to("now") + ) + .aggregation( + // 计数 + AggregationBuilders.terms(ESMessage::api.name).field(ESMessage::api.name) + .subAggregation( + AggregationBuilders.terms(ESMessage::key.name).field(ESMessage::key.name) + .subAggregation( + AggregationBuilders.terms(ESMessage::projectId.name) + .field(ESMessage::projectId.name) + .subAggregation( + AggregationBuilders.count("count").field(ESMessage::timestamp.name) + ) + ) + ) + ) + // 不返回任何文档，只返回聚合结果。 + .size(0) + ) + + val response: SearchResponse = logClient.hashClient(indexName) + .restClient + .search(searchRequest, RequestOptions.DEFAULT) + + // 处理聚合结果 + val apiAggregation: Terms = response.aggregations.get(ESMessage::api.name) + + for (apiBucket in apiAggregation.buckets) { + val apiName = apiBucket.keyAsString + val keyAggregation: Terms = apiBucket.aggregations.get(ESMessage::key.name) + + for (keyBucket in keyAggregation.buckets) { + val keyName: String = keyBucket.keyAsString + val projectAggregation: Terms = apiBucket.aggregations.get(ESMessage::projectId.name) + + for (projectBucket in projectAggregation.buckets) { + val projectName: String = projectBucket.keyAsString + val count = projectBucket.docCount.toInt() + keyList.add( + MetricsProjectData( + api = apiName, + key = keyName, + projectId = projectName, + callHistory = count + ) + ) + } + } + } + } } diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/service/MetricsService.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/service/MetricsService.kt new file mode 100644 index 00000000000..8fa1fcba824 --- /dev/null +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/service/MetricsService.kt @@ -0,0 +1,113 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.tencent.devops.openapi.service + +import com.tencent.devops.common.redis.RedisLock +import com.tencent.devops.common.redis.RedisOperation +import com.tencent.devops.openapi.dao.MetricsForApiDao +import com.tencent.devops.openapi.dao.MetricsForProjectDao +import com.tencent.devops.openapi.es.ESServiceImpl +import com.tencent.devops.openapi.pojo.MetricsApiData +import com.tencent.devops.openapi.pojo.MetricsProjectData +import java.time.Duration +import java.time.LocalTime +import org.jooq.DSLContext +import org.slf4j.LoggerFactory +import org.springframework.scheduling.annotation.Scheduled +import org.springframework.stereotype.Service + +@Service +class MetricsService( + private val dslContext: DSLContext, + private val apiDao: MetricsForApiDao, + private val projectDao: MetricsForProjectDao, + private val esServiceImpl: ESServiceImpl, + private val redisOperation: RedisOperation +) { + companion object { + private val logger = LoggerFactory.getLogger(MetricsService::class.java) + private const val ES_INDEX_CLOSE_JOB_KEY = "openapi:es:index:close:job:lock:key" + private const val EACH_HOUR = 12 + private const val MOD_HOUR = 4 + private const val EACH_DAY = 288 + private const val MOD_DAY = 10 + } + + /** + * every 5m + */ + @Scheduled(cron = "0 0/5 * * * ?") + fun job() { + logger.info("Start to openapi metrics job") + RedisLock(redisOperation, ES_INDEX_CLOSE_JOB_KEY, 60).use { + val begin = System.currentTimeMillis() + val keyMap = mutableMapOf() + apiDao.batchGet(dslContext).associateByTo(keyMap) { "${it.api}@${it.key}" } + val between = Duration.between(LocalTime.of(0, 0), LocalTime.now()).toMinutes().toInt() / 5 + val dayMod = between % EACH_DAY + esServiceImpl.executeElasticsearchQueryS(keyMap = keyMap, newDay = dayMod == MOD_DAY) + // 每小时 + if (between % EACH_HOUR == MOD_HOUR) { + logger.info("Start to openapi metrics job for 1H") + jobFor1H(keyMap) + } + // 每天 + if (dayMod == MOD_DAY) { + logger.info("Start to openapi metrics job for 1D") + jobFor24H(keyMap) + jobFor7D(keyMap) + } + apiDao.createOrUpdate(dslContext, keyMap.values.toList()) + + logger.info("execution time ${System.currentTimeMillis() - begin} millisecond") + } + } + + private fun jobFor1H(keyMap: MutableMap) { + esServiceImpl.executeElasticsearchQueryM(keyMap, "1h") { count, data -> + data.call1h = count + return@executeElasticsearchQueryM data + } + val projects = mutableListOf() + esServiceImpl.executeElasticsearchQueryP(projects) + projectDao.createOrUpdate(dslContext, projects) + } + + private fun jobFor24H(keyMap: MutableMap) { + esServiceImpl.executeElasticsearchQueryM(keyMap, "1d") { count, data -> + data.call24h = count + return@executeElasticsearchQueryM data + } + } + + private fun jobFor7D(keyMap: MutableMap) { + esServiceImpl.executeElasticsearchQueryM(keyMap, "7d") { count, data -> + data.call7d = count + return@executeElasticsearchQueryM data + } + } +} diff --git a/support-files/sql/1001_ci_openapi_ddl_mysql.sql b/support-files/sql/1001_ci_openapi_ddl_mysql.sql index ee00e0de9be..0937428db21 100644 --- a/support-files/sql/1001_ci_openapi_ddl_mysql.sql +++ b/support-files/sql/1001_ci_openapi_ddl_mysql.sql @@ -57,17 +57,15 @@ CREATE TABLE IF NOT EXISTS `T_APP_USER_INFO`( -- ---------------------------- CREATE TABLE IF NOT EXISTS `T_OPENAPI_METRICS_FOR_API`( `API` VARCHAR(64) NOT NULL COMMENT 'api接口代码', - `APP_CODE` VARCHAR(64) NOT NULL COMMENT 'APP编码', - `USER_ID` VARCHAR(64) NOT NULL COMMENT 'api请求用户', + `KEY` VARCHAR(64) NOT NULL COMMENT 'APP编码/api请求用户', `SECOND_LEVEL_CONCURRENCY` int(11) NOT NULL COMMENT '秒级并发量', `PEAK_CONCURRENCY` int(11) NOT NULL COMMENT '峰值并发量', `CALL_5M` int(11) NOT NULL COMMENT '5min调用量', `CALL_1H` int(11) NOT NULL COMMENT '1h调用量', `CALL_24H` int(11) NOT NULL COMMENT '24h调用量', `CALL_7D` int(11) NOT NULL COMMENT '7d调用量', - INDEX `IDX_APP` (`APP_CODE`), - INDEX `IDX_USER_ID` (`USER_ID`), - UNIQUE INDEX `IDX_APP_USER` (`API`, `APP_CODE`, `USER_ID`) + INDEX `IDX_KEY` (`KEY`), + UNIQUE INDEX `IDX_API_KEY` (`API`, `KEY`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='接口维度度量表'; -- ---------------------------- @@ -76,13 +74,11 @@ CREATE TABLE IF NOT EXISTS `T_OPENAPI_METRICS_FOR_API`( CREATE TABLE IF NOT EXISTS `T_OPENAPI_METRICS_FOR_PROJECT`( `PROJECT` VARCHAR(64) NOT NULL COMMENT '项目id', `API` VARCHAR(64) NOT NULL COMMENT 'api接口代码', - `APP_CODE` VARCHAR(64) NOT NULL COMMENT 'APP编码', - `USER_ID` VARCHAR(64) NOT NULL COMMENT 'api请求用户', - `CALL_LAST_DAY` int(11) NOT NULL COMMENT '昨日', + `KEY` VARCHAR(64) NOT NULL COMMENT 'APP编码/api请求用户', + `CALL_HISTORY` int(11) NOT NULL COMMENT '历史累计调用', INDEX `IDX_API` (`API`), - INDEX `IDX_APP` (`APP_CODE`), - INDEX `IDX_USER_ID` (`USER_ID`), - UNIQUE INDEX `IDX_APP_USER` (`PROJECT`, `API`, `APP_CODE`, `USER_ID`) + INDEX `IDX_KEY` (`KEY`), + UNIQUE INDEX `IDX_APP_USER` (`PROJECT`, `API`, `KEY`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='项目维度度量表'; SET FOREIGN_KEY_CHECKS = 1; From 99def2ad4c7542e25c39f5f3e3007effc702a9ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?yongyiduan=28=E6=AE=B5=E6=B0=B8=E5=84=84=29?= Date: Tue, 7 Nov 2023 17:04:50 +0800 Subject: [PATCH 13/31] =?UTF-8?q?feat:=20openapi=E6=96=B0=E5=A2=9E?= =?UTF-8?q?=E5=BA=A6=E9=87=8F=E8=83=BD=E5=8A=9B=20#9638?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tencent/devops/openapi/es/ESServiceImpl.kt | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESServiceImpl.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESServiceImpl.kt index fd4c5a3753c..42dcefc5a92 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESServiceImpl.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESServiceImpl.kt @@ -57,6 +57,8 @@ import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInter import org.elasticsearch.search.aggregations.bucket.terms.Terms import org.elasticsearch.search.aggregations.metrics.Avg import org.elasticsearch.search.aggregations.metrics.Max +import org.elasticsearch.search.aggregations.pipeline.AvgBucketPipelineAggregationBuilder +import org.elasticsearch.search.aggregations.pipeline.MaxBucketPipelineAggregationBuilder import org.elasticsearch.search.builder.SearchSourceBuilder import org.slf4j.LoggerFactory @@ -328,8 +330,18 @@ class ESServiceImpl constructor( AggregationBuilders.count("count").field(ESMessage::timestamp.name) ) ) - .subAggregation(AggregationBuilders.max("max_concurrency").field("count")) - .subAggregation(AggregationBuilders.avg("avg_concurrency").field("count")) + .subAggregation( + MaxBucketPipelineAggregationBuilder( + "max_concurrency", + "concurrency>count" + ) + ) + .subAggregation( + AvgBucketPipelineAggregationBuilder( + "avg_concurrency", + "concurrency>count" + ) + ) ) ) // 不返回任何文档，只返回聚合结果。 From 47975d52628ac6f2e481d77a9f154a7f04cb6e26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?yongyiduan=28=E6=AE=B5=E6=B0=B8=E5=84=84=29?= Date: Tue, 7 Nov 2023 17:13:46 +0800 Subject: [PATCH 14/31] =?UTF-8?q?feat:=20openapi=E6=96=B0=E5=A2=9E?= =?UTF-8?q?=E5=BA=A6=E9=87=8F=E8=83=BD=E5=8A=9B=20#9638?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kotlin/com/tencent/devops/openapi/es/ESServiceImpl.kt | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESServiceImpl.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESServiceImpl.kt index 42dcefc5a92..0b469fd520d 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESServiceImpl.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESServiceImpl.kt @@ -55,9 +55,8 @@ import org.elasticsearch.index.query.QueryBuilders import org.elasticsearch.search.aggregations.AggregationBuilders import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval import org.elasticsearch.search.aggregations.bucket.terms.Terms -import org.elasticsearch.search.aggregations.metrics.Avg -import org.elasticsearch.search.aggregations.metrics.Max import org.elasticsearch.search.aggregations.pipeline.AvgBucketPipelineAggregationBuilder +import org.elasticsearch.search.aggregations.pipeline.BucketMetricValue import org.elasticsearch.search.aggregations.pipeline.MaxBucketPipelineAggregationBuilder import org.elasticsearch.search.builder.SearchSourceBuilder import org.slf4j.LoggerFactory @@ -361,8 +360,8 @@ class ESServiceImpl constructor( for (keyBucket in keyAggregation.buckets) { val keyName: String = keyBucket.keyAsString - val max = keyBucket.aggregations.get("max_concurrency").value.toInt() - val avg = keyBucket.aggregations.get("avg_concurrency").value.toInt() + val max = keyBucket.aggregations.get("max_concurrency").value().toInt() + val avg = keyBucket.aggregations.get("avg_concurrency").value().toInt() val count = keyBucket.docCount.toInt() keyMap["$apiName@$keyName"]?.apply { secondLevelConcurrency = avg From 88abc697993bcad1eacb6e4119aadaf0d2895156 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?yongyiduan=28=E6=AE=B5=E6=B0=B8=E5=84=84=29?= Date: Tue, 7 Nov 2023 17:22:20 +0800 Subject: [PATCH 15/31] =?UTF-8?q?feat:=20openapi=E6=96=B0=E5=A2=9E?= =?UTF-8?q?=E5=BA=A6=E9=87=8F=E8=83=BD=E5=8A=9B=20#9638?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/kotlin/com/tencent/devops/openapi/es/ESServiceImpl.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESServiceImpl.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESServiceImpl.kt index 0b469fd520d..2334bad1914 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESServiceImpl.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESServiceImpl.kt @@ -58,6 +58,7 @@ import org.elasticsearch.search.aggregations.bucket.terms.Terms import org.elasticsearch.search.aggregations.pipeline.AvgBucketPipelineAggregationBuilder import org.elasticsearch.search.aggregations.pipeline.BucketMetricValue import org.elasticsearch.search.aggregations.pipeline.MaxBucketPipelineAggregationBuilder +import org.elasticsearch.search.aggregations.pipeline.SimpleValue import org.elasticsearch.search.builder.SearchSourceBuilder import org.slf4j.LoggerFactory @@ -361,7 +362,7 @@ class ESServiceImpl constructor( for (keyBucket in keyAggregation.buckets) { val keyName: String = keyBucket.keyAsString val max = keyBucket.aggregations.get("max_concurrency").value().toInt() - val avg = keyBucket.aggregations.get("avg_concurrency").value().toInt() + val avg = keyBucket.aggregations.get("avg_concurrency").value().toInt() val count = keyBucket.docCount.toInt() keyMap["$apiName@$keyName"]?.apply { secondLevelConcurrency = avg From 9921ee4870fe47942b2803a870fb2c50462ee094 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?yongyiduan=28=E6=AE=B5=E6=B0=B8=E5=84=84=29?= Date: Tue, 7 Nov 2023 17:32:48 +0800 Subject: [PATCH 16/31] =?UTF-8?q?feat:=20openapi=E6=96=B0=E5=A2=9E?= =?UTF-8?q?=E5=BA=A6=E9=87=8F=E8=83=BD=E5=8A=9B=20#9638?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kotlin/com/tencent/devops/openapi/service/MetricsService.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/service/MetricsService.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/service/MetricsService.kt index 8fa1fcba824..df1ce2282ae 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/service/MetricsService.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/service/MetricsService.kt @@ -69,6 +69,7 @@ class MetricsService( apiDao.batchGet(dslContext).associateByTo(keyMap) { "${it.api}@${it.key}" } val between = Duration.between(LocalTime.of(0, 0), LocalTime.now()).toMinutes().toInt() / 5 val dayMod = between % EACH_DAY + logger.info("openapi metrics job dayMod=$dayMod") esServiceImpl.executeElasticsearchQueryS(keyMap = keyMap, newDay = dayMod == MOD_DAY) // 每小时 if (between % EACH_HOUR == MOD_HOUR) { From a099b288d5b4a2a689d2cd2014dea8206eb9aca0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?yongyiduan=28=E6=AE=B5=E6=B0=B8=E5=84=84=29?= Date: Tue, 7 Nov 2023 18:31:37 +0800 Subject: [PATCH 17/31] =?UTF-8?q?feat:=20openapi=E6=96=B0=E5=A2=9E?= =?UTF-8?q?=E5=BA=A6=E9=87=8F=E8=83=BD=E5=8A=9B=20#9638?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/kotlin/com/tencent/devops/openapi/es/ESServiceImpl.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESServiceImpl.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESServiceImpl.kt index 2334bad1914..cb62c553dcd 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESServiceImpl.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESServiceImpl.kt @@ -494,7 +494,7 @@ class ESServiceImpl constructor( for (keyBucket in keyAggregation.buckets) { val keyName: String = keyBucket.keyAsString - val projectAggregation: Terms = apiBucket.aggregations.get(ESMessage::projectId.name) + val projectAggregation: Terms = keyBucket.aggregations.get(ESMessage::projectId.name) for (projectBucket in projectAggregation.buckets) { val projectName: String = projectBucket.keyAsString From d35cd343f377724d8218155054a2117992b7130a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?yongyiduan=28=E6=AE=B5=E6=B0=B8=E5=84=84=29?= Date: Wed, 8 Nov 2023 16:38:39 +0800 Subject: [PATCH 18/31] =?UTF-8?q?feat:=20openapi=E6=96=B0=E5=A2=9E?= =?UTF-8?q?=E5=BA=A6=E9=87=8F=E8=83=BD=E5=8A=9B=20#9638?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../devops/openapi/aspect/ApiAspect.kt | 33 +++++++++++-------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/aspect/ApiAspect.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/aspect/ApiAspect.kt index ad694ccbdb6..3fb33aa38f1 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/aspect/ApiAspect.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/aspect/ApiAspect.kt @@ -110,21 +110,26 @@ class ApiAspect( else -> Unit } } - val request = (RequestContextHolder.currentRequestAttributes() as ServletRequestAttributes).request - val apiType = apigwType?.split("-")?.getOrNull(1) ?: "" - esServiceImpl.addMessage( - ESMessage( - api = getApiTag(jp = jp, apiType = apiType), - key = when { - apiType.isBlank() -> "null" - apiType.contains("user") -> "user:$userId" - else -> "app:$appCode" - }, - projectId = projectId ?: "", - path = request.requestURI, - timestamp = System.currentTimeMillis() + + kotlin.runCatching { + val request = (RequestContextHolder.currentRequestAttributes() as ServletRequestAttributes).request + val apiType = apigwType?.split("-")?.getOrNull(1) ?: "" + esServiceImpl.addMessage( + ESMessage( + api = getApiTag(jp = jp, apiType = apiType), + key = when { + apiType.isBlank() -> "null" + apiType.contains("user") -> "user:$userId" + else -> "app:$appCode" + }, + projectId = projectId ?: "", + path = request.requestURI, + timestamp = System.currentTimeMillis() + ) ) - ) + }.onFailure { + logger.error("es add message error ${it.message}", it) + } if (logger.isDebugEnabled) { From b0e21a0c35f932af02c0916b832a428a43744ec2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?yongyiduan=28=E6=AE=B5=E6=B0=B8=E5=84=84=29?= Date: Thu, 9 Nov 2023 10:12:51 +0800 Subject: [PATCH 19/31] =?UTF-8?q?feat:=20openapi=E6=96=B0=E5=A2=9E?= =?UTF-8?q?=E5=BA=A6=E9=87=8F=E8=83=BD=E5=8A=9B=20#9638?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/tencent/devops/openapi/es/ESServiceImpl.kt | 13 +++++++------ .../devops/openapi/es/config/ESAutoConfiguration.kt | 4 ++++ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESServiceImpl.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESServiceImpl.kt index cb62c553dcd..76c38b1ce69 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESServiceImpl.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESServiceImpl.kt @@ -33,6 +33,7 @@ import com.tencent.devops.common.es.ESClient import com.tencent.devops.common.es.client.LogClient import com.tencent.devops.common.redis.RedisLock import com.tencent.devops.common.redis.RedisOperation +import com.tencent.devops.openapi.es.config.ESAutoConfiguration import com.tencent.devops.openapi.es.mq.ESEvent import com.tencent.devops.openapi.es.mq.MQDispatcher import com.tencent.devops.openapi.pojo.MetricsApiData @@ -65,15 +66,13 @@ import org.slf4j.LoggerFactory class ESServiceImpl constructor( private val logClient: LogClient, private val redisOperation: RedisOperation, - private val dispatcher: MQDispatcher + private val dispatcher: MQDispatcher, + private val configuration: ESAutoConfiguration ) { companion object { private val logger = LoggerFactory.getLogger(ESServiceImpl::class.java) - private const val LONG_SEARCH_TIME: Long = 64000 - private const val SHORT_SEARCH_TIME: Long = 32000 private const val SEARCH_TIMEOUT_SECONDS = 60L - private const val SEARCH_FRAGMENT_SIZE = 100000 private const val INDEX_CACHE_MAX_SIZE = 10L private const val INDEX_CACHE_EXPIRE_DAY = 1L private const val INDEX_LOCK_EXPIRE_SECONDS = 10L @@ -93,7 +92,9 @@ class ESServiceImpl constructor( private val executor = Executors.newCachedThreadPool() init { - executor.submit(BulkSend()) + repeat(configuration.consumerCount) { + executor.submit(BulkSend()) + } } fun addMessage(message: ESMessage) { @@ -112,7 +113,7 @@ class ESServiceImpl constructor( val buf = mutableListOf() override fun run() { while (true) { - val message = queue.poll() ?: continue + val message = queue.take() ?: continue buf.add(message) if (buf.size == BULK_BUFFER_SIZE) { val currentEpoch = System.currentTimeMillis() diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/config/ESAutoConfiguration.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/config/ESAutoConfiguration.kt index fe77eaf9493..396eac25e44 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/config/ESAutoConfiguration.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/config/ESAutoConfiguration.kt @@ -34,6 +34,7 @@ import com.tencent.devops.common.redis.RedisOperation import com.tencent.devops.openapi.es.ESServiceImpl import com.tencent.devops.openapi.es.mq.MQDispatcher import org.springframework.beans.factory.annotation.Autowired +import org.springframework.beans.factory.annotation.Value import org.springframework.boot.autoconfigure.AutoConfigureAfter import org.springframework.boot.autoconfigure.AutoConfigureOrder import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty @@ -49,6 +50,9 @@ import org.springframework.core.Ordered @AutoConfigureAfter(ESAutoConfiguration::class) @EnableConfigurationProperties(ESProperties::class) class ESAutoConfiguration { + + @Value("\${log.elasticsearch.consumerCount:1}") + val consumerCount: Int = 1 @Bean fun esLogService( @Autowired logESClient: LogClient, From 67b0dc89d7725aead80d1fd5fd1e697da8c11c35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?yongyiduan=28=E6=AE=B5=E6=B0=B8=E5=84=84=29?= Date: Thu, 9 Nov 2023 10:24:40 +0800 Subject: [PATCH 20/31] =?UTF-8?q?feat:=20openapi=E6=96=B0=E5=A2=9E?= =?UTF-8?q?=E5=BA=A6=E9=87=8F=E8=83=BD=E5=8A=9B=20#9638?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kotlin/com/tencent/devops/openapi/dao/MetricsForApiDao.kt | 2 +- .../com/tencent/devops/openapi/dao/MetricsForProjectDao.kt | 2 +- .../main/kotlin/com/tencent/devops/openapi/es/ESServiceImpl.kt | 1 - .../com/tencent/devops/openapi/es/mq/MQListenerService.kt | 1 - 4 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/dao/MetricsForApiDao.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/dao/MetricsForApiDao.kt index 59e6ac76575..195eb404bc2 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/dao/MetricsForApiDao.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/dao/MetricsForApiDao.kt @@ -38,7 +38,7 @@ import org.springframework.stereotype.Repository class MetricsForApiDao { fun createOrUpdate( dslContext: DSLContext, - metricsApis: List, + metricsApis: List ): Int { return with(TOpenapiMetricsForApi.T_OPENAPI_METRICS_FOR_API) { dslContext.batch( diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/dao/MetricsForProjectDao.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/dao/MetricsForProjectDao.kt index 18bbd5ad303..6459c454b51 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/dao/MetricsForProjectDao.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/dao/MetricsForProjectDao.kt @@ -37,7 +37,7 @@ import org.springframework.stereotype.Repository class MetricsForProjectDao { fun createOrUpdate( dslContext: DSLContext, - metricsApis: List, + metricsApis: List ): Int { return with(TOpenapiMetricsForProject.T_OPENAPI_METRICS_FOR_PROJECT) { dslContext.batch( diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESServiceImpl.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESServiceImpl.kt index 76c38b1ce69..1977d992d2a 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESServiceImpl.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESServiceImpl.kt @@ -298,7 +298,6 @@ class ESServiceImpl constructor( return builder.build() } - /* * 每5分钟执行一次 * 根据"api"字段进行分组，然后在每个分组内根据"key"字段进行进一步的分组。 diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/mq/MQListenerService.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/mq/MQListenerService.kt index 27c27240660..92b2e4eb1c6 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/mq/MQListenerService.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/mq/MQListenerService.kt @@ -61,7 +61,6 @@ class MQListenerService @Autowired constructor( } } - companion object { private val logger = LoggerFactory.getLogger(MQListenerService::class.java) } From 42b2a2b366114b90b9e5afa7d96baed7c660d157 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?yongyiduan=28=E6=AE=B5=E6=B0=B8=E5=84=84=29?= Date: Thu, 9 Nov 2023 11:07:24 +0800 Subject: [PATCH 21/31] =?UTF-8?q?feat:=20openapi=E6=96=B0=E5=A2=9E?= =?UTF-8?q?=E5=BA=A6=E9=87=8F=E8=83=BD=E5=8A=9B=20#9638?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../devops/openapi/dao/MetricsForApiDao.kt | 22 +++++++++++-------- .../devops/openapi/es/ESServiceImpl.kt | 2 +- .../devops/openapi/es/IndexNameUtils.kt | 4 ++++ .../devops/openapi/service/MetricsService.kt | 15 ++++++++++--- 4 files changed, 30 insertions(+), 13 deletions(-) diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/dao/MetricsForApiDao.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/dao/MetricsForApiDao.kt index 195eb404bc2..76c5ec30a64 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/dao/MetricsForApiDao.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/dao/MetricsForApiDao.kt @@ -31,14 +31,15 @@ import com.tencent.devops.common.db.utils.skipCheck import com.tencent.devops.model.openapi.tables.TOpenapiMetricsForApi import com.tencent.devops.openapi.pojo.MetricsApiData import org.jooq.DSLContext -import org.jooq.InsertOnDuplicateSetMoreStep import org.springframework.stereotype.Repository @Repository class MetricsForApiDao { fun createOrUpdate( dslContext: DSLContext, - metricsApis: List + metricsApis: List, + perHour: Boolean, + perDay: Boolean ): Int { return with(TOpenapiMetricsForApi.T_OPENAPI_METRICS_FOR_API) { dslContext.batch( @@ -64,24 +65,27 @@ class MetricsForApiDao { metricsApi.call7d ?: 0 ).onDuplicateKeyUpdate() .let { u -> - metricsApi.secondLevelConcurrency?.let { i -> u.set(SECOND_LEVEL_CONCURRENCY, i) } ?: u + metricsApi.secondLevelConcurrency?.let { i -> + u.set(SECOND_LEVEL_CONCURRENCY, i) + } ?: u.set(SECOND_LEVEL_CONCURRENCY, 0) } .let { u -> - metricsApi.peakConcurrency?.let { i -> u.set(PEAK_CONCURRENCY, i) } ?: u + metricsApi.peakConcurrency?.let { i -> + u.set(PEAK_CONCURRENCY, i) + } ?: u.set(PEAK_CONCURRENCY, 0) } .let { u -> - metricsApi.call5m?.let { i -> u.set(CALL_5M, i) } ?: u + metricsApi.call5m?.let { i -> u.set(CALL_5M, i) } ?: u.set(CALL_5M, 0) } .let { u -> - metricsApi.call1h?.let { i -> u.set(CALL_1H, i) } ?: u + metricsApi.call1h?.let { i -> u.set(CALL_1H, i) } ?: if (perHour) u.set(CALL_1H, 0) else u } .let { u -> - metricsApi.call24h?.let { i -> u.set(CALL_24H, i) } ?: u + metricsApi.call24h?.let { i -> u.set(CALL_24H, i) } ?: if (perDay) u.set(CALL_24H, 0) else u } .let { u -> - metricsApi.call7d?.let { i -> u.set(CALL_7D, i) } ?: u + metricsApi.call7d?.let { i -> u.set(CALL_7D, i) } ?: if (perDay) u.set(CALL_7D, 0) else u } - .let { if (it is InsertOnDuplicateSetMoreStep) it else return@mapNotNull null } } ).execute().sum() } diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESServiceImpl.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESServiceImpl.kt index 1977d992d2a..44746a64dc4 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESServiceImpl.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESServiceImpl.kt @@ -391,7 +391,7 @@ class ESServiceImpl constructor( time: String, f: (count: Int, data: MetricsApiData) -> MetricsApiData ) { - val indexName = IndexNameUtils.getIndexName() + val indexName = IndexNameUtils.getIndexNamePrefix() val searchRequest = SearchRequest(indexName) .source( diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/IndexNameUtils.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/IndexNameUtils.kt index cc86cccf5e5..a2f1a294ca1 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/IndexNameUtils.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/IndexNameUtils.kt @@ -37,6 +37,10 @@ object IndexNameUtils { return LOG_INDEX_PREFIX + formatter.format(LocalDateTime.now()) } + fun getIndexNamePrefix(): String { + return "$LOG_INDEX_PREFIX*" + } + fun getNextIndexName(): String { val formatter = DateTimeFormatter.ofPattern(LOG_INDEX_DATE_FORMAT) return LOG_INDEX_PREFIX + formatter.format(LocalDateTime.now().plusDays(1)) diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/service/MetricsService.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/service/MetricsService.kt index df1ce2282ae..32d19d74379 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/service/MetricsService.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/service/MetricsService.kt @@ -63,16 +63,20 @@ class MetricsService( @Scheduled(cron = "0 0/5 * * * ?") fun job() { logger.info("Start to openapi metrics job") - RedisLock(redisOperation, ES_INDEX_CLOSE_JOB_KEY, 60).use { + RedisLock(redisOperation, ES_INDEX_CLOSE_JOB_KEY, 60).run { + if (!this.tryLock()) { + return + } val begin = System.currentTimeMillis() val keyMap = mutableMapOf() apiDao.batchGet(dslContext).associateByTo(keyMap) { "${it.api}@${it.key}" } val between = Duration.between(LocalTime.of(0, 0), LocalTime.now()).toMinutes().toInt() / 5 + val hourMod = between % EACH_HOUR val dayMod = between % EACH_DAY logger.info("openapi metrics job dayMod=$dayMod") esServiceImpl.executeElasticsearchQueryS(keyMap = keyMap, newDay = dayMod == MOD_DAY) // 每小时 - if (between % EACH_HOUR == MOD_HOUR) { + if (hourMod == MOD_HOUR) { logger.info("Start to openapi metrics job for 1H") jobFor1H(keyMap) } @@ -82,7 +86,12 @@ class MetricsService( jobFor24H(keyMap) jobFor7D(keyMap) } - apiDao.createOrUpdate(dslContext, keyMap.values.toList()) + apiDao.createOrUpdate( + dslContext = dslContext, + metricsApis = keyMap.values.toList(), + perHour = hourMod == MOD_HOUR, + perDay = dayMod == MOD_DAY + ) logger.info("execution time ${System.currentTimeMillis() - begin} millisecond") } From 373c1fb5eb5750f3b01f845f5b47328f78a6b756 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?yongyiduan=28=E6=AE=B5=E6=B0=B8=E5=84=84=29?= Date: Thu, 9 Nov 2023 11:10:51 +0800 Subject: [PATCH 22/31] =?UTF-8?q?feat:=20openapi=E6=96=B0=E5=A2=9E?= =?UTF-8?q?=E5=BA=A6=E9=87=8F=E8=83=BD=E5=8A=9B=20#9638?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tencent/devops/openapi/es/config/ESAutoConfiguration.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/config/ESAutoConfiguration.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/config/ESAutoConfiguration.kt index 396eac25e44..ab565d94b66 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/config/ESAutoConfiguration.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/config/ESAutoConfiguration.kt @@ -62,7 +62,8 @@ class ESAutoConfiguration { return ESServiceImpl( logClient = logESClient, redisOperation = redisOperation, - dispatcher = openapiMQDispatcher + dispatcher = openapiMQDispatcher, + configuration = this ) } } From 967444a8ea28c37d264481d3b234d5785af362c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?yongyiduan=28=E6=AE=B5=E6=B0=B8=E5=84=84=29?= Date: Thu, 9 Nov 2023 11:31:37 +0800 Subject: [PATCH 23/31] =?UTF-8?q?feat:=20openapi=E6=96=B0=E5=A2=9E?= =?UTF-8?q?=E5=BA=A6=E9=87=8F=E8=83=BD=E5=8A=9B=20#9638?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kotlin/com/tencent/devops/openapi/dao/MetricsForApiDao.kt | 1 + .../main/kotlin/com/tencent/devops/openapi/es/ESServiceImpl.kt | 1 + 2 files changed, 2 insertions(+) diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/dao/MetricsForApiDao.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/dao/MetricsForApiDao.kt index 76c5ec30a64..efef3cb6a39 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/dao/MetricsForApiDao.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/dao/MetricsForApiDao.kt @@ -35,6 +35,7 @@ import org.springframework.stereotype.Repository @Repository class MetricsForApiDao { + @Suppress("ComplexMethod") fun createOrUpdate( dslContext: DSLContext, metricsApis: List, diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESServiceImpl.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESServiceImpl.kt index 44746a64dc4..3b2943d57a5 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESServiceImpl.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESServiceImpl.kt @@ -63,6 +63,7 @@ import org.elasticsearch.search.aggregations.pipeline.SimpleValue import org.elasticsearch.search.builder.SearchSourceBuilder import org.slf4j.LoggerFactory +@Suppress("NestedBlockDepth") class ESServiceImpl constructor( private val logClient: LogClient, private val redisOperation: RedisOperation, From c04517f90e95bffa1de5cbc890c7dcadc52b2539 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?yongyiduan=28=E6=AE=B5=E6=B0=B8=E5=84=84=29?= Date: Thu, 9 Nov 2023 13:09:04 +0800 Subject: [PATCH 24/31] =?UTF-8?q?feat:=20openapi=E6=96=B0=E5=A2=9E?= =?UTF-8?q?=E5=BA=A6=E9=87=8F=E8=83=BD=E5=8A=9B=20#9638?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../devops/openapi/aspect/ApiAspect.kt | 34 +++++++++-------- .../tencent/devops/openapi/es/IESService.kt | 38 +++++++++++++++++++ .../openapi/{service => es}/MetricsService.kt | 7 ++-- .../openapi/es/config/ESAutoConfiguration.kt | 37 +++++++++++++++++- .../openapi/es/config/MQConfiguration.kt | 2 + .../openapi/es/impl/DefaultESServiceImpl.kt | 38 +++++++++++++++++++ .../openapi/es/{ => impl}/ESServiceImpl.kt | 14 +++++-- .../devops/openapi/es/mq/MQListenerService.kt | 4 +- 8 files changed, 146 insertions(+), 28 deletions(-) create mode 100644 src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/IESService.kt rename src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/{service => es}/MetricsService.kt (97%) create mode 100644 src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/impl/DefaultESServiceImpl.kt rename src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/{ => impl}/ESServiceImpl.kt (98%) diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/aspect/ApiAspect.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/aspect/ApiAspect.kt index 3fb33aa38f1..a1e6d3ca51e 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/aspect/ApiAspect.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/aspect/ApiAspect.kt @@ -39,7 +39,7 @@ import com.tencent.devops.common.web.utils.I18nUtil import com.tencent.devops.openapi.IgnoreProjectId import com.tencent.devops.openapi.constant.OpenAPIMessageCode.PARAM_VERIFY_FAIL import com.tencent.devops.openapi.es.ESMessage -import com.tencent.devops.openapi.es.ESServiceImpl +import com.tencent.devops.openapi.es.IESService import com.tencent.devops.openapi.service.OpenapiPermissionService import com.tencent.devops.openapi.service.op.AppCodeService import com.tencent.devops.openapi.utils.ApiGatewayUtil @@ -65,7 +65,7 @@ class ApiAspect( private val redisOperation: RedisOperation, private val bkTag: BkTag, private val permissionService: OpenapiPermissionService, - private val esServiceImpl: ESServiceImpl + private val esService: IESService ) { companion object { @@ -112,21 +112,23 @@ class ApiAspect( } kotlin.runCatching { - val request = (RequestContextHolder.currentRequestAttributes() as ServletRequestAttributes).request - val apiType = apigwType?.split("-")?.getOrNull(1) ?: "" - esServiceImpl.addMessage( - ESMessage( - api = getApiTag(jp = jp, apiType = apiType), - key = when { - apiType.isBlank() -> "null" - apiType.contains("user") -> "user:$userId" - else -> "app:$appCode" - }, - projectId = projectId ?: "", - path = request.requestURI, - timestamp = System.currentTimeMillis() + if (esService.esReady()) { + val request = (RequestContextHolder.currentRequestAttributes() as ServletRequestAttributes).request + val apiType = apigwType?.split("-")?.getOrNull(1) ?: "" + esService.addMessage( + ESMessage( + api = getApiTag(jp = jp, apiType = apiType), + key = when { + apiType.isBlank() -> "null" + apiType.contains("user") -> "user:$userId" + else -> "app:$appCode" + }, + projectId = projectId ?: "", + path = request.requestURI, + timestamp = System.currentTimeMillis() + ) ) - ) + } }.onFailure { logger.error("es add message error ${it.message}", it) } diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/IESService.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/IESService.kt new file mode 100644 index 00000000000..590164e8f62 --- /dev/null +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/IESService.kt @@ -0,0 +1,38 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.openapi.es + +import com.tencent.devops.openapi.es.mq.ESEvent + +interface IESService { + + fun addMessage(message: ESMessage) + fun esAddMessage(event: ESEvent) + + fun esReady() = false +} diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/service/MetricsService.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/MetricsService.kt similarity index 97% rename from src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/service/MetricsService.kt rename to src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/MetricsService.kt index 32d19d74379..bdefd873fb9 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/service/MetricsService.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/MetricsService.kt @@ -24,13 +24,13 @@ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.devops.openapi.service +package com.tencent.devops.openapi.es import com.tencent.devops.common.redis.RedisLock import com.tencent.devops.common.redis.RedisOperation import com.tencent.devops.openapi.dao.MetricsForApiDao import com.tencent.devops.openapi.dao.MetricsForProjectDao -import com.tencent.devops.openapi.es.ESServiceImpl +import com.tencent.devops.openapi.es.impl.ESServiceImpl import com.tencent.devops.openapi.pojo.MetricsApiData import com.tencent.devops.openapi.pojo.MetricsProjectData import java.time.Duration @@ -38,9 +38,8 @@ import java.time.LocalTime import org.jooq.DSLContext import org.slf4j.LoggerFactory import org.springframework.scheduling.annotation.Scheduled -import org.springframework.stereotype.Service -@Service + class MetricsService( private val dslContext: DSLContext, private val apiDao: MetricsForApiDao, diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/config/ESAutoConfiguration.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/config/ESAutoConfiguration.kt index ab565d94b66..1f5564d01f8 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/config/ESAutoConfiguration.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/config/ESAutoConfiguration.kt @@ -31,12 +31,19 @@ import com.tencent.devops.common.es.ESAutoConfiguration import com.tencent.devops.common.es.ESProperties import com.tencent.devops.common.es.client.LogClient import com.tencent.devops.common.redis.RedisOperation -import com.tencent.devops.openapi.es.ESServiceImpl +import com.tencent.devops.openapi.dao.MetricsForApiDao +import com.tencent.devops.openapi.dao.MetricsForProjectDao +import com.tencent.devops.openapi.es.impl.DefaultESServiceImpl +import com.tencent.devops.openapi.es.impl.ESServiceImpl +import com.tencent.devops.openapi.es.IESService import com.tencent.devops.openapi.es.mq.MQDispatcher +import com.tencent.devops.openapi.es.MetricsService +import org.jooq.DSLContext import org.springframework.beans.factory.annotation.Autowired import org.springframework.beans.factory.annotation.Value import org.springframework.boot.autoconfigure.AutoConfigureAfter import org.springframework.boot.autoconfigure.AutoConfigureOrder +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty import org.springframework.boot.context.properties.EnableConfigurationProperties import org.springframework.context.annotation.Bean @@ -45,7 +52,6 @@ import org.springframework.core.Ordered @Suppress("ALL") @Configuration -@ConditionalOnProperty(prefix = "log.storage", name = ["type"], havingValue = "elasticsearch") @AutoConfigureOrder(Ordered.HIGHEST_PRECEDENCE) @AutoConfigureAfter(ESAutoConfiguration::class) @EnableConfigurationProperties(ESProperties::class) @@ -53,7 +59,9 @@ class ESAutoConfiguration { @Value("\${log.elasticsearch.consumerCount:1}") val consumerCount: Int = 1 + @Bean + @ConditionalOnProperty(prefix = "log.storage", name = ["type"], havingValue = "elasticsearch") fun esLogService( @Autowired logESClient: LogClient, @Autowired redisOperation: RedisOperation, @@ -66,4 +74,29 @@ class ESAutoConfiguration { configuration = this ) } + + @Bean + @ConditionalOnProperty(prefix = "log.storage", name = ["type"], havingValue = "elasticsearch") + fun metricsService( + @Autowired dslContext: DSLContext, + @Autowired apiDao: MetricsForApiDao, + @Autowired projectDao: MetricsForProjectDao, + @Autowired esServiceImpl: ESServiceImpl, + @Autowired redisOperation: RedisOperation + ): MetricsService { + return MetricsService( + dslContext = dslContext, + apiDao = apiDao, + projectDao = projectDao, + esServiceImpl = esServiceImpl, + redisOperation = redisOperation + ) + } + + @Bean + @ConditionalOnMissingBean(IESService::class) + fun defaultLogService( + ): DefaultESServiceImpl { + return DefaultESServiceImpl() + } } diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/config/MQConfiguration.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/config/MQConfiguration.kt index 01933c5d149..941fc7a6882 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/config/MQConfiguration.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/config/MQConfiguration.kt @@ -34,6 +34,7 @@ import com.tencent.devops.openapi.es.mq.MQListenerService import java.util.function.Consumer import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.autoconfigure.AutoConfigureOrder +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty import org.springframework.boot.autoconfigure.condition.ConditionalOnWebApplication import org.springframework.cloud.stream.function.StreamBridge import org.springframework.context.annotation.Bean @@ -43,6 +44,7 @@ import org.springframework.messaging.Message @Configuration @ConditionalOnWebApplication +@ConditionalOnProperty(prefix = "log.storage", name = ["type"], havingValue = "elasticsearch") @AutoConfigureOrder(Ordered.LOWEST_PRECEDENCE) class MQConfiguration @Autowired constructor() { diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/impl/DefaultESServiceImpl.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/impl/DefaultESServiceImpl.kt new file mode 100644 index 00000000000..62175d3a28f --- /dev/null +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/impl/DefaultESServiceImpl.kt @@ -0,0 +1,38 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.openapi.es.impl + +import com.tencent.devops.openapi.es.ESMessage +import com.tencent.devops.openapi.es.IESService +import com.tencent.devops.openapi.es.mq.ESEvent + +class DefaultESServiceImpl : IESService { + + override fun addMessage(message: ESMessage) {} + override fun esAddMessage(event: ESEvent) {} +} diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESServiceImpl.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/impl/ESServiceImpl.kt similarity index 98% rename from src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESServiceImpl.kt rename to src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/impl/ESServiceImpl.kt index 3b2943d57a5..5f587cac6b2 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/ESServiceImpl.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/impl/ESServiceImpl.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.devops.openapi.es +package com.tencent.devops.openapi.es.impl import com.github.benmanes.caffeine.cache.Caffeine import com.tencent.devops.common.api.exception.ExecuteException @@ -33,6 +33,10 @@ import com.tencent.devops.common.es.ESClient import com.tencent.devops.common.es.client.LogClient import com.tencent.devops.common.redis.RedisLock import com.tencent.devops.common.redis.RedisOperation +import com.tencent.devops.openapi.es.ESIndexUtils +import com.tencent.devops.openapi.es.ESMessage +import com.tencent.devops.openapi.es.IESService +import com.tencent.devops.openapi.es.IndexNameUtils import com.tencent.devops.openapi.es.config.ESAutoConfiguration import com.tencent.devops.openapi.es.mq.ESEvent import com.tencent.devops.openapi.es.mq.MQDispatcher @@ -69,7 +73,7 @@ class ESServiceImpl constructor( private val redisOperation: RedisOperation, private val dispatcher: MQDispatcher, private val configuration: ESAutoConfiguration -) { +) : IESService { companion object { private val logger = LoggerFactory.getLogger(ESServiceImpl::class.java) @@ -98,7 +102,7 @@ class ESServiceImpl constructor( } } - fun addMessage(message: ESMessage) { + override fun addMessage(message: ESMessage) { if (queue.size >= maxQueueSize) { dispatcher.dispatchEvent(ESEvent(message)) // 将消息推送到es } else { @@ -106,7 +110,9 @@ class ESServiceImpl constructor( } } - fun esAddMessage(event: ESEvent) { + override fun esReady() = true + + override fun esAddMessage(event: ESEvent) { queue.put(event.logs) } diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/mq/MQListenerService.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/mq/MQListenerService.kt index 92b2e4eb1c6..a5c4e2c833f 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/mq/MQListenerService.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/mq/MQListenerService.kt @@ -27,14 +27,14 @@ package com.tencent.devops.openapi.es.mq -import com.tencent.devops.openapi.es.ESServiceImpl +import com.tencent.devops.openapi.es.IESService import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Component @Component class MQListenerService @Autowired constructor( - private val logService: ESServiceImpl, + private val logService: IESService, private val dispatcher: MQDispatcher ) { From 5f2851e4aa2ac325184e573205cf18a73782c6cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?yongyiduan=28=E6=AE=B5=E6=B0=B8=E5=84=84=29?= Date: Thu, 9 Nov 2023 13:27:27 +0800 Subject: [PATCH 25/31] =?UTF-8?q?feat:=20openapi=E6=96=B0=E5=A2=9E?= =?UTF-8?q?=E5=BA=A6=E9=87=8F=E8=83=BD=E5=8A=9B=20#9638?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../devops/openapi/es/config/MQConfiguration.kt | 11 +++++++++-- .../tencent/devops/openapi/es/mq/MQListenerService.kt | 5 +---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/config/MQConfiguration.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/config/MQConfiguration.kt index 941fc7a6882..1acebcb0d87 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/config/MQConfiguration.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/config/MQConfiguration.kt @@ -28,6 +28,7 @@ package com.tencent.devops.openapi.es.config import com.tencent.devops.common.stream.constants.StreamBinding +import com.tencent.devops.openapi.es.IESService import com.tencent.devops.openapi.es.mq.ESEvent import com.tencent.devops.openapi.es.mq.MQDispatcher import com.tencent.devops.openapi.es.mq.MQListenerService @@ -50,15 +51,21 @@ class MQConfiguration @Autowired constructor() { @Bean fun openapiMQDispatcher( - streamBridge: StreamBridge + @Autowired streamBridge: StreamBridge ) = MQDispatcher(streamBridge) @Bean(StreamBinding.BINDING_OPENAPI_LOG_EVENT_IN) fun openapiLogEventIn( - listenerService: MQListenerService + @Autowired listenerService: MQListenerService ): Consumer> { return Consumer { event: Message -> listenerService.handleEvent(event.payload) } } + + @Bean + fun mqListenerService( + @Autowired logService: IESService, + @Autowired dispatcher: MQDispatcher + ) = MQListenerService(logService, dispatcher) } diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/mq/MQListenerService.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/mq/MQListenerService.kt index a5c4e2c833f..57aeabed9e6 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/mq/MQListenerService.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/mq/MQListenerService.kt @@ -29,11 +29,8 @@ package com.tencent.devops.openapi.es.mq import com.tencent.devops.openapi.es.IESService import org.slf4j.LoggerFactory -import org.springframework.beans.factory.annotation.Autowired -import org.springframework.stereotype.Component -@Component -class MQListenerService @Autowired constructor( +class MQListenerService constructor( private val logService: IESService, private val dispatcher: MQDispatcher ) { From d4aa56aca15ed0eeeaa79738d5b1ef0015b8c34f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?yongyiduan=28=E6=AE=B5=E6=B0=B8=E5=84=84=29?= Date: Thu, 9 Nov 2023 13:35:48 +0800 Subject: [PATCH 26/31] =?UTF-8?q?feat:=20openapi=E6=96=B0=E5=A2=9E?= =?UTF-8?q?=E5=BA=A6=E9=87=8F=E8=83=BD=E5=8A=9B=20#9638?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kotlin/com/tencent/devops/openapi/es/MetricsService.kt | 1 - .../devops/openapi/es/config/ESAutoConfiguration.kt | 7 +++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/MetricsService.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/MetricsService.kt index bdefd873fb9..99b198c26c6 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/MetricsService.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/MetricsService.kt @@ -39,7 +39,6 @@ import org.jooq.DSLContext import org.slf4j.LoggerFactory import org.springframework.scheduling.annotation.Scheduled - class MetricsService( private val dslContext: DSLContext, private val apiDao: MetricsForApiDao, diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/config/ESAutoConfiguration.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/config/ESAutoConfiguration.kt index 1f5564d01f8..b8f84d0d622 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/config/ESAutoConfiguration.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/config/ESAutoConfiguration.kt @@ -33,11 +33,11 @@ import com.tencent.devops.common.es.client.LogClient import com.tencent.devops.common.redis.RedisOperation import com.tencent.devops.openapi.dao.MetricsForApiDao import com.tencent.devops.openapi.dao.MetricsForProjectDao +import com.tencent.devops.openapi.es.IESService +import com.tencent.devops.openapi.es.MetricsService import com.tencent.devops.openapi.es.impl.DefaultESServiceImpl import com.tencent.devops.openapi.es.impl.ESServiceImpl -import com.tencent.devops.openapi.es.IESService import com.tencent.devops.openapi.es.mq.MQDispatcher -import com.tencent.devops.openapi.es.MetricsService import org.jooq.DSLContext import org.springframework.beans.factory.annotation.Autowired import org.springframework.beans.factory.annotation.Value @@ -95,8 +95,7 @@ class ESAutoConfiguration { @Bean @ConditionalOnMissingBean(IESService::class) - fun defaultLogService( - ): DefaultESServiceImpl { + fun defaultLogService(): DefaultESServiceImpl { return DefaultESServiceImpl() } } From 9a300da1d0499d3a405cff066f09df12e9fbe862 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?yongyiduan=28=E6=AE=B5=E6=B0=B8=E5=84=84=29?= Date: Thu, 9 Nov 2023 14:20:50 +0800 Subject: [PATCH 27/31] =?UTF-8?q?feat:=20openapi=E6=96=B0=E5=A2=9E?= =?UTF-8?q?=E5=BA=A6=E9=87=8F=E8=83=BD=E5=8A=9B=20#9638?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tencent/devops/openapi/es/impl/DefaultESServiceImpl.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/impl/DefaultESServiceImpl.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/impl/DefaultESServiceImpl.kt index 62175d3a28f..61579f688ea 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/impl/DefaultESServiceImpl.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/impl/DefaultESServiceImpl.kt @@ -33,6 +33,6 @@ import com.tencent.devops.openapi.es.mq.ESEvent class DefaultESServiceImpl : IESService { - override fun addMessage(message: ESMessage) {} - override fun esAddMessage(event: ESEvent) {} + override fun addMessage(message: ESMessage) = Unit + override fun esAddMessage(event: ESEvent) = Unit } From d700a0c620fbc1dba0a48c88be1d51907364fb81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?yongyiduan=28=E6=AE=B5=E6=B0=B8=E5=84=84=29?= Date: Thu, 9 Nov 2023 16:37:07 +0800 Subject: [PATCH 28/31] =?UTF-8?q?feat:=20openapi=E6=96=B0=E5=A2=9E?= =?UTF-8?q?=E5=BA=A6=E9=87=8F=E8=83=BD=E5=8A=9B=20#9638?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/kotlin/com/tencent/devops/common/es/ESConfigUtils.kt | 2 +- .../kotlin/com/tencent/devops/openapi/es/impl/ESServiceImpl.kt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/ci/core/common/common-es/src/main/kotlin/com/tencent/devops/common/es/ESConfigUtils.kt b/src/backend/ci/core/common/common-es/src/main/kotlin/com/tencent/devops/common/es/ESConfigUtils.kt index 2f441cd7393..0b073536cf9 100644 --- a/src/backend/ci/core/common/common-es/src/main/kotlin/com/tencent/devops/common/es/ESConfigUtils.kt +++ b/src/backend/ci/core/common/common-es/src/main/kotlin/com/tencent/devops/common/es/ESConfigUtils.kt @@ -40,7 +40,7 @@ import org.elasticsearch.client.RestClient import org.elasticsearch.client.RestClientBuilder import org.slf4j.LoggerFactory -@Suppress("LongParameterList", "MagicNumber") +@Suppress("LongParameterList", "MagicNumber", "ComplexMethod") object ESConfigUtils { fun getClientBuilder( diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/impl/ESServiceImpl.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/impl/ESServiceImpl.kt index 5f587cac6b2..74719d65fde 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/impl/ESServiceImpl.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/impl/ESServiceImpl.kt @@ -67,7 +67,7 @@ import org.elasticsearch.search.aggregations.pipeline.SimpleValue import org.elasticsearch.search.builder.SearchSourceBuilder import org.slf4j.LoggerFactory -@Suppress("NestedBlockDepth") +@Suppress("NestedBlockDepth", "ComplexMethod") class ESServiceImpl constructor( private val logClient: LogClient, private val redisOperation: RedisOperation, From d95605756730347e0d6021d3a5e8c39ada7acd30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?yongyiduan=28=E6=AE=B5=E6=B0=B8=E5=84=84=29?= Date: Sat, 11 Nov 2023 15:53:53 +0800 Subject: [PATCH 29/31] =?UTF-8?q?feat:=20openapi=E6=96=B0=E5=A2=9E?= =?UTF-8?q?=E5=BA=A6=E9=87=8F=E8=83=BD=E5=8A=9B=20#9638?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/tencent/devops/openapi/dao/MetricsForApiDao.kt | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/dao/MetricsForApiDao.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/dao/MetricsForApiDao.kt index efef3cb6a39..6afc0def3dc 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/dao/MetricsForApiDao.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/dao/MetricsForApiDao.kt @@ -35,6 +35,11 @@ import org.springframework.stereotype.Repository @Repository class MetricsForApiDao { + + companion object { + const val batchGetLimit = Short.MAX_VALUE + } + @Suppress("ComplexMethod") fun createOrUpdate( dslContext: DSLContext, @@ -96,7 +101,7 @@ class MetricsForApiDao { dslContext: DSLContext ): List { return with(TOpenapiMetricsForApi.T_OPENAPI_METRICS_FOR_API) { - dslContext.select(API, KEY, PEAK_CONCURRENCY).from(this).skipCheck().fetch { + dslContext.select(API, KEY, PEAK_CONCURRENCY).from(this).limit(batchGetLimit).skipCheck().fetch { MetricsApiData( api = it.value1(), key = it.value2(), From 2c5946345668d090c4d014049e376ff94515826e Mon Sep 17 00:00:00 2001 From: yongyiduan Date: Sun, 12 Nov 2023 23:08:12 +0800 Subject: [PATCH 30/31] =?UTF-8?q?feat:=20openapi=E6=96=B0=E5=A2=9E?= =?UTF-8?q?=E5=BA=A6=E9=87=8F=E8=83=BD=E5=8A=9B=20#9638=20=E5=85=88?= =?UTF-8?q?=E5=85=B3=E6=8E=89=E5=AE=9A=E6=97=B6=E4=BB=BB=E5=8A=A1=EF=BC=8C?= =?UTF-8?q?=E5=85=88es=E5=AD=98=E5=85=A5=E6=95=B0=E6=8D=AE=EF=BC=8C?= =?UTF-8?q?=E5=BE=85=E9=87=87=E9=9B=86=E5=88=B0=E8=B6=B3=E9=87=8F=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E5=90=8E=E6=A0=B9=E6=8D=AE=E5=AE=9E=E9=99=85=E6=83=85?= =?UTF-8?q?=E5=86=B5=E8=BF=9B=E8=A1=8C=E5=AE=9A=E6=97=B6=E4=BB=BB=E5=8A=A1?= =?UTF-8?q?=E7=9A=84=E6=80=A7=E8=83=BD=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/kotlin/com/tencent/devops/openapi/es/MetricsService.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/MetricsService.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/MetricsService.kt index 99b198c26c6..f920c7f10f9 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/MetricsService.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/MetricsService.kt @@ -58,7 +58,7 @@ class MetricsService( /** * every 5m */ - @Scheduled(cron = "0 0/5 * * * ?") +// @Scheduled(cron = "0 0/5 * * * ?") fun job() { logger.info("Start to openapi metrics job") RedisLock(redisOperation, ES_INDEX_CLOSE_JOB_KEY, 60).run { From ec3f5321b62348e32285b713a45f4ed626649483 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?yongyiduan=28=E6=AE=B5=E6=B0=B8=E5=84=84=29?= Date: Mon, 13 Nov 2023 09:25:59 +0800 Subject: [PATCH 31/31] =?UTF-8?q?feat:=20openapi=E6=96=B0=E5=A2=9E?= =?UTF-8?q?=E5=BA=A6=E9=87=8F=E8=83=BD=E5=8A=9B=20#9638=20=E5=85=88?= =?UTF-8?q?=E5=85=B3=E6=8E=89=E5=AE=9A=E6=97=B6=E4=BB=BB=E5=8A=A1=EF=BC=8C?= =?UTF-8?q?=E5=85=88es=E5=AD=98=E5=85=A5=E6=95=B0=E6=8D=AE=EF=BC=8C?= =?UTF-8?q?=E5=BE=85=E9=87=87=E9=9B=86=E5=88=B0=E8=B6=B3=E9=87=8F=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E5=90=8E=E6=A0=B9=E6=8D=AE=E5=AE=9E=E9=99=85=E6=83=85?= =?UTF-8?q?=E5=86=B5=E8=BF=9B=E8=A1=8C=E5=AE=9A=E6=97=B6=E4=BB=BB=E5=8A=A1?= =?UTF-8?q?=E7=9A=84=E6=80=A7=E8=83=BD=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/kotlin/com/tencent/devops/openapi/es/MetricsService.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/MetricsService.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/MetricsService.kt index f920c7f10f9..f2a2d86173a 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/MetricsService.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/es/MetricsService.kt @@ -37,7 +37,6 @@ import java.time.Duration import java.time.LocalTime import org.jooq.DSLContext import org.slf4j.LoggerFactory -import org.springframework.scheduling.annotation.Scheduled class MetricsService( private val dslContext: DSLContext,