Skip to content

Commit

Permalink
Merge pull request #9670 from yongyiduan/issue_9638
Browse files Browse the repository at this point in the history
feat: openapi新增度量能力 #9638
  • Loading branch information
bkci-bot authored Nov 14, 2023
2 parents d184b54 + ec3f532 commit 020ba51
Show file tree
Hide file tree
Showing 36 changed files with 1,730 additions and 122 deletions.
33 changes: 33 additions & 0 deletions src/backend/ci/core/common/common-es/build.gradle.kts
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
/*
* Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
*
* Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved.
*
* BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
*
* A copy of the MIT License is included in this file.
*
*
* Terms of the MIT License:
* ---------------------------------------------------
* Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
* documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all copies or substantial portions of
* the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
* LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
* NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

dependencies {
api("org.elasticsearch:elasticsearch")
api("org.elasticsearch.client:elasticsearch-rest-client")
api("org.elasticsearch.client:elasticsearch-rest-high-level-client")
api(project(":core:common:common-web"))
}
Original file line number Diff line number Diff line change
Expand Up @@ -25,21 +25,15 @@
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

package com.tencent.devops.log.es
package com.tencent.devops.common.es

import com.tencent.devops.common.redis.RedisOperation
import com.tencent.devops.common.es.client.LogClient
import com.tencent.devops.common.es.client.impl.LogClientImpl
import com.tencent.devops.common.web.WebAutoConfiguration
import com.tencent.devops.log.client.LogClient
import com.tencent.devops.log.client.impl.LogClientImpl
import com.tencent.devops.log.jmx.CreateIndexBean
import com.tencent.devops.log.jmx.LogStorageBean
import com.tencent.devops.log.service.BuildLogPrintService
import com.tencent.devops.log.service.IndexService
import com.tencent.devops.log.service.LogService
import com.tencent.devops.log.service.LogStatusService
import com.tencent.devops.log.service.LogTagService
import com.tencent.devops.log.service.impl.LogServiceESImpl
import com.tencent.devops.log.util.ESConfigUtils
import java.io.File
import java.io.FileInputStream
import java.security.KeyStore
import javax.net.ssl.SSLContext
import org.apache.http.auth.AuthScope
import org.apache.http.auth.UsernamePasswordCredentials
import org.apache.http.impl.client.BasicCredentialsProvider
Expand All @@ -57,10 +51,6 @@ import org.springframework.context.annotation.Bean
import org.springframework.context.annotation.Configuration
import org.springframework.context.annotation.Primary
import org.springframework.core.Ordered
import java.io.File
import java.io.FileInputStream
import java.security.KeyStore
import javax.net.ssl.SSLContext

@Suppress("ALL")
@Configuration
Expand All @@ -71,6 +61,7 @@ import javax.net.ssl.SSLContext
class ESAutoConfiguration : DisposableBean {
@Value("\${log.elasticsearch.ip}")
private val host: String? = null

@Value("\${log.elasticsearch.port}")
private val port: Int? = null

Expand Down Expand Up @@ -221,30 +212,6 @@ class ESAutoConfiguration : DisposableBean {
)
}

@Bean
fun esLogService(
@Autowired logESClient: LogClient,
@Autowired indexService: IndexService,
@Autowired logStatusService: LogStatusService,
@Autowired logTagService: LogTagService,
@Autowired defaultKeywords: List<String>,
@Autowired createIndexBean: CreateIndexBean,
@Autowired logStorageBean: LogStorageBean,
@Autowired redisOperation: RedisOperation,
@Autowired buildLogPrintService: BuildLogPrintService
): LogService {
return LogServiceESImpl(
logClient = logESClient,
indexService = indexService,
logStatusService = logStatusService,
logTagService = logTagService,
logStorageBean = logStorageBean,
createIndexBean = createIndexBean,
buildLogPrintService = buildLogPrintService,
redisOperation = redisOperation
)
}

@Bean
@ConditionalOnMissingBean
fun logClient(@Autowired transportClient: ESClient): LogClient =
Expand All @@ -264,8 +231,8 @@ class ESAutoConfiguration : DisposableBean {

private fun hasCertificateConfig(): Boolean {
return !keystoreFilePath.isNullOrBlank() ||
!truststoreFilePath.isNullOrBlank() ||
!keystorePassword.isNullOrBlank() ||
!truststorePassword.isNullOrBlank()
!truststoreFilePath.isNullOrBlank() ||
!keystorePassword.isNullOrBlank() ||
!truststorePassword.isNullOrBlank()
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

package com.tencent.devops.log.es
package com.tencent.devops.common.es

import org.elasticsearch.client.RestHighLevelClient

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,14 +25,8 @@
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

package com.tencent.devops.log.util
package com.tencent.devops.common.es

import com.tencent.devops.common.log.constant.LogMessageCode.FILE_NOT_FOUND_CHECK_PATH
import com.tencent.devops.common.web.utils.I18nUtil
import com.tencent.devops.log.es.NormalX509ExtendedTrustManager
import java.io.File
import java.io.FileInputStream
import java.security.KeyStore
import java.security.SecureRandom
import javax.net.ssl.SSLContext
import org.apache.http.HeaderElementIterator
Expand All @@ -42,12 +36,11 @@ import org.apache.http.client.CredentialsProvider
import org.apache.http.message.BasicHeaderElementIterator
import org.apache.http.protocol.HTTP
import org.apache.http.protocol.HttpContext
import org.apache.http.ssl.SSLContexts
import org.elasticsearch.client.RestClient
import org.elasticsearch.client.RestClientBuilder
import org.slf4j.LoggerFactory

@Suppress("LongParameterList", "MagicNumber")
@Suppress("LongParameterList", "MagicNumber", "ComplexMethod")
object ESConfigUtils {

fun getClientBuilder(
Expand Down Expand Up @@ -110,39 +103,5 @@ object ESConfigUtils {
}
}

fun getSSLContext(
keystoreFilePath: String,
truststoreFilePath: String,
keystorePassword: String,
truststorePassword: String
): SSLContext {
val keystoreFile = File(keystoreFilePath)
if (!keystoreFile.exists()) {
throw IllegalArgumentException(
I18nUtil.getCodeLanMessage(messageCode = FILE_NOT_FOUND_CHECK_PATH, params = arrayOf("keystore")) +
keystoreFilePath
)
}
val truststoreFile = File(truststoreFilePath)
if (!truststoreFile.exists()) {
throw IllegalArgumentException(
I18nUtil.getCodeLanMessage(
messageCode = FILE_NOT_FOUND_CHECK_PATH,
params = arrayOf("truststore")
) + truststoreFilePath
)
}
val keyStore = KeyStore.getInstance(KeyStore.getDefaultType())
val keystorePasswordCharArray = keystorePassword.toCharArray()
keyStore.load(FileInputStream(keystoreFile), keystorePasswordCharArray)
val truststore = KeyStore.getInstance(KeyStore.getDefaultType())
val truststorePasswordCharArray = truststorePassword.toCharArray()
truststore.load(FileInputStream(truststoreFile), truststorePasswordCharArray)
return SSLContexts.custom()
.loadTrustMaterial(truststore, null)
.loadKeyMaterial(keyStore, keystorePasswordCharArray)
.build()
}

private val logger = LoggerFactory.getLogger(ESConfigUtils::class.java)
}
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

package com.tencent.devops.log.es
package com.tencent.devops.common.es

import org.springframework.boot.context.properties.ConfigurationProperties
import org.springframework.boot.context.properties.ConstructorBinding
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

package com.tencent.devops.log.es
package com.tencent.devops.common.es

import java.net.Socket
import javax.net.ssl.SSLEngine
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,9 @@
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

package com.tencent.devops.log.client
package com.tencent.devops.common.es.client

import com.tencent.devops.log.es.ESClient
import com.tencent.devops.common.es.ESClient

interface LogClient {

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,10 @@
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

package com.tencent.devops.log.client.impl
package com.tencent.devops.common.es.client.impl

import com.tencent.devops.log.client.LogClient
import com.tencent.devops.log.es.ESClient
import com.tencent.devops.common.es.client.LogClient
import com.tencent.devops.common.es.ESClient

class LogClientImpl constructor(private val client: ESClient) : LogClient {

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
org.springframework.boot.autoconfigure.EnableAutoConfiguration=\
com.tencent.devops.common.es.ESAutoConfiguration
Original file line number Diff line number Diff line change
Expand Up @@ -39,4 +39,8 @@ object StreamBinding {
// 日志构建状态事件
const val BINDING_LOG_STATUS_EVENT_IN = "logStatusEventIn"
const val BINDING_LOG_STATUS_EVENT_OUT = "logStatusEventOut"

// openapi审计日志预处理事件
const val BINDING_OPENAPI_LOG_EVENT_IN = "openapiLogOriginEventIn"
const val BINDING_OPENAPI_LOG_EVENT_OUT = "openapiLogOriginEventOut"
}
4 changes: 1 addition & 3 deletions src/backend/ci/core/log/biz-log/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -33,11 +33,9 @@ dependencies {
api(project(":core:common:common-auth:common-auth-api"))
api(project(":core:common:common-db"))
api(project(":core:common:common-stream"))
api(project(":core:common:common-es"))
api(project(":core:auth:api-auth"))

api("org.elasticsearch:elasticsearch")
api("org.elasticsearch.client:elasticsearch-rest-client")
api("org.elasticsearch.client:elasticsearch-rest-high-level-client")
api("org.apache.logging.log4j:log4j-core")
api("org.apache.logging.log4j:log4j-api")
api("com.github.ben-manes.caffeine:caffeine")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,14 @@

package com.tencent.devops.log.cron.impl

import com.tencent.devops.common.es.client.LogClient
import com.tencent.devops.common.redis.RedisLock
import com.tencent.devops.common.redis.RedisOperation
import com.tencent.devops.log.client.LogClient
import com.tencent.devops.log.configuration.StorageProperties
import com.tencent.devops.log.cron.IndexCleanJob
import com.tencent.devops.log.util.IndexNameUtils.LOG_INDEX_PREFIX
import java.time.LocalDateTime
import java.time.temporal.ChronoUnit
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest
import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest
import org.elasticsearch.client.RequestOptions
Expand All @@ -44,8 +46,6 @@ import org.springframework.beans.factory.annotation.Autowired
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty
import org.springframework.scheduling.annotation.Scheduled
import org.springframework.stereotype.Component
import java.time.LocalDateTime
import java.time.temporal.ChronoUnit

@Suppress("MagicNumber")
@Component
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
/*
* Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
*
* Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved.
*
* BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
*
* A copy of the MIT License is included in this file.
*
*
* Terms of the MIT License:
* ---------------------------------------------------
* Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
* documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all copies or substantial portions of
* the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
* LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
* NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

package com.tencent.devops.log.es

import com.tencent.devops.common.es.ESAutoConfiguration
import com.tencent.devops.common.es.ESProperties
import com.tencent.devops.common.es.client.LogClient
import com.tencent.devops.common.redis.RedisOperation
import com.tencent.devops.log.jmx.CreateIndexBean
import com.tencent.devops.log.jmx.LogStorageBean
import com.tencent.devops.log.service.BuildLogPrintService
import com.tencent.devops.log.service.IndexService
import com.tencent.devops.log.service.LogService
import com.tencent.devops.log.service.LogStatusService
import com.tencent.devops.log.service.LogTagService
import com.tencent.devops.log.service.impl.LogServiceESImpl
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.boot.autoconfigure.AutoConfigureAfter
import org.springframework.boot.autoconfigure.AutoConfigureOrder
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty
import org.springframework.boot.context.properties.EnableConfigurationProperties
import org.springframework.context.annotation.Bean
import org.springframework.context.annotation.Configuration
import org.springframework.core.Ordered

@Suppress("ALL")
@Configuration
@ConditionalOnProperty(prefix = "log.storage", name = ["type"], havingValue = "elasticsearch")
@AutoConfigureOrder(Ordered.HIGHEST_PRECEDENCE)
@AutoConfigureAfter(ESAutoConfiguration::class)
@EnableConfigurationProperties(ESProperties::class)
class LogESAutoConfiguration {
@Bean
fun esLogService(
@Autowired logESClient: LogClient,
@Autowired indexService: IndexService,
@Autowired logStatusService: LogStatusService,
@Autowired logTagService: LogTagService,
@Autowired defaultKeywords: List<String>,
@Autowired createIndexBean: CreateIndexBean,
@Autowired logStorageBean: LogStorageBean,
@Autowired redisOperation: RedisOperation,
@Autowired buildLogPrintService: BuildLogPrintService
): LogService {
return LogServiceESImpl(
logClient = logESClient,
indexService = indexService,
logStatusService = logStatusService,
logTagService = logTagService,
logStorageBean = logStorageBean,
createIndexBean = createIndexBean,
buildLogPrintService = buildLogPrintService,
redisOperation = redisOperation
)
}
}
Loading

0 comments on commit 020ba51

Please sign in to comment.