From be721eb88fa282b3bc5f87df1bcf6fc53810e16a Mon Sep 17 00:00:00 2001 From: royalhuang Date: Wed, 25 Oct 2023 16:09:43 +0800 Subject: [PATCH 1/9] =?UTF-8?q?bug:=20=E6=9E=84=E5=BB=BA=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E7=9A=84=E6=9C=8D=E5=8A=A1=E8=B0=83=E7=94=A8=E7=AB=AF=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0=E8=AF=B7=E6=B1=82=E7=86=94=E6=96=AD=E6=9C=BA=E5=88=B6?= =?UTF-8?q?=20#9602?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ci/core/log/api-log/build.gradle.kts | 1 + .../common/log/utils/BuildLogPrinter.kt | 153 +++++++++++------- .../configuration/LogPrinterConfiguration.kt | 4 +- 3 files changed, 96 insertions(+), 62 deletions(-) diff --git a/src/backend/ci/core/log/api-log/build.gradle.kts b/src/backend/ci/core/log/api-log/build.gradle.kts index 7fc1073d439..c413a4e764f 100644 --- a/src/backend/ci/core/log/api-log/build.gradle.kts +++ b/src/backend/ci/core/log/api-log/build.gradle.kts @@ -28,6 +28,7 @@ dependencies { api(project(":core:common:common-api")) api(project(":core:common:common-client")) + api("io.github.resilience4j:resilience4j-circuitbreaker") } plugins { diff --git a/src/backend/ci/core/log/api-log/src/main/kotlin/com/tencent/devops/common/log/utils/BuildLogPrinter.kt b/src/backend/ci/core/log/api-log/src/main/kotlin/com/tencent/devops/common/log/utils/BuildLogPrinter.kt index 54fa3d74271..5e5241f9d6d 100644 --- a/src/backend/ci/core/log/api-log/src/main/kotlin/com/tencent/devops/common/log/utils/BuildLogPrinter.kt +++ b/src/backend/ci/core/log/api-log/src/main/kotlin/com/tencent/devops/common/log/utils/BuildLogPrinter.kt @@ -32,11 +32,15 @@ import com.tencent.devops.common.log.pojo.enums.LogType import com.tencent.devops.common.log.pojo.message.LogMessage import com.tencent.devops.log.api.print.ServiceLogPrintResource import com.tencent.devops.log.meta.Ansi +import io.github.resilience4j.circuitbreaker.CircuitBreakerRegistry import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Autowired @Suppress("LongParameterList", "TooManyFunctions") class BuildLogPrinter( - private val client: Client + private val client: Client, + @Autowired(required = false) + private val circuitBreakerRegistry: CircuitBreakerRegistry? = null ) { fun addLine( @@ -48,17 +52,21 @@ class BuildLogPrinter( subTag: String? = null ) { try { - genLogPrintPrintResource().addLogLine( - buildId = buildId, - logMessage = genLogMessage( - message = message, - tag = tag, - subTag = subTag, - jobId = jobId, - logType = LogType.LOG, - executeCount = executeCount - ) - ) + doWithCircuitBreaker { + circuitBreakerRegistry?.let { + genLogPrintPrintResource().addLogLine( + buildId = buildId, + logMessage = genLogMessage( + message = message, + tag = tag, + subTag = subTag, + jobId = jobId, + logType = LogType.LOG, + executeCount = executeCount + ) + ) + } + } } catch (ignore: Exception) { logger.error("[$buildId]|addLine error|message=$message", ignore) } @@ -66,10 +74,12 @@ class BuildLogPrinter( fun addLines(buildId: String, logMessages: List) { try { - genLogPrintPrintResource().addLogMultiLine( - buildId = buildId, - logMessages = logMessages - ) + doWithCircuitBreaker { + genLogPrintPrintResource().addLogMultiLine( + buildId = buildId, + logMessages = logMessages + ) + } } catch (ignore: Exception) { logger.error("[$buildId]|addLines error|logMessages=$logMessages", ignore) } @@ -116,17 +126,19 @@ class BuildLogPrinter( subTag: String? = null ) { try { - genLogPrintPrintResource().addLogLine( - buildId = buildId, - logMessage = genLogMessage( - message = "$LOG_ERROR_FLAG${message.replace("\n", "\n$LOG_ERROR_FLAG")}", - tag = tag, - subTag = subTag, - jobId = jobId, - logType = LogType.ERROR, - executeCount = executeCount + doWithCircuitBreaker { + genLogPrintPrintResource().addLogLine( + buildId = buildId, + logMessage = genLogMessage( + message = "$LOG_ERROR_FLAG${message.replace("\n", "\n$LOG_ERROR_FLAG")}", + tag = tag, + subTag = subTag, + jobId = jobId, + logType = LogType.ERROR, + executeCount = executeCount + ) ) - ) + } } catch (ignore: Exception) { logger.error("[$buildId]|addErrorLine error|message=$message", ignore) } @@ -141,17 +153,19 @@ class BuildLogPrinter( subTag: String? = null ) { try { - genLogPrintPrintResource().addLogLine( - buildId = buildId, - logMessage = genLogMessage( - message = "$LOG_DEBUG_FLAG${message.replace("\n", "\n$LOG_DEBUG_FLAG")}", - tag = tag, - subTag = subTag, - jobId = jobId, - logType = LogType.DEBUG, - executeCount = executeCount + doWithCircuitBreaker { + genLogPrintPrintResource().addLogLine( + buildId = buildId, + logMessage = genLogMessage( + message = "$LOG_DEBUG_FLAG${message.replace("\n", "\n$LOG_DEBUG_FLAG")}", + tag = tag, + subTag = subTag, + jobId = jobId, + logType = LogType.DEBUG, + executeCount = executeCount + ) ) - ) + } } catch (ignore: Exception) { logger.error("[$buildId]|addDebugLine error|message=$message", ignore) } @@ -166,17 +180,19 @@ class BuildLogPrinter( subTag: String? = null ) { try { - genLogPrintPrintResource().addLogLine( - buildId = buildId, - logMessage = genLogMessage( - message = "$LOG_WARN_FLAG${message.replace("\n", "\n$LOG_WARN_FLAG")}", - tag = tag, - subTag = subTag, - jobId = jobId, - logType = LogType.DEBUG, - executeCount = executeCount + doWithCircuitBreaker { + genLogPrintPrintResource().addLogLine( + buildId = buildId, + logMessage = genLogMessage( + message = "$LOG_WARN_FLAG${message.replace("\n", "\n$LOG_WARN_FLAG")}", + tag = tag, + subTag = subTag, + jobId = jobId, + logType = LogType.DEBUG, + executeCount = executeCount + ) ) - ) + } } catch (ignore: Exception) { logger.error("[$buildId]|addWarnLine error|message=$message", ignore) } @@ -239,14 +255,16 @@ class BuildLogPrinter( subTag: String? = null ) { try { - genLogPrintPrintResource().updateLogStatus( - buildId = buildId, - finished = true, - tag = tag, - subTag = subTag, - jobId = jobId, - executeCount = executeCount - ) + doWithCircuitBreaker { + genLogPrintPrintResource().updateLogStatus( + buildId = buildId, + finished = true, + tag = tag, + subTag = subTag, + jobId = jobId, + executeCount = executeCount + ) + } } catch (ignore: Exception) { logger.error("[$buildId]|stopLog fail", ignore) } @@ -260,13 +278,15 @@ class BuildLogPrinter( subTag: String? = null ) { try { - genLogPrintPrintResource().addLogStatus( - buildId = buildId, - tag = tag, - subTag = subTag, - jobId = jobId, - executeCount = executeCount - ) + doWithCircuitBreaker { + genLogPrintPrintResource().addLogStatus( + buildId = buildId, + tag = tag, + subTag = subTag, + jobId = jobId, + executeCount = executeCount + ) + } } catch (ignore: Exception) { logger.error("[$buildId]|stopLog fail", ignore) } @@ -293,6 +313,17 @@ class BuildLogPrinter( return client.get(ServiceLogPrintResource::class) } + private fun doWithCircuitBreaker( + action: () -> T + ): T { + return circuitBreakerRegistry?.let { + val breaker = it.circuitBreaker(this.javaClass.name) + breaker.executeCallable { + action() + } + } ?: action() + } + companion object { private val logger = LoggerFactory.getLogger(BuildLogPrinter::class.java) diff --git a/src/backend/ci/core/log/api-log/src/main/kotlin/com/tencent/devops/log/configuration/LogPrinterConfiguration.kt b/src/backend/ci/core/log/api-log/src/main/kotlin/com/tencent/devops/log/configuration/LogPrinterConfiguration.kt index 22a6e00b767..6e20e1496b7 100644 --- a/src/backend/ci/core/log/api-log/src/main/kotlin/com/tencent/devops/log/configuration/LogPrinterConfiguration.kt +++ b/src/backend/ci/core/log/api-log/src/main/kotlin/com/tencent/devops/log/configuration/LogPrinterConfiguration.kt @@ -29,6 +29,7 @@ package com.tencent.devops.log.configuration import com.tencent.devops.common.client.Client import com.tencent.devops.common.log.utils.BuildLogPrinter +import io.github.resilience4j.circuitbreaker.CircuitBreakerRegistry import org.springframework.boot.autoconfigure.AutoConfigureOrder import org.springframework.boot.autoconfigure.condition.ConditionalOnWebApplication import org.springframework.context.annotation.Bean @@ -41,5 +42,6 @@ import org.springframework.core.Ordered class LogPrinterConfiguration { @Bean - fun buildLogPrinter(client: Client) = BuildLogPrinter(client) + fun buildLogPrinter(client: Client, circuitBreakerRegistry: CircuitBreakerRegistry?) = + BuildLogPrinter(client, circuitBreakerRegistry) } From 24d1e7d5f8bc318b7fd757619c08bd425149561a Mon Sep 17 00:00:00 2001 From: royalhuang Date: Thu, 26 Oct 2023 19:36:23 +0800 Subject: [PATCH 2/9] =?UTF-8?q?bug:=20=E6=9E=84=E5=BB=BA=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E7=9A=84=E6=9C=8D=E5=8A=A1=E8=B0=83=E7=94=A8=E7=AB=AF=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0=E8=AF=B7=E6=B1=82=E7=86=94=E6=96=AD=E6=9C=BA=E5=88=B6?= =?UTF-8?q?=20#9602=20=E5=A2=9E=E5=8A=A0=E7=86=94=E6=96=AD=E5=99=A8?= =?UTF-8?q?=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../common/log/utils/BuildLogPrinter.kt | 40 +++++++++++-------- .../configuration/LogPrinterConfiguration.kt | 20 +++++++++- 2 files changed, 42 insertions(+), 18 deletions(-) diff --git a/src/backend/ci/core/log/api-log/src/main/kotlin/com/tencent/devops/common/log/utils/BuildLogPrinter.kt b/src/backend/ci/core/log/api-log/src/main/kotlin/com/tencent/devops/common/log/utils/BuildLogPrinter.kt index 5e5241f9d6d..f7232c7a949 100644 --- a/src/backend/ci/core/log/api-log/src/main/kotlin/com/tencent/devops/common/log/utils/BuildLogPrinter.kt +++ b/src/backend/ci/core/log/api-log/src/main/kotlin/com/tencent/devops/common/log/utils/BuildLogPrinter.kt @@ -32,6 +32,7 @@ import com.tencent.devops.common.log.pojo.enums.LogType import com.tencent.devops.common.log.pojo.message.LogMessage import com.tencent.devops.log.api.print.ServiceLogPrintResource import com.tencent.devops.log.meta.Ansi +import io.github.resilience4j.circuitbreaker.CallNotPermittedException import io.github.resilience4j.circuitbreaker.CircuitBreakerRegistry import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired @@ -39,8 +40,7 @@ import org.springframework.beans.factory.annotation.Autowired @Suppress("LongParameterList", "TooManyFunctions") class BuildLogPrinter( private val client: Client, - @Autowired(required = false) - private val circuitBreakerRegistry: CircuitBreakerRegistry? = null + private val circuitBreakerRegistry: CircuitBreakerRegistry ) { fun addLine( @@ -53,20 +53,20 @@ class BuildLogPrinter( ) { try { doWithCircuitBreaker { - circuitBreakerRegistry?.let { - genLogPrintPrintResource().addLogLine( - buildId = buildId, - logMessage = genLogMessage( - message = message, - tag = tag, - subTag = subTag, - jobId = jobId, - logType = LogType.LOG, - executeCount = executeCount - ) + genLogPrintPrintResource().addLogLine( + buildId = buildId, + logMessage = genLogMessage( + message = message, + tag = tag, + subTag = subTag, + jobId = jobId, + logType = LogType.LOG, + executeCount = executeCount ) - } + ) } + } catch (e: CallNotPermittedException) { + logger.warn("[LOG]|LOG_SERVER_ERROR|causingCircuitBreakerName=${e.causingCircuitBreakerName}|$buildId|") } catch (ignore: Exception) { logger.error("[$buildId]|addLine error|message=$message", ignore) } @@ -80,6 +80,8 @@ class BuildLogPrinter( logMessages = logMessages ) } + } catch (e: CallNotPermittedException) { + logger.warn("[LOG]|LOG_SERVER_ERROR|causingCircuitBreakerName=${e.causingCircuitBreakerName}|$buildId|") } catch (ignore: Exception) { logger.error("[$buildId]|addLines error|logMessages=$logMessages", ignore) } @@ -139,6 +141,8 @@ class BuildLogPrinter( ) ) } + } catch (e: CallNotPermittedException) { + logger.warn("[LOG]|LOG_SERVER_ERROR|causingCircuitBreakerName=${e.causingCircuitBreakerName}|$buildId|") } catch (ignore: Exception) { logger.error("[$buildId]|addErrorLine error|message=$message", ignore) } @@ -166,6 +170,8 @@ class BuildLogPrinter( ) ) } + } catch (e: CallNotPermittedException) { + logger.warn("[LOG]|LOG_SERVER_ERROR|causingCircuitBreakerName=${e.causingCircuitBreakerName}|$buildId|") } catch (ignore: Exception) { logger.error("[$buildId]|addDebugLine error|message=$message", ignore) } @@ -193,6 +199,8 @@ class BuildLogPrinter( ) ) } + } catch (e: CallNotPermittedException) { + logger.warn("[LOG]|LOG_SERVER_ERROR|causingCircuitBreakerName=${e.causingCircuitBreakerName}|$buildId|") } catch (ignore: Exception) { logger.error("[$buildId]|addWarnLine error|message=$message", ignore) } @@ -316,12 +324,12 @@ class BuildLogPrinter( private fun doWithCircuitBreaker( action: () -> T ): T { - return circuitBreakerRegistry?.let { + return circuitBreakerRegistry.let { val breaker = it.circuitBreaker(this.javaClass.name) breaker.executeCallable { action() } - } ?: action() + } } companion object { diff --git a/src/backend/ci/core/log/api-log/src/main/kotlin/com/tencent/devops/log/configuration/LogPrinterConfiguration.kt b/src/backend/ci/core/log/api-log/src/main/kotlin/com/tencent/devops/log/configuration/LogPrinterConfiguration.kt index 6e20e1496b7..812cbb972bc 100644 --- a/src/backend/ci/core/log/api-log/src/main/kotlin/com/tencent/devops/log/configuration/LogPrinterConfiguration.kt +++ b/src/backend/ci/core/log/api-log/src/main/kotlin/com/tencent/devops/log/configuration/LogPrinterConfiguration.kt @@ -29,12 +29,14 @@ package com.tencent.devops.log.configuration import com.tencent.devops.common.client.Client import com.tencent.devops.common.log.utils.BuildLogPrinter +import io.github.resilience4j.circuitbreaker.CircuitBreakerConfig import io.github.resilience4j.circuitbreaker.CircuitBreakerRegistry import org.springframework.boot.autoconfigure.AutoConfigureOrder import org.springframework.boot.autoconfigure.condition.ConditionalOnWebApplication import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration import org.springframework.core.Ordered +import java.time.Duration @Configuration @ConditionalOnWebApplication @@ -42,6 +44,20 @@ import org.springframework.core.Ordered class LogPrinterConfiguration { @Bean - fun buildLogPrinter(client: Client, circuitBreakerRegistry: CircuitBreakerRegistry?) = - BuildLogPrinter(client, circuitBreakerRegistry) + fun buildLogPrinter(client: Client): BuildLogPrinter { + val builder = CircuitBreakerConfig.custom() + builder.enableAutomaticTransitionFromOpenToHalfOpen() + builder.writableStackTraceEnabled(false) + // 当熔断后等待 300s 放开熔断 + builder.waitDurationInOpenState(Duration.ofSeconds(300)) + // 熔断放开后，运行通过的请求数，如果达到熔断条件，继续熔断 + builder.permittedNumberOfCallsInHalfOpenState(100) + // 当错误率达到 10% 开启熔断 + builder.failureRateThreshold(10.0F) + // 请求超过 1s 就是慢请求 + builder.slowCallRateThreshold(10.0F) + // 慢请求超过 10% 开启熔断 + builder.slowCallDurationThreshold(Duration.ofSeconds(1)) + return BuildLogPrinter(client, CircuitBreakerRegistry.of(builder.build())) + } } From bf6a294c437efebc3bb5184225daa4b6c6261394 Mon Sep 17 00:00:00 2001 From: royalhuang Date: Thu, 26 Oct 2023 20:26:32 +0800 Subject: [PATCH 3/9] =?UTF-8?q?bug:=20=E6=9E=84=E5=BB=BA=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E7=9A=84=E6=9C=8D=E5=8A=A1=E8=B0=83=E7=94=A8=E7=AB=AF=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0=E8=AF=B7=E6=B1=82=E7=86=94=E6=96=AD=E6=9C=BA=E5=88=B6?= =?UTF-8?q?=20#9602=20=E5=A2=9E=E5=8A=A0=E7=86=94=E6=96=AD=E5=99=A8?= =?UTF-8?q?=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../devops/log/configuration/LogPrinterConfiguration.kt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/backend/ci/core/log/api-log/src/main/kotlin/com/tencent/devops/log/configuration/LogPrinterConfiguration.kt b/src/backend/ci/core/log/api-log/src/main/kotlin/com/tencent/devops/log/configuration/LogPrinterConfiguration.kt index 812cbb972bc..84d66840568 100644 --- a/src/backend/ci/core/log/api-log/src/main/kotlin/com/tencent/devops/log/configuration/LogPrinterConfiguration.kt +++ b/src/backend/ci/core/log/api-log/src/main/kotlin/com/tencent/devops/log/configuration/LogPrinterConfiguration.kt @@ -54,10 +54,12 @@ class LogPrinterConfiguration { builder.permittedNumberOfCallsInHalfOpenState(100) // 当错误率达到 10% 开启熔断 builder.failureRateThreshold(10.0F) - // 请求超过 1s 就是慢请求 - builder.slowCallRateThreshold(10.0F) // 慢请求超过 10% 开启熔断 + builder.slowCallRateThreshold(10.0F) + // 请求超过 1s 就是慢请求 builder.slowCallDurationThreshold(Duration.ofSeconds(1)) + // 滑动窗口大小为 100，默认值 + builder.slidingWindowSize(100) return BuildLogPrinter(client, CircuitBreakerRegistry.of(builder.build())) } } From 07c57832c13fa4b8979c125eab2bfd56025ee3e9 Mon Sep 17 00:00:00 2001 From: royalhuang Date: Fri, 27 Oct 2023 17:21:24 +0800 Subject: [PATCH 4/9] =?UTF-8?q?bug:=20=E6=9E=84=E5=BB=BA=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E7=9A=84=E6=9C=8D=E5=8A=A1=E8=B0=83=E7=94=A8=E7=AB=AF=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0=E8=AF=B7=E6=B1=82=E7=86=94=E6=96=AD=E6=9C=BA=E5=88=B6?= =?UTF-8?q?=20#9602?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/tencent/devops/common/log/utils/BuildLogPrinter.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/src/backend/ci/core/log/api-log/src/main/kotlin/com/tencent/devops/common/log/utils/BuildLogPrinter.kt b/src/backend/ci/core/log/api-log/src/main/kotlin/com/tencent/devops/common/log/utils/BuildLogPrinter.kt index f7232c7a949..bf1768db9a5 100644 --- a/src/backend/ci/core/log/api-log/src/main/kotlin/com/tencent/devops/common/log/utils/BuildLogPrinter.kt +++ b/src/backend/ci/core/log/api-log/src/main/kotlin/com/tencent/devops/common/log/utils/BuildLogPrinter.kt @@ -35,7 +35,6 @@ import com.tencent.devops.log.meta.Ansi import io.github.resilience4j.circuitbreaker.CallNotPermittedException import io.github.resilience4j.circuitbreaker.CircuitBreakerRegistry import org.slf4j.LoggerFactory -import org.springframework.beans.factory.annotation.Autowired @Suppress("LongParameterList", "TooManyFunctions") class BuildLogPrinter( From 541ebd4558c6cb57137d571d449f9510a516274a Mon Sep 17 00:00:00 2001 From: royalhuang Date: Fri, 27 Oct 2023 17:38:56 +0800 Subject: [PATCH 5/9] =?UTF-8?q?bug:=20=E6=9E=84=E5=BB=BA=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E7=9A=84=E6=9C=8D=E5=8A=A1=E8=B0=83=E7=94=A8=E7=AB=AF=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0=E8=AF=B7=E6=B1=82=E7=86=94=E6=96=AD=E6=9C=BA=E5=88=B6?= =?UTF-8?q?=20#9602?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../worker/worker-common/build.gradle.kts | 1 + .../worker/common/logger/LoggerService.kt | 56 ++++++++++++++++--- 2 files changed, 48 insertions(+), 9 deletions(-) diff --git a/src/backend/ci/core/worker/worker-common/build.gradle.kts b/src/backend/ci/core/worker/worker-common/build.gradle.kts index 62d27c5b106..aaf9eab3946 100644 --- a/src/backend/ci/core/worker/worker-common/build.gradle.kts +++ b/src/backend/ci/core/worker/worker-common/build.gradle.kts @@ -46,6 +46,7 @@ dependencies { api("ch.qos.logback:logback-core") api("ch.qos.logback:logback-classic") api("com.github.ben-manes.caffeine:caffeine") + api("io.github.resilience4j:resilience4j-circuitbreaker") api(fileTree(mapOf("dir" to "lib", "includes" to listOf("*.jar")))) } diff --git a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/logger/LoggerService.kt b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/logger/LoggerService.kt index c364da0dcff..baea84bc452 100644 --- a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/logger/LoggerService.kt +++ b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/logger/LoggerService.kt @@ -53,10 +53,13 @@ import com.tencent.devops.worker.common.service.SensitiveValueService import com.tencent.devops.worker.common.utils.ArchiveUtils import com.tencent.devops.worker.common.utils.FileUtils import com.tencent.devops.worker.common.utils.WorkspaceUtils +import io.github.resilience4j.circuitbreaker.CircuitBreakerConfig +import io.github.resilience4j.circuitbreaker.CircuitBreakerRegistry import org.slf4j.LoggerFactory import java.io.File import java.sql.Date import java.text.SimpleDateFormat +import java.time.Duration import java.util.concurrent.Callable import java.util.concurrent.Executors import java.util.concurrent.Future @@ -74,6 +77,24 @@ object LoggerService { private val running = AtomicBoolean(true) private var currentTaskLineNo = 0 private val sdf = SimpleDateFormat("yyyy-MM-dd HH:mm:ss:SSS") + private val circuitBreakerRegistry = CircuitBreakerRegistry.of( + CircuitBreakerConfig.custom() + .enableAutomaticTransitionFromOpenToHalfOpen() + .writableStackTraceEnabled(false) + // 当熔断后等待 300s 放开熔断 + .waitDurationInOpenState(Duration.ofSeconds(300)) + // 熔断放开后，运行通过的请求数，如果达到熔断条件，继续熔断 + .permittedNumberOfCallsInHalfOpenState(100) + // 当错误率达到 10% 开启熔断 + .failureRateThreshold(10.0F) + // 慢请求超过 10% 开启熔断 + .slowCallRateThreshold(10.0F) + // 请求超过 1s 就是慢请求 + .slowCallDurationThreshold(Duration.ofSeconds(1)) + // 滑动窗口大小为 100，默认值 + .slidingWindowSize(100) + .build() + ) /** * 构建日志处理的异步线程池 @@ -390,7 +411,9 @@ object LoggerService { logger.info("Finished archiving log $archivedCount files") // 同步所有存储状态到log服务端 - logResourceApi.updateStorageMode(elementId2LogProperty.values.toList(), executeCount) + doWithCircuitBreaker { + logResourceApi.updateStorageMode(elementId2LogProperty.values.toList(), executeCount) + } logger.info("Finished update mode to log service.") } catch (ignored: Throwable) { logger.warn("Fail to archive log files", ignored) @@ -412,7 +435,9 @@ object LoggerService { } // 通过上报的结果感知是否需要调整模式 - val result = logResourceApi.addLogMultiLine(buildVariables?.buildId ?: "", logMessages) + val result = doWithCircuitBreaker { + logResourceApi.addLogMultiLine(buildVariables?.buildId ?: "", logMessages) + } when { // 当log服务返回拒绝请求或者并发量超限制时，自动切换模式为本地保存并归档 result.status == 503 || result.status == 509 -> { @@ -460,13 +485,15 @@ object LoggerService { try { currentTaskLineNo = 0 logger.info("Start to finish the log, property: ${elementId2LogProperty[tag]}") - val result = logResourceApi.finishLog( - tag = tag, - jobId = jobId, - executeCount = executeCount, - subTag = subTag, - logMode = elementId2LogProperty[tag]?.logStorageMode - ) + val result = doWithCircuitBreaker { + logResourceApi.finishLog( + tag = tag, + jobId = jobId, + executeCount = executeCount, + subTag = subTag, + logMode = elementId2LogProperty[tag]?.logStorageMode + ) + } if (result.isNotOk()) { logger.error("Fail to send the log status ：${result.message}") } @@ -485,4 +512,15 @@ object LoggerService { logger.warn("Set AgentEnv logMode to ${LogStorageMode.LOCAL.name}") AgentEnv.setLogMode(LogStorageMode.LOCAL) } + + private fun doWithCircuitBreaker( + action: () -> T + ): T { + return circuitBreakerRegistry.let { + val breaker = it.circuitBreaker(this.javaClass.name) + breaker.executeCallable { + action() + } + } + } } From 579052a14cefbeb93ac74cd5bebef49fbf71ed84 Mon Sep 17 00:00:00 2001 From: royalhuang Date: Fri, 27 Oct 2023 17:41:10 +0800 Subject: [PATCH 6/9] =?UTF-8?q?bug:=20=E6=9E=84=E5=BB=BA=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E7=9A=84=E6=9C=8D=E5=8A=A1=E8=B0=83=E7=94=A8=E7=AB=AF=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0=E8=AF=B7=E6=B1=82=E7=86=94=E6=96=AD=E6=9C=BA=E5=88=B6?= =?UTF-8?q?=20#9602?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../devops/process/engine/control/DependOnControlTest.kt | 3 ++- .../tencent/devops/process/engine/control/MutexControlTest.kt | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/backend/ci/core/process/biz-engine/src/test/kotlin/com/tencent/devops/process/engine/control/DependOnControlTest.kt b/src/backend/ci/core/process/biz-engine/src/test/kotlin/com/tencent/devops/process/engine/control/DependOnControlTest.kt index cc8954f2fef..89b55e2dd0b 100644 --- a/src/backend/ci/core/process/biz-engine/src/test/kotlin/com/tencent/devops/process/engine/control/DependOnControlTest.kt +++ b/src/backend/ci/core/process/biz-engine/src/test/kotlin/com/tencent/devops/process/engine/control/DependOnControlTest.kt @@ -39,11 +39,12 @@ import io.mockk.mockk import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test +import java.time.Duration class DependOnControlTest { private val pipelineContainerService: PipelineContainerService = mockk() private val client: Client = mockk() - private val buildLogPrinter: BuildLogPrinter = BuildLogPrinter(client) + private val buildLogPrinter: BuildLogPrinter = BuildLogPrinter(client, mockk()) private val dependOnControl = DependOnControl( pipelineContainerService = pipelineContainerService, buildLogPrinter = buildLogPrinter diff --git a/src/backend/ci/core/process/biz-engine/src/test/kotlin/com/tencent/devops/process/engine/control/MutexControlTest.kt b/src/backend/ci/core/process/biz-engine/src/test/kotlin/com/tencent/devops/process/engine/control/MutexControlTest.kt index fd63cdd994b..706d7ceacff 100644 --- a/src/backend/ci/core/process/biz-engine/src/test/kotlin/com/tencent/devops/process/engine/control/MutexControlTest.kt +++ b/src/backend/ci/core/process/biz-engine/src/test/kotlin/com/tencent/devops/process/engine/control/MutexControlTest.kt @@ -43,7 +43,7 @@ import org.junit.jupiter.api.Test @Suppress("ALL", "UNUSED") class MutexControlTest { - private val buildLogPrinter: BuildLogPrinter = BuildLogPrinter(mockk()) + private val buildLogPrinter: BuildLogPrinter = BuildLogPrinter(mockk(), mockk()) private val redisOperation: RedisOperation = RedisOperation(mockk()) private val variables: Map = mapOf(Pair("var1", "Test")) private val buildId: String = "b-12345678901234567890123456789012" From 264290e13d5baf98517d95a1628e0178ea8b189a Mon Sep 17 00:00:00 2001 From: royalhuang Date: Fri, 27 Oct 2023 17:41:22 +0800 Subject: [PATCH 7/9] =?UTF-8?q?bug:=20=E6=9E=84=E5=BB=BA=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E7=9A=84=E6=9C=8D=E5=8A=A1=E8=B0=83=E7=94=A8=E7=AB=AF=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0=E8=AF=B7=E6=B1=82=E7=86=94=E6=96=AD=E6=9C=BA=E5=88=B6?= =?UTF-8?q?=20#9602?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tencent/devops/process/engine/control/DependOnControlTest.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/src/backend/ci/core/process/biz-engine/src/test/kotlin/com/tencent/devops/process/engine/control/DependOnControlTest.kt b/src/backend/ci/core/process/biz-engine/src/test/kotlin/com/tencent/devops/process/engine/control/DependOnControlTest.kt index 89b55e2dd0b..05369547219 100644 --- a/src/backend/ci/core/process/biz-engine/src/test/kotlin/com/tencent/devops/process/engine/control/DependOnControlTest.kt +++ b/src/backend/ci/core/process/biz-engine/src/test/kotlin/com/tencent/devops/process/engine/control/DependOnControlTest.kt @@ -39,7 +39,6 @@ import io.mockk.mockk import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test -import java.time.Duration class DependOnControlTest { private val pipelineContainerService: PipelineContainerService = mockk() From 91c5c6df93d9d74f34c5f2f81a2c018f4485a052 Mon Sep 17 00:00:00 2001 From: royalhuang Date: Sat, 28 Oct 2023 14:28:00 +0800 Subject: [PATCH 8/9] =?UTF-8?q?bug:=20=E6=9E=84=E5=BB=BA=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E7=9A=84=E6=9C=8D=E5=8A=A1=E8=B0=83=E7=94=A8=E7=AB=AF=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0=E8=AF=B7=E6=B1=82=E7=86=94=E6=96=AD=E6=9C=BA=E5=88=B6?= =?UTF-8?q?=20#9602?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../log/cron/impl/IndexCleanJobESImpl.kt | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/cron/impl/IndexCleanJobESImpl.kt b/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/cron/impl/IndexCleanJobESImpl.kt index 76c50006db2..2507aeb8c39 100644 --- a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/cron/impl/IndexCleanJobESImpl.kt +++ b/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/cron/impl/IndexCleanJobESImpl.kt @@ -67,18 +67,14 @@ class IndexCleanJobESImpl @Autowired constructor( @Scheduled(cron = "0 0 2 * * ?") override fun cleanIndex() { logger.info("Start to clean index") - val redisLock = RedisLock(redisOperation, ES_INDEX_CLOSE_JOB_KEY, 20) - try { - if (!redisLock.tryLock()) { - logger.info("The other process is processing clean job, ignore") - return + // #9602 每个实例轮流获得锁进行幂等清理操作 + RedisLock(redisOperation, ES_INDEX_CLOSE_JOB_KEY, 20).use { + try { + makeColdESIndexes() + deleteESIndexes() + } catch (ignore: Throwable) { + logger.warn("Fail to clean the index", ignore) } - makeColdESIndexes() - deleteESIndexes() - } catch (ignore: Throwable) { - logger.warn("Fail to clean the index", ignore) - } finally { - redisLock.unlock() } } From fe80dee5de9f7da5dc7c42f96b7ab95541b5faf7 Mon Sep 17 00:00:00 2001 From: royalhuang Date: Tue, 31 Oct 2023 15:25:30 +0800 Subject: [PATCH 9/9] =?UTF-8?q?bug:=20=E6=9E=84=E5=BB=BA=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E7=9A=84=E6=9C=8D=E5=8A=A1=E8=B0=83=E7=94=A8=E7=AB=AF=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0=E8=AF=B7=E6=B1=82=E7=86=94=E6=96=AD=E6=9C=BA=E5=88=B6?= =?UTF-8?q?=20#9602?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../devops/log/service/impl/LogServiceESImpl.kt | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/service/impl/LogServiceESImpl.kt b/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/service/impl/LogServiceESImpl.kt index c9540de3fe3..d20e3cc680f 100644 --- a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/service/impl/LogServiceESImpl.kt +++ b/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/service/impl/LogServiceESImpl.kt @@ -43,6 +43,7 @@ import com.tencent.devops.common.redis.RedisLock import com.tencent.devops.common.redis.RedisOperation import com.tencent.devops.common.web.utils.I18nUtil import com.tencent.devops.log.client.LogClient +import com.tencent.devops.log.es.ESClient import com.tencent.devops.log.event.LogOriginEvent import com.tencent.devops.log.event.LogStatusEvent import com.tencent.devops.log.event.LogStorageEvent @@ -55,6 +56,7 @@ import com.tencent.devops.log.service.LogStatusService import com.tencent.devops.log.service.LogTagService import com.tencent.devops.log.util.Constants import com.tencent.devops.log.util.ESIndexUtils +import com.tencent.devops.log.util.IndexNameUtils import org.elasticsearch.ElasticsearchStatusException import org.elasticsearch.action.admin.indices.open.OpenIndexRequest import org.elasticsearch.action.bulk.BulkRequest @@ -1093,7 +1095,7 @@ class LogServiceESImpl constructor( indexCache.put(index, true) true } else { - true + false } } @@ -1121,10 +1123,16 @@ class LogServiceESImpl constructor( } private fun createIndex(buildId: String, index: String): Boolean { + val createClient = logClient.hashClient(buildId) + // 提前创建第二天的索引备用 + createESIndex(createClient, IndexNameUtils.getNextIndexName()) + return createESIndex(createClient, index) + } + + private fun createESIndex(createClient: ESClient, index: String): Boolean { logger.info("[$index] Create index") var success = false val startEpoch = System.currentTimeMillis() - val createClient = logClient.hashClient(buildId) return try { logger.info( "[${createClient.clusterName}][$index]|createIndex|: shards[${createClient.shards}]" +