diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7e3228c4..4d4848e7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,9 @@
# Change Log
+## 2024-03-08
+### Added
+- Support for AWS EKS system and control plane logs collection.
+
## 2024-02-13
### Added
- Changes to support Kubernetes Solution Pages Offering by OCI Logging Analytics.
@@ -7,7 +11,6 @@
- A new CronJob to handle the Kubernetes Objects discovery and Objects Logs collection using oci-logging-analytics-kubernetes-discovery Gem.
### Changed
- Moving forward, Kubernetes Objects logs would be collected using Kubernetes Discovery CronJob along with the (optional) Discovery data instead of Fluentd based Deployment.
-
## 2024-01-18
### Changed
- Management Agent docker image has been updated to version 1.2.0
diff --git a/charts/logan/Chart.yaml b/charts/logan/Chart.yaml
index f9c34098..1924bae4 100644
--- a/charts/logan/Chart.yaml
+++ b/charts/logan/Chart.yaml
@@ -5,7 +5,7 @@ apiVersion: v2
name: oci-onm-logan
description: Charts for sending Kubernetes platform logs, compute logs, and Kubernetes Objects information to OCI Logging Analytics.
type: application
-version: 3.3.0
+version: 3.4.0
appVersion: "3.0.0"
dependencies:
diff --git a/charts/logan/templates/ekscp-logs-configmap.yaml b/charts/logan/templates/ekscp-logs-configmap.yaml
new file mode 100644
index 00000000..5bd9942d
--- /dev/null
+++ b/charts/logan/templates/ekscp-logs-configmap.yaml
@@ -0,0 +1,307 @@
+# Copyright (c) 2023, Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v1.0 as shown at https://oss.oracle.com/licenses/upl.
+{{- if .Values.enableEKSControlPlaneLogs }}
+{{- $kubernetesClusterName := (include "logan.kubernetesClusterName" .) }}
+{{- $kubernetesClusterId := (include "logan.kubernetesClusterId" .) }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: {{ include "logan.resourceNamePrefix" . }}-ekscp-logs
+ namespace: {{ include "logan.namespace" . }}
+data:
+ # file-like keys
+ fluent.conf: |
+
+ {{- $authtype := .Values.authtype | lower }}
+
+ {{- $multiWorkersEnabled := false }}
+ {{- $workers := (int .Values.fluentd.multiProcessWorkers | default 0) }}
+ {{- if gt $workers 0 }}
+ {{- $multiWorkersEnabled = true }}
+
+ workers {{ $workers }}
+
+ {{- else }}
+ {{- /* fake it to run at least one range loop if no multiProcessWorkers enabled. */}}
+ {{- $workers = 1 }}
+ {{- end }}
+
+ {{- range until $workers }}
+ {{- $currWorker := . }}
+
+ {{- if $multiWorkersEnabled }}
+
+ {{- end }}
+
+ # To ignore all the fluentd core generated events
+
+
+ {{- if $.Values.fluentd.eksControlPlane }}
+
+ {{- range $name, $logDefinition := $.Values.fluentd.eksControlPlane.logs }}
+ {{- $workerId := 0 }}
+ {{- if $multiWorkersEnabled }}
+ {{- if and (eq "audit" $name) (eq "cloudwatch" $.Values.fluentd.eksControlPlane.collectionType) }}
+ {{- if $.Values.fluentd.eksControlPlane.logs.apiserver.worker }}
+ {{ $workerId = $.Values.fluentd.eksControlPlane.logs.apiserver.worker }}
+ {{- else if $.Values.fluentd.eksControlPlane.worker }}
+ {{ $workerId = $.Values.fluentd.eksControlPlane.worker }}
+ {{- end }}
+ {{- else }}
+ {{- if $logDefinition.worker }}
+ {{ $workerId = $logDefinition.worker }}
+ {{- else if $.Values.fluentd.eksControlPlane.worker }}
+ {{ $workerId = $.Values.fluentd.eksControlPlane.worker }}
+ {{- end }}
+ {{- end }}
+ {{- end }}
+ {{- if eq $currWorker (int $workerId) }}
+
+ {{- if eq "cloudwatch" $.Values.fluentd.eksControlPlane.collectionType }}
+ {{- if eq "apiserver" $name }}
+
+ @type cloudwatch_logs
+ @id {{ $name }}{{- ternary (print "_" $currWorker) "" $multiWorkersEnabled }}
+ tag rewrite.eks{{- ternary (print "." $currWorker) "" $multiWorkersEnabled }}.cp.{{ $name }}.*
+ region {{ $.Values.fluentd.eksControlPlane.region | required (printf "fluentd.eksControlPlane.region is required") }}
+ {{- if eq $.Values.fluentd.eksControlPlane.awsUseSts true }}
+ aws_use_sts true
+ aws_sts_role_arn "{{ $.Values.fluentd.eksControlPlane.awsStsRoleArn | required (printf "fluentd.eksControlPlane.awsStsRoleArn is required") }}"
+ {{- else }}
+ aws_key_id "{{ $.Values.fluentd.eksControlPlane.awsKeyId | required (printf "fluentd.eksControlPlane.awsKeyId is required") }}"
+ aws_sec_key "{{ $.Values.fluentd.eksControlPlane.awsSecKey | required (printf "fluentd.eksControlPlane.awsSecKey is required") }}"
+ {{- end }}
+ {{- if $.Values.fluentd.eksControlPlane.cwLogGroupName }}
+ log_group_name "{{ $.Values.fluentd.eksControlPlane.cwLogGroupName }}"
+ {{- else if contains "/" $kubernetesClusterId}}
+ {{- $clusterNameFromId := (splitList "/" $kubernetesClusterId) | last }}
+ log_group_name "{{ printf "/aws/eks/%s/cluster" $clusterNameFromId }}"
+ {{- end }}
+ use_log_stream_name_prefix true
+ log_stream_name {{ $logDefinition.cwLogStreamName | required (printf "fluentd.eksControlPlane.logs.%s.cwLogStreamName is required" $name) }}
+ include_metadata true
+
+ @type local
+ path {{ $.Values.fluentd.baseDir }}/oci_la_fluentd_outplugin/pos/ekscp.{{ $name }}.logs.pos
+
+
+ @type none
+
+
+
+ @type rewrite_tag_filter
+ hostname_command "cat /etc/hostname"
+
+ key message
+ pattern /\\?"kind\\?":\\?"Event\\?"/
+ tag eks{{- ternary (print "." $currWorker) "" $multiWorkersEnabled }}.cp.audit.*
+
+
+ key message
+ pattern /\\?"kind\\?":\\?"Event\\?"/
+ invert true
+ tag eks{{- ternary (print "." $currWorker) "" $multiWorkersEnabled }}.cp.{{ $name }}.*
+
+
+ {{- end }}
+
+ {{- if or (eq "authenticator" $name) (eq "kubecontrollermanager" $name) (eq "cloudcontrollermanager" $name) (eq "scheduler" $name) }}
+
+ @type cloudwatch_logs
+ @id {{ $name }}{{- ternary (print "_" $currWorker) "" $multiWorkersEnabled }}
+ tag eks{{- ternary (print "." $currWorker) "" $multiWorkersEnabled }}.cp.{{ $name }}.*
+ region {{ $.Values.fluentd.eksControlPlane.region | required (printf "fluentd.eksControlPlane.region is required") }}
+ {{- if eq $.Values.fluentd.eksControlPlane.awsUseSts true }}
+ aws_use_sts true
+ aws_sts_role_arn "{{ $.Values.fluentd.eksControlPlane.awsStsRoleArn | required (printf "fluentd.eksControlPlane.awsStsRoleArn is required") }}"
+ {{- else }}
+ aws_key_id "{{ $.Values.fluentd.eksControlPlane.awsKeyId | required (printf "fluentd.eksControlPlane.awsKeyId is required") }}"
+ aws_sec_key "{{ $.Values.fluentd.eksControlPlane.awsSecKey | required (printf "fluentd.eksControlPlane.awsSecKey is required") }}"
+ {{- end }}
+ {{- if $.Values.fluentd.eksControlPlane.cwLogGroupName }}
+ log_group_name "{{ $.Values.fluentd.eksControlPlane.cwLogGroupName }}"
+ {{- else if contains "/" $kubernetesClusterId}}
+ {{- $clusterNameFromId := (splitList "/" $kubernetesClusterId) | last }}
+ log_group_name "{{ printf "/aws/eks/%s/cluster" $clusterNameFromId }}"
+ {{- end }}
+ use_log_stream_name_prefix true
+ log_stream_name {{ $logDefinition.cwLogStreamName | required (printf "fluentd.eksControlPlane.logs.%s.cwLogStreamName is required" $name) }}
+ include_metadata true
+
+ @type local
+ path {{ $.Values.fluentd.baseDir }}/oci_la_fluentd_outplugin/pos/ekscp.{{ $name }}.logs.pos
+
+
+ @type none
+
+
+ {{- end }}
+
+
+ @type record_transformer
+ enable_ruby true
+
+ {{- if $logDefinition.metadata }}
+ oci_la_metadata ${{"{{"}}"Kubernetes Cluster Name":"{{ $kubernetesClusterName }}", "Kubernetes Cluster ID": "{{ $kubernetesClusterId }}", "Node": "#{ENV['K8S_NODE_NAME'] || 'UNDEFINED'}" {{- range $k, $v := $logDefinition.metadata }},{{ $k | quote }}: {{ $v | quote -}} {{- end }}{{"}}"}}
+ {{- else if $.Values.fluentd.eksControlPlane.metadata }}
+ oci_la_metadata ${{"{{"}}"Kubernetes Cluster Name":"{{ $kubernetesClusterName }}", "Kubernetes Cluster ID": "{{ $kubernetesClusterId }}", "Node": "#{ENV['K8S_NODE_NAME'] || 'UNDEFINED'}" {{- range $k, $v := $.Values.fluentd.eksControlPlane.metadata }},{{ $k | quote }}: {{ $v | quote -}} {{- end }}{{"}}"}}
+ {{- else }}
+ oci_la_metadata ${{"{{"}}"Kubernetes Cluster Name":"{{ $kubernetesClusterName }}", "Kubernetes Cluster ID": "{{ $kubernetesClusterId }}", "Node": "#{ENV['K8S_NODE_NAME'] || 'UNDEFINED'}" {{- range $k, $v := $.Values.metadata }},{{ $k | quote }}: {{ $v | quote -}} {{- end }}{{"}}"}}
+ {{- end }}
+ {{- if $logDefinition.ociLALogGroupID }}
+ oci_la_log_group_id "{{ $logDefinition.ociLALogGroupID }}"
+ {{- else if $.Values.fluentd.eksControlPlane.ociLALogGroupID }}
+ oci_la_log_group_id "{{ $.Values.fluentd.eksControlPlane.ociLALogGroupID }}"
+ {{- else }}
+ oci_la_log_group_id "{{ required "ociLALogGroupID is required" $.Values.ociLALogGroupID }}"
+ {{- end }}
+ oci_la_log_source_name "{{ $logDefinition.ociLALogSourceName | required (printf "fluentd.eksControlPlane.logs.%s.ociLALogSourceName is required" $name) }}"
+ {{- if $logDefinition.ociLALogSet }}
+ oci_la_log_set "{{ $logDefinition.ociLALogSet }}"
+ {{- else }}
+ oci_la_log_set "{{ $.Values.fluentd.eksControlPlane.ociLALogSet | default $.Values.ociLALogSet }}"
+ {{- end }}
+ message "${record['message']}"
+ tag ${tag}
+
+
+
+ @type record_transformer
+ enable_ruby true
+
+ oci_la_metadata ${record["oci_la_metadata"].merge({"cloudwatchloggroupname" => record.dig("metadata", "log_group_name"), "cloudwatchlogstreamname" => record.dig("metadata", "log_stream_name")})}
+
+ remove_keys $.metadata
+
+
+ {{- else }}
+
+ @type s3
+ @id {{ $name }}{{- ternary (print "_" $currWorker) "" $multiWorkersEnabled }}
+ tag eks{{- ternary (print "." $currWorker) "" $multiWorkersEnabled }}.cp.{{ $name }}.*
+ s3_bucket {{ $.Values.fluentd.eksControlPlane.s3Bucket | required (printf "fluentd.eksControlPlane.s3Bucket is required") }}
+ s3_region {{ $.Values.fluentd.eksControlPlane.region | required (printf "fluentd.eksControlPlane.region is required") }}
+ {{- if eq $.Values.fluentd.eksControlPlane.awsUseSts true }}
+
+ role_arn "{{ $.Values.fluentd.eksControlPlane.awsStsRoleArn | required (printf "fluentd.eksControlPlane.awsStsRoleArn is required") }}"
+ role_session_name oci-kubernetes-monitoring
+
+ {{- else }}
+ aws_key_id "{{ $.Values.fluentd.eksControlPlane.awsKeyId | required (printf "fluentd.eksControlPlane.awsKeyId is required") }}"
+ aws_sec_key "{{ $.Values.fluentd.eksControlPlane.awsSecKey | required (printf "fluentd.eksControlPlane.awsSecKey is required") }}"
+ {{- end }}
+ add_object_metadata true
+ store_as gzip
+
+ queue_name {{ $logDefinition.sqsQueue }}
+
+ match_regexp {{ $logDefinition.objectKey }}.*
+
+ @type none
+
+
+ {{- if $logDefinition.multilineStartRegExp }}
+ # Concat filter to handle multi-line log records.
+
+ @type concat
+ key message
+ stream_identity_key stream
+ flush_interval "{{ $.Values.fluentd.tailPlugin.flushInterval }}"
+ timeout_label "@NORMAL{{- ternary (print "." $currWorker) "" $multiWorkersEnabled }}"
+ multiline_start_regexp {{ $logDefinition.multilineStartRegExp }}
+
+ {{- end }}
+
+ @type record_transformer
+ enable_ruby true
+
+ {{- if $logDefinition.metadata }}
+ oci_la_metadata ${{"{{"}}"Kubernetes Cluster Name":"{{ $kubernetesClusterName }}", "Kubernetes Cluster ID": "{{ $kubernetesClusterId }}", "Node": "#{ENV['K8S_NODE_NAME'] || 'UNDEFINED'}" {{- range $k, $v := $logDefinition.metadata }},{{ $k | quote }}: {{ $v | quote -}} {{- end }}{{"}}"}}
+ {{- else if $.Values.fluentd.eksControlPlane.metadata }}
+ oci_la_metadata ${{"{{"}}"Kubernetes Cluster Name":"{{ $kubernetesClusterName }}", "Kubernetes Cluster ID": "{{ $kubernetesClusterId }}", "Node": "#{ENV['K8S_NODE_NAME'] || 'UNDEFINED'}" {{- range $k, $v := $.Values.fluentd.eksControlPlane.metadata }},{{ $k | quote }}: {{ $v | quote -}} {{- end }}{{"}}"}}
+ {{- else }}
+ oci_la_metadata ${{"{{"}}"Kubernetes Cluster Name":"{{ $kubernetesClusterName }}", "Kubernetes Cluster ID": "{{ $kubernetesClusterId }}", "Node": "#{ENV['K8S_NODE_NAME'] || 'UNDEFINED'}" {{- range $k, $v := $.Values.metadata }},{{ $k | quote }}: {{ $v | quote -}} {{- end }}{{"}}"}}
+ {{- end }}
+ {{- if $logDefinition.ociLALogGroupID }}
+ oci_la_log_group_id "{{ $logDefinition.ociLALogGroupID }}"
+ {{- else if $.Values.fluentd.eksControlPlane.ociLALogGroupID }}
+ oci_la_log_group_id "{{ $.Values.fluentd.eksControlPlane.ociLALogGroupID }}"
+ {{- else }}
+ oci_la_log_group_id "{{ required "ociLALogGroupID is required" $.Values.ociLALogGroupID }}"
+ {{- end }}
+ oci_la_log_source_name "{{ $logDefinition.ociLALogSourceName | required (printf "fluentd.eksControlPlane.logs.%s.ociLALogSourceName is required" $name) }}"
+ {{- if $logDefinition.ociLALogSet }}
+ oci_la_log_set "{{ $logDefinition.ociLALogSet }}"
+ {{- else }}
+ oci_la_log_set "{{ $.Values.fluentd.eksControlPlane.ociLALogSet | default $.Values.ociLALogSet }}"
+ {{- end }}
+ message "${record['message']}"
+ tag ${tag}
+
+
+
+ @type record_transformer
+ enable_ruby true
+
+ oci_la_metadata ${record["oci_la_metadata"].merge({"cloudwatchloggroupname" => record["s3_key"].split("/")[0].gsub("_", "/"), "cloudwatchlogstreamname" => record["s3_key"].split("/")[2].gsub("_", "/")})}
+
+ remove_keys $.metadata
+
+ {{- end }}
+
+ {{- end }}
+ {{- end }}
+ {{- end }}
+
+ # Match block to ensure all the logs including concat plugin timeout logs will have same label
+
+ @type relabel
+ @label @NORMAL{{- ternary (print "." $currWorker) "" $multiWorkersEnabled }}
+
+
+ # Match block to set info required for oci-logging-analytics fluentd output plugin
+
+
+ {{- if $multiWorkersEnabled }}
+
+
+ {{- end }}
+ {{- end }}
+{{- end }}
diff --git a/charts/logan/templates/fluentd-deployment.yaml b/charts/logan/templates/fluentd-deployment.yaml
new file mode 100644
index 00000000..582f0a8d
--- /dev/null
+++ b/charts/logan/templates/fluentd-deployment.yaml
@@ -0,0 +1,106 @@
+# Copyright (c) 2023, Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v1.0 as shown at https://oss.oracle.com/licenses/upl.
+
+---
+{{- if .Values.enableEKSControlPlaneLogs }}
+{{- $authtype := .Values.authtype | lower }}
+{{- $imagePullSecrets := .Values.image.imagePullSecrets }}
+{{- $resourceNamePrefix := (include "logan.resourceNamePrefix" .) }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: {{ $resourceNamePrefix }}-logan
+ namespace: {{ include "logan.namespace" . }}
+ labels:
+ app: {{ $resourceNamePrefix }}-logan
+ version: v1
+spec:
+ selector:
+ matchLabels:
+ app: {{ $resourceNamePrefix }}-logan
+ version: v1
+ template:
+ metadata:
+ annotations:
+ {{- if eq $authtype "config" }}
+ checksum/secrets: {{ include (print $.Template.BasePath "/oci-config-secret.yaml") . | sha256sum }}
+ {{- end}}
+ checksum/ekscpconfigmap: {{ include (print $.Template.BasePath "/ekscp-logs-configmap.yaml") . | sha256sum }}
+ labels:
+ app: {{ $resourceNamePrefix }}-logan
+ version: v1
+ spec:
+ serviceAccountName: {{ include "logan.serviceAccount" . }}
+ {{- if $imagePullSecrets }}
+ imagePullSecrets:
+ - name: {{ .Values.image.imagePullSecrets }}
+ {{- end}}
+ containers:
+ - name: {{ $resourceNamePrefix }}-ekscp-fluentd
+ image: {{ .Values.image.url }}
+ imagePullPolicy: {{ default "IfNotPresent" .Values.image.imagePullPolicy }}
+ env:
+ - name: FLUENTD_CONF
+ value: {{ .Values.fluentd.path }}/{{ .Values.fluentd.file }}
+ - name: K8S_NODE_NAME
+ valueFrom:
+ fieldRef:
+ fieldPath: spec.nodeName
+ - name: FLUENT_OCI_DEFAULT_LOGGROUP_ID
+ value: {{ .Values.ociLALogGroupID }}
+ - name: FLUENT_OCI_NAMESPACE
+ value: {{ .Values.ociLANamespace }}
+ - name: FLUENT_OCI_KUBERNETES_CLUSTER_ID
+ value: {{ include "logan.kubernetesClusterId" . }}
+ - name: FLUENT_OCI_KUBERNETES_CLUSTER_NAME
+ value: {{ include "logan.kubernetesClusterName" . }}
+ {{- if eq $authtype "config" }}
+ - name: FLUENT_OCI_CONFIG_LOCATION
+ value: {{ .Values.oci.path }}/{{ .Values.oci.file }}
+ {{- end }}
+ {{- if .Values.extraEnv }}
+ {{- toYaml .Values.extraEnv | nindent 10 }}
+ {{- end }}
+ {{- if .Values.resources }}
+ resources: {{- toYaml .Values.resources | nindent 10 }}
+ {{- end }}
+ volumeMounts:
+ # RW mount to store tail plugin output plugin buffer and logs
+ - name: basedir
+ mountPath: {{ .Values.fluentd.baseDir }}
+ {{- if eq $authtype "config" }}
+ # Mount directory where oci config exists
+ - name: ociconfigdir
+ mountPath: {{ .Values.oci.path }}
+ readOnly: true
+ {{- end }}
+ # Mount directory where fluentd config exists
+ - name: ekscpfluentdconfigdir
+ mountPath: {{ .Values.fluentd.path }}
+ readOnly: true
+ {{- if .Values.extraVolumeMounts }}
+ {{- toYaml .Values.extraVolumeMounts | nindent 8 }}
+ {{- end }}
+ terminationGracePeriodSeconds: 30
+ volumes:
+ {{- if .Values.extraVolumes }}
+ {{- toYaml .Values.extraVolumes | nindent 6 }}
+ {{- end }}
+ # RW mount to store tail plugin output plugin buffer and logs
+ - name: basedir
+ hostPath:
+ path: {{ .Values.fluentd.baseDir }}
+ {{- if eq $authtype "config" }}
+ # Mount directory where oci config exists
+ - name: ociconfigdir
+ projected:
+ sources:
+ - secret:
+ name: {{ $resourceNamePrefix }}-oci-config
+ {{- end }}
+ # Mount directory where fluentd ekscp config exists
+ - name: ekscpfluentdconfigdir
+ configMap:
+ # Provide the name of the ConfigMap to mount.
+ name: {{ $resourceNamePrefix }}-ekscp-logs
+{{- end }}
diff --git a/charts/logan/values.schema.json b/charts/logan/values.schema.json
index 27e9f3c3..8c539e67 100644
--- a/charts/logan/values.schema.json
+++ b/charts/logan/values.schema.json
@@ -52,6 +52,18 @@
"type": "string"
}
}
+ },
+ "collectionType": {
+ "type": "string",
+ "enum": ["cloudwatch", "s3"]
+ },
+ "region": {
+ "type": "string"
+ },
+ "s3Bucket": {
+ "type": "string",
+ "minLength": 3,
+ "maxLength": 63
}
}
}
diff --git a/charts/logan/values.yaml b/charts/logan/values.yaml
index fa534b51..d2d119a7 100644
--- a/charts/logan/values.yaml
+++ b/charts/logan/values.yaml
@@ -48,7 +48,7 @@ image:
# Image pull secrets for. Secret must be in the namespace defined by namespace
imagePullSecrets:
# -- Replace this value with actual docker image url
- url: container-registry.oracle.com/oci_observability_management/oci-la-fluentd-collector:1.3.0
+ url: container-registry.oracle.com/oci_observability_management/oci-la-fluentd-collector:1.4.0
# -- Image pull policy
imagePullPolicy: Always
@@ -60,7 +60,7 @@ ociLANamespace:
# e.g. ocid1.loganalyticsloggroup.oc1.phx.amaaaaasdfaskriauucc55rlwlxe4ahe2vfmtuoqa6qsgu7mb6jugxacsk6a
ociLALogGroupID:
-# -- OKE Cluster OCID
+# -- OKE Cluster OCID/EKS Cluster ARN etc.
# e.g. ocid1.cluster.oc1.phx.aaaaaaaahhbadf3rxa62faaeixanvr7vftmkg6hupycbf4qszctf2wbmqqxq
kubernetesClusterID:
@@ -77,6 +77,9 @@ ociLAClusterEntityID:
# In Kubernetes environments where SELinux mode is enforced, set this flag to 'true' to allow fluentd pods to access log files.
privileged: false
+# -- Enables collection of AWS EKS Control Plane logs through CloudWatch or S3 Fluentd plugin
+enableEKSControlPlaneLogs: false
+
# Logging Analytics additional metadata. Use this to tag all the collected logs with one or more key:value pairs.
# Key must be a valid field in Logging Analytics
#metadata:
@@ -289,7 +292,7 @@ fluentd:
# -- Kubernetes CSI Node Driver Logs collection configuration
csinode:
# csinode log files location.
- path: /var/log/containers/csi-oci-node-*.log
+ path: /var/log/containers/csi-oci-node-*.log,/var/log/containers/ebs-csi-node-*.log
# Logging Analytics log source to use for parsing and processing Kubernetes CSI Node Driver Logs.
ociLALogSourceName: "Kubernetes CSI Node Driver Logs"
@@ -309,6 +312,13 @@ fluentd:
# The regular expression pattern for the starting line in case of multi-line logs.
multilineStartRegExp: /^\S\d{2}\d{2}\s+[^\:]+:[^\:]+:[^\.]+\.\d{0,3}/
+ # -- Kubernetes CSI Controller Logs collection configuration
+ csi-controller:
+ # csi controller log files location.
+ path: /var/log/containers/ebs-csi-controller-*.log
+ # Logging Analytics log source to use for parsing and processing Kubernetes CSI Controller Logs.
+ ociLALogSourceName: "Kubernetes CSI Controller Logs"
+
# Config specific to API Server Logs Collection
kube-apiserver:
# The path to the source files.
@@ -425,6 +435,122 @@ fluentd:
# Logging Analytics log source to use for parsing and processing Linux YUM Logs.
ociLALogSourceName: "Linux YUM Logs"
+ # Configuration for AWS EKS Control Plane logs like API Server, Audit, Authenticator etc.
+ eksControlPlane:
+ # Collection Type (cloudwatch or s3)
+ collectionType: "cloudwatch"
+ # AWS region
+ region:
+ # Use AssumeRoleCredentials (https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/AssumeRoleCredentials.html) to authenticate
+ # Default is true. Set to false to use access keys
+ awsUseSts: true
+ # The role ARN to assume when using AWS Security Token Service authentication
+ awsStsRoleArn:
+ # AWS access key and secret access key, needed only when awsUseSts is explicitly set to false
+ #awsKeyId:
+ #awsSecKey:
+ # CloudWatch Log Group name of the EKS cluster. Automatically determined by extracting cluster name from kubernetesClusterId and
+ # following naming syntax as "aws/eks//cluster". Below field can be used to override this behavior.
+ #cwLogGroupName:
+ # S3 related settings
+ # S3 bucket name to which EKS Control Plane logs are being streamed using a subscription filter
+ s3Bucket:
+ ociLALogGroupID:
+ #metadata:
+ #"Client Host Region": "America"
+ #"Environment": "Production"
+ #"Third Key": "Third Value"
+ # Worker number in case of multi process workers enabled. If not set when multi process workers enabled, then it defaults to 0.
+ #worker:
+ logs:
+ # If using cloudwatch collection mechanism, apiserver and audit logs need to be part of the same worker as they share the same log stream name prefix.
+ # Thus "worker" variable is only picked up from "apiserver" section.
+ apiserver:
+ # CloudWatch Log Stream name
+ cwLogStreamName: "kube-apiserver"
+ # SQS queue name which is notified when apiserver log object is created in S3 bucket
+ sqsQueue: "apiserver"
+ # S3 object key
+ objectKey: .*?kube-apiserver/
+ # Logging Analytics log source to use for parsing and processing EKS Control Plane API Server Logs.
+ ociLALogSourceName: "Kubernetes API Server Logs"
+ multilineStartRegExp: /^\S\d{2}\d{2}\s+[^\:]+:[^\:]+:[^\.]+\.\d{0,3}/
+ #metadata:
+ #"Client Host Region": "America"
+ #"Environment": "Production"
+ #"Third Key": "Third Value"
+ #ociLALogGroupID:
+ # Worker number in case of multi process workers enabled. If not set when multi process workers enabled, then it defaults to 0.
+ #worker:
+ audit:
+ sqsQueue: "audit"
+ # S3 object key
+ objectKey: .*?kube-apiserver-audit
+ # Logging Analytics log source to use for parsing and processing EKS Control Plane Audit Logs.
+ ociLALogSourceName: "Kubernetes Audit Logs"
+ #metadata:
+ #"Client Host Region": "America"
+ #"Environment": "Production"
+ #"Third Key": "Third Value"
+ #ociLALogGroupID:
+ #worker:
+ authenticator:
+ cwLogStreamName: "authenticator"
+ sqsQueue: "authenticator"
+ # S3 object key
+ objectKey: .*?authenticator
+ # Logging Analytics log source to use for parsing and processing EKS Control Plane Authenticator Logs.
+ ociLALogSourceName: "AWS EKS Authenticator Logs"
+ multilineStartRegExp: /^time=/
+ #metadata:
+ #"Client Host Region": "America"
+ #"Environment": "Production"
+ #"Third Key": "Third Value"
+ #ociLALogGroupID:
+ #worker:
+ kubecontrollermanager:
+ cwLogStreamName: "kube-controller-manager"
+ sqsQueue: "kube-controller-manager"
+ # S3 object key
+ objectKey: .*?kube-controller-manager
+ # Logging Analytics log source to use for parsing and processing EKS Control Plane Kube Controller Manager Logs.
+ ociLALogSourceName: "Kubernetes Controller Manager Logs"
+ multilineStartRegExp: /^\S\d{2}\d{2}\s+[^\:]+:[^\:]+:[^\.]+\.\d{0,3}/
+ #metadata:
+ #"Client Host Region": "America"
+ #"Environment": "Production"
+ #"Third Key": "Third Value"
+ #ociLALogGroupID:
+ #worker:
+ cloudcontrollermanager:
+ cwLogStreamName: "cloud-controller-manager"
+ sqsQueue: "cloud-controller-manager"
+ # S3 object key
+ objectKey: .*?cloud-controller-manager
+ # Logging Analytics log source to use for parsing and processing EKS Control Plane Cloud Controller Manager Logs.
+ ociLALogSourceName: "Cloud Controller Manager Logs"
+ multilineStartRegExp: /^\S\d{2}\d{2}\s+[^\:]+:[^\:]+:[^\.]+\.\d{0,3}/
+ #metadata:
+ #"Client Host Region": "America"
+ #"Environment": "Production"
+ #"Third Key": "Third Value"
+ #ociLALogGroupID:
+ #worker:
+ scheduler:
+ cwLogStreamName: "kube-scheduler"
+ sqsQueue: "scheduler"
+ # S3 object key
+ objectKey: .*?kube-scheduler
+ # Logging Analytics log source to use for parsing and processing EKS Control Plane Scheduler Logs.
+ ociLALogSourceName: "Kubernetes Scheduler Logs"
+ multilineStartRegExp: /^\S\d{2}\d{2}\s+[^\:]+:[^\:]+:[^\.]+\.\d{0,3}/
+ #metadata:
+ #"Client Host Region": "America"
+ #"Environment": "Production"
+ #"Third Key": "Third Value"
+ #ociLALogGroupID:
+ #worker:
+
# Generic configuration for all container/pod logs
genericContainerLogs:
# -- Default Logging Analytics log source to use for parsing and processing the logs: Kubernetes Container Generic Logs.
@@ -440,6 +566,8 @@ fluentd:
- '"/var/log/containers/csi-oci-node-*.log"'
- '"/var/log/containers/proxymux-client-*.log"'
- '"/var/log/containers/cluster-autoscaler-*.log"'
+ - '"/var/log/containers/ebs-csi-node-*.log"'
+ - '"/var/log/containers/ebs-csi-controller-*.log"'
- '"/var/log/containers/kube-apiserver-*.log"'
- '"/var/log/containers/etcd-*.log"'
- '"/var/log/containers/kube-controller-manager-*.log"'
diff --git a/charts/oci-onm/Chart.yaml b/charts/oci-onm/Chart.yaml
index fa3223c9..b64d398e 100644
--- a/charts/oci-onm/Chart.yaml
+++ b/charts/oci-onm/Chart.yaml
@@ -18,7 +18,7 @@ type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 3.3.0
+version: 3.4.0
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to
@@ -32,7 +32,7 @@ dependencies:
repository: "file://../common"
condition: oci-onm-common.enabled
- name: oci-onm-logan
- version: "3.3.0"
+ version: "3.4.0"
repository: "file://../logan"
condition: oci-onm-logan.enabled
- name: oci-onm-mgmt-agent
diff --git a/charts/oci-onm/values.yaml b/charts/oci-onm/values.yaml
index cf3c2638..70415ef9 100644
--- a/charts/oci-onm/values.yaml
+++ b/charts/oci-onm/values.yaml
@@ -31,7 +31,7 @@ oci-onm-logan:
kubernetesClusterID: "{{ .Values.global.kubernetesClusterID }}"
kubernetesClusterName: "{{ .Values.global.kubernetesClusterName }}"
image:
- url: container-registry.oracle.com/oci_observability_management/oci-la-fluentd-collector:1.3.0
+ url: container-registry.oracle.com/oci_observability_management/oci-la-fluentd-collector:1.4.0
# Go to OCI Logging Analytics Administration, click Service Details, and note the namespace value.
ociLANamespace:
# OCI Logging Analytics Default Log Group OCID
diff --git a/docs/FAQ.md b/docs/FAQ.md
index cf2fbb84..be3f6deb 100644
--- a/docs/FAQ.md
+++ b/docs/FAQ.md
@@ -337,4 +337,45 @@ oci-onm-logan:
containerdataHostPath: /var/lib/docker/containers
```
+### Control plane log collection for AWS EKS (Amazon Elastic Kubernetes Service)
+AWS EKS control plane logs are available in CloudWatch.
+Once the control plane log collection is enabled, the logs are directly pulled from CloudWatch and ingested into OCI Logging Analytics for further analysis. Alternatively, the logs can be routed over to S3 and pulled from there.
+
+#### How to collect EKS control plane logs from CloudWatch?
+To collect the logs from CloudWatch directly, modify your override_values.yaml to add the following EKS specific variables. Various other variables are available in the values.yaml file and can be updated as necessary.
+
+```
+..
+..
+oci-onm-logan:
+ ..
+ ..
+ enableEKSControlPlaneLogs: true
+ fluentd:
+ ...
+ ...
+ eksControlPlane:
+ region:
+ awsStsRoleArn:
+```
+
+#### How to collect EKS control plane logs from S3?
+If you run into [CloudWatch service quotas](https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/cloudwatch_limits_cwl.html), you can alternatively route the logs to S3 and collect them. The control plane logs in S3 need to be in a specific format for the default log collection to work. Please refer [EKS CP Logs Streaming to S3](./eks-cp-logs.md) for instructions on how to configure streaming of Control Plane logs to S3 and subsequenty collect them in OCI Logging Analytics. Once the streaming of logs is setup, modify your override_values.yaml to add the following EKS specific variables. Various other variables are available in the values.yaml file and can be updated as necessary.
+
+```
+..
+..
+oci-onm-logan:
+ ..
+ ..
+ enableEKSControlPlaneLogs: true
+ fluentd:
+ ...
+ ...
+ eksControlPlane:
+ collectionType:"s3"
+ region:
+ awsStsRoleArn:
+ s3Bucket:
+```
diff --git a/docs/eks-cp-logs-streaming.png b/docs/eks-cp-logs-streaming.png
new file mode 100644
index 00000000..00bb5095
Binary files /dev/null and b/docs/eks-cp-logs-streaming.png differ
diff --git a/docs/eks-cp-logs.md b/docs/eks-cp-logs.md
new file mode 100644
index 00000000..a6671bc8
--- /dev/null
+++ b/docs/eks-cp-logs.md
@@ -0,0 +1,292 @@
+## Streaming of Control Plane logs from CloudWatch to S3
+
+We can use a CloudWatch logs subscription to stream log data in near real-time to AWS S3. Once available in S3, the log data can be pulled and ingested into OCI Logging Analytics.
+
+The high level flow of CloudWatch logs to S3 looks as follows
+
+
+
+The steps to be followed include:
+
+### Create a new Lambda function
+
+Create a new Lambda function using "Process CloudWatch logs sent to Kinesis Firehose" blueprint, preferably with Node.js 14.x runtime. Once created, update Lambda's *processRecords* function in *index.mjs* file with the below code. Take a note of the Function ARN as it would be needed during the creation of Firehose delivery stream.
+
+```
+function processRecords (records) {
+ return records.map(r => {
+ const data = loadJsonGzipBase64(r.data)
+ const recId = r.recordId
+ // CONTROL_MESSAGE are sent by CWL to check if the subscription is reachable.
+ // They do not contain actual data.
+ if (data.messageType === 'CONTROL_MESSAGE') {
+ return {
+ result: 'Dropped',
+ recordId: recId
+ }
+ } else if (data.messageType === 'DATA_MESSAGE') {
+ // Replace "/" with an "_"
+ let logGroupName = data.logGroup.replace(/\//g, '_')
+ let logStreamName = data.logStream.replace(/\//g, '_')
+ let prefix
+ if (logStreamName.startsWith("kube-apiserver-audit")) {
+ prefix = logGroupName + "/" + "kube-apiserver-audit/" + logStreamName
+ } else if (logStreamName.startsWith("kube-apiserver")) {
+ prefix = logGroupName + "/" + "kube-apiserver/" + logStreamName
+ } else if (logStreamName.startsWith("authenticator")) {
+ prefix = logGroupName + "/" + "authenticator/" + logStreamName
+ } else if (logStreamName.startsWith("kube-controller-manager")) {
+ prefix = logGroupName + "/" + "kube-controller-manager/" + logStreamName
+ } else if (logStreamName.startsWith("cloud-controller-manager")) {
+ prefix = logGroupName + "/" + "cloud-controller-manager/" + logStreamName
+ } else if (logStreamName.startsWith("kube-scheduler")) {
+ prefix = logGroupName + "/" + "kube-scheduler/" + logStreamName
+ } else {
+ prefix = "default"
+ }
+ const partition_keys = {
+ object_prefix: prefix
+ };
+ const joinedData = data.logEvents.map(e => transformLogEvent(e)).join('')
+ const encodedData = Buffer.from(joinedData, 'utf-8').toString('base64')
+ return {
+ data: encodedData,
+ result: 'Ok',
+ recordId: recId,
+ metadata: { partitionKeys: partition_keys }
+ }
+ } else {
+ return {
+ result: 'ProcessingFailed',
+ recordId: recId
+ }
+ }
+ })
+}
+```
+
+### Create a subscription filter with Amazon Kinesis Data Firehose
+
+Once Lambda function is created, follow the below steps.
+
+Create a S3 bucket with the name "\" in "\" region. You can select the S3 bucket name and region as per your choice.
+
+```
+aws s3api create-bucket --bucket --create-bucket-configuration LocationConstraint=
+```
+
+Create IAM role "FirehosetoS3Role", specifying the trust policy file "TrustPolicyForFirehose.json" as shown below. This role grants Kinesis Data Firehose permission to put data into the S3 bucket created above.
+
+
+ TrustPolicyForFirehose.json
+
+```
+{
+ "Version": "2008-10-17",
+ "Statement": [
+ {
+ "Effect": "Allow",
+ "Principal": {
+ "Service": "firehose.amazonaws.com"
+ },
+ "Action": "sts:AssumeRole"
+ }
+ ]
+}
+```
+
+
+```
+aws iam create-role --role-name FirehosetoS3Role --assume-role-policy-document file://./TrustPolicyForFirehose.json
+```
+
+Create a permissions policy in file "PermissionsForFirehose.json" to define what actions Kinesis Data Firehose can do and associate it with the role "FirehosetoS3Role". Permission actions include putting objects into S3 bucket "\" and invoking Lambda function "\".
+
+
+ PermissionsForFirehose.json
+
+```
+{
+ "Statement": [
+ {
+ "Effect": "Allow",
+ "Action": [
+ "s3:AbortMultipartUpload",
+ "s3:GetBucketLocation",
+ "s3:GetObject",
+ "s3:ListBucket",
+ "s3:ListBucketMultipartUploads",
+ "s3:PutObject",
+ "lambda:InvokeFunction"
+ ],
+ "Resource": [
+ "arn:aws:s3:::",
+ "arn:aws:s3:::/*",
+ "arn:aws:lambda:::function:"
+ ]
+ }
+ ]
+}
+```
+
+
+```
+aws iam put-role-policy --role-name FirehosetoS3Role --policy-name Permissions-Policy-For-Firehose --policy-document file://./PermissionsForFirehose.json
+```
+
+#### Create Firehose delivery stream
+
+Create a destination Kinesis Data Firehose delivery stream "\". This stream uses the Lambda function "\" created earlier to extract the log events and partition them before storage into S3.
+
+```
+aws firehose create-delivery-stream --delivery-stream-name '' --extended-s3-destination-configuration '{"RoleARN": "arn:aws:iam:::role/FirehosetoS3Role", "BucketARN": "arn:aws:s3:::", "Prefix": "!{partitionKeyFromLambda:object\_prefix}/", "ErrorOutputPrefix": "errors/", "CompressionFormat": "GZIP", "DynamicPartitioningConfiguration": {"Enabled": true}, "ProcessingConfiguration": {"Enabled": true, "Processors": \[{"Type": "AppendDelimiterToRecord"},{"Type": "Lambda", "Parameters": \[{"ParameterName" :"LambdaArn", "ParameterValue" : "arn:aws:lambda:::function:"}\]}\]}}'
+```
+
+Create an IAM role "CWLtoKinesisFirehoseRole" that grants CloudWatch logs permission to put data into Kinesis Data Firehose delivery stream created above.
+
+
+ TrustPolicyForCWL.json
+
+```
+{
+ "Version": "2008-10-17",
+ "Statement": [
+ {
+ "Effect": "Allow",
+ "Principal": {
+ "Service": "logs.amazonaws.com"
+ },
+ "Action": "sts:AssumeRole",
+ "Condition": {
+ "StringLike": {
+ "aws:SourceArn": "arn:aws:logs:::*"
+ }
+ }
+ }
+ ]
+}
+```
+
+
+```
+aws iam create-role --role-name CWLtoKinesisFirehoseRole --assume-role-policy-document file://./TrustPolicyForCWL.json
+```
+
+Create a permissions policy to define what actions CloudWatch logs can do.
+
+
+ PermissionsForCWL.json
+
+```
+{
+ "Statement": [
+ {
+ "Effect": "Allow",
+ "Action": [
+ "firehose:PutRecord"
+ ],
+ "Resource": [
+ "arn:aws:firehose:::deliverystream/"
+ ]
+ }
+ ]
+}
+```
+
+
+```
+aws iam put-role-policy --role-name CWLtoKinesisFirehoseRole --policy-name Permissions-Policy-For-CWL --policy-document file://./PermissionsForCWL.json
+```
+
+#### Create logs subscription filter
+
+Create CloudWatch Logs subscription filter, choosing the appropriate CloudWatch log group name.
+
+```
+aws logs put-subscription-filter --log-group-name "/aws/eks//cluster" --filter-name "CWLToS3" --filter-pattern " " --destination-arn "arn:aws:firehose:::deliverystream/" --role-arn "arn:aws:iam:::role/CWLtoKinesisFirehoseRole"
+```
+
+Once the above steps are completed, the CloudWatch Logs will start appearing in S3 bucket. The logs would be written under the S3 bucket \_aws\_eks\_\\_cluster/logStreamType/\/ as shown below.
+
+
+
+We need to create and configure few other resources to enable us to collect the logs from S3.
+
+**Create SQS Queues**
+
+Create six SQS queues *apiserver*, *audit*, *authenticator*, *kube-controller-manager*, *cloud-controller-manager*, *scheduler* of "Standard" type and note down their ARN.
+
+**Create SNS topic**
+
+Create SNS topic like "\". Once created, edit it to add six new subscriptions, one for each of the SQS queues created above. For every subscription ensure that "Enable raw message delivery" is explicitly enabled.
+
+**SQS access policy**
+
+The SQS access policy is needed for each of the six SQS queues.
+
+The below access policy is for *apiserver* SQS queue. Update the name of the queue when creating similar policy for other SQS queues.
+
+
+ SQS access policy
+
+```
+{
+ "Statement": [
+ {
+ "Effect": "Allow",
+ "Principal": {
+ "Service": "sns.amazonaws.com"
+ },
+ "Action": "sqs:SendMessage",
+ "Resource": "arn:aws:sqs:::apiserver",
+ "Condition": {
+ "ArnEquals": {
+ "aws:SourceArn": "arn:aws:sns:::"
+ }
+ }
+ }
+ ]
+}
+```
+
+
+**SNS access policy**
+
+Also update its access policy (illustrated below) to allow S3 bucket "\" to publish to it.
+
+
+ SNS access policy
+
+```
+{
+ "Version": "2012-10-17",
+ "Id": "example-ID",
+ "Statement": [
+ {
+ "Sid": "Example SNS topic policy",
+ "Effect": "Allow",
+ "Principal": {
+ "Service": "s3.amazonaws.com"
+ },
+ "Action": "SNS:Publish",
+ "Resource": "arn:aws:sns:::",
+ "Condition": {
+ "StringEquals": {
+ "aws:SourceAccount": ""
+ },
+ "ArnLike": {
+ "aws:SourceArn": "arn:aws:s3:*:*:"
+ }
+ }
+ }
+ ]
+}
+```
+
+
+**Update S3 bucket to send notifications**
+
+Go to the bucket properties and select "Create event notification" under "Event notifications". Select "All object create events" under "Event types". In Destination, select "SNS topic" and select the SNS topic created earlier, to which the event needs to be published.
+
+##References
+[FilterWithFirehose](https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/SubscriptionFilters.html#FirehoseExample) documents the steps needed to push the logs to S3 using Kinesis Data Firehose.
diff --git a/docs/s3-partitioned-logs.png b/docs/s3-partitioned-logs.png
new file mode 100644
index 00000000..595ba38d
Binary files /dev/null and b/docs/s3-partitioned-logs.png differ
diff --git a/logan/docker-images/v1.0/oraclelinux/8-slim/Gemfile b/logan/docker-images/v1.0/oraclelinux/8-slim/Gemfile
index 986c61c6..c71b4f14 100644
--- a/logan/docker-images/v1.0/oraclelinux/8-slim/Gemfile
+++ b/logan/docker-images/v1.0/oraclelinux/8-slim/Gemfile
@@ -12,3 +12,6 @@ gem "fluent-plugin-rewrite-tag-filter", "~> 2.4.0"
gem "fluent-plugin-parser-cri", "~> 0.1.1"
gem "fluent-plugin-kubernetes_metadata_filter", "3.3.0"
gem "fluent-plugin-kubernetes-objects", "1.2.3"
+gem "fluent-plugin-record-modifier", "2.1.1"
+gem "fluent-plugin-cloudwatch-logs", "0.14.3"
+gem "fluent-plugin-s3", "1.7.2"