FluentD to Fluenbit for S3 Shipping

[max-blue]

Hello - I am trying to convert fluentd with fluentbit to ship logs to s3. Fluentd is giving us trouble, as logs don’t show correctly. I have already converted the config, I was wondering if someone could check the config and let me know if it is correct:

FluentD config:

04_outputs.conf: |-
        <label @OUTPUT>
          <filter **>
            @type record_transformer
            enable_ruby
            <record>
              environment ${record["kubernetes"]["namespace_name"]}
              pod ${record["kubernetes"]["pod_name"]}
              podid ${record["kubernetes"]["pod_id"]}
              container ${record["kubernetes"]["container_name"]}
            </record>
          </filter>
          <match **>
            @type s3
            s3_bucket mybucket
            s3_region us-east-1
            s3_object_key_format %Y/%m/%d/${environment}/${container}/${container}-${environment}-%Y%m%d-%H%M-${podid}-%{index}.%{file_extension}
            store_as text
            <format>
              @type single_value
              message_key log
            </format>
            # if you want to use ${tag} or %Y/%m/%d/ like syntax in path / s3_object_key_format,
            # need to specify tag for ${tag} and time for %Y/%m/%d in <buffer> argument.
            <buffer environment,container,podid,time>
              @type file
              path /var/log/fluentd-s3-buffers
              timekey 300 # 5 minute partition
              timekey_wait 60
              timekey_use_utc false
              timekey_zone US/Eastern
              chunk_limit_size 256m
              flush_thread_count 8
            </buffer>
          </match>
        </label>

Fluentbit config:

config:
      service: |
        [SERVICE]
            Daemon Off
            Flush {{ .Values.flush }}
            Log_Level {{ .Values.logLevel }}
            Parsers_File parsers.conf
            Parsers_File custom_parsers.conf
            HTTP_Server On
            HTTP_Listen 0.0.0.0
            HTTP_Port {{ .Values.metricsPort }}
            Health_Check On

      inputs: |
        [INPUT]
            Name              tail
            Tag               s3logs.*
            Path              /var/log/containers/*.log
            parser            cri         
            Mem_Buf_Limit     5MB
            Skip_Long_Lines   On
            Skip_Empty_Lines  On
            Refresh_Interval  10
        [FILTER]
            Name              modify
            Match             s3logs*
            Add               environment ${kubernetes.namespace_name}
            Add               pod ${kubernetes.pod_name}
            Add               podid ${kubernetes.pod_id}
            Add               container ${kubernetes.container_name}                

      outputs: |
        [OUTPUT]
            Name s3
            Match s3logs.*
            bucket                       mybucket
            region                       us-east-1
            s3_key_format                %Y/%m/%d/${environment}/${container}/${container}-${environment}-%Y%m%d-%H%M-${podid}-%{index}.%{file_extension}
            store_dir                    /var/log/fluentbit-s3-buffers
            total_file_size              256MB
      customParsers: |
        [PARSER]
            Name        cri
            Format      regex
            Regex       ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<log>.*)$
            Time_Key    time
            Time_Format %Y-%m-%dT%H:%M:%S.%L%z    

Any help is appreciated.

[Bharathkumarraju]

@max-blue mine is like this but failing to get the namespace name

kind: ConfigMap
metadata:
  name: fluent-bit
  namespace: exchange-utilities
data:
  custom_parsers.conf: |
    [PARSER]
        Name docker_no_time
        Format json
        Time_Keep Off
        Time_Key time
        Time_Format %Y-%m-%dT%H:%M:%S.%L

  fluent-bit.conf: |
    [SERVICE]
        Daemon Off
        Flush 1
        Log_Level debug
        Parsers_File /fluent-bit/etc/parsers.conf
        Parsers_File /fluent-bit/etc/conf/custom_parsers.conf
        HTTP_Server On
        HTTP_Listen 0.0.0.0
        HTTP_Port 2020
        Health_Check On

    [INPUT]
        Name tail
        Path /var/log/containers/flask-app*.log
        multiline.parser docker, cri
        Tag kube.*
        Mem_Buf_Limit 5MB
        Skip_Long_Lines On

    [FILTER]
        Name kubernetes
        Match kube.*
        Merge_Log On
        Keep_Log Off
        K8S-Logging.Parser On
        K8S-Logging.Exclude Off
        Labels Off
        Annotations Off
        Kube_URL https://kubernetes.default.svc:443
        Kube_Tag_Prefix kube.var.log.containers.
        Use_Kubelet Off
        Kubelet_Port 10250
        Buffer_Size 0

    [OUTPUT]
        Name s3
        Match kube.*
        bucket abex-app-compliance-bucket-staging-sin
        region ap-southeast-1
        use_put_object On
        total_file_size 1M
        upload_timeout 1m
        store_dir /fluent-bit/s3
        s3_key_format /aws/eks/${CLUSTER_NAME}/${kubernetes['namespace_name']}/${kubernetes['pod_name']}/${kubernetes['container_name']}/%Y/%m/%d/%H/%M/%S/log.gz
        compression gzip
        tls On
        ```