jenkins-pipeline/k8s_yaml/ELK/filebast/02-filebeat-configmap.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: filebeat-config
  namespace: kube-system
data:
  filebeat.yml: |
    setup.ilm.enabled: false
    setup.template.enabled: false

    filebeat.autodiscover:
      providers:
        - type: kubernetes
          templates:
            # ---------- ↓ json格式日志 ↓ ----------
            - condition:
                and:
                  - regexp:
                      kubernetes.namespace: "^(sit|apex-evaluation)$"
                  - regexp:
                      kubernetes.labels.app: "^(lessie-go-api|apex|lessie-review-service|lessie-search-service)$"
              config:
                - type: filestream
                  id: "container-${data.kubernetes.container.id}"
                  prospector.scanner.symlinks: true
                  close.on_state_change.removed: false
                  parsers:
                    - container: ~
                  paths:
                    - /var/log/containers/*-${data.kubernetes.container.id}.log
                  processors:
                    - add_kubernetes_metadata:
                        host: ${NODE_NAME}
                    - decode_json_fields:
                        fields: ["message"]
                        target: ""
                        overwrite_keys: true
                        add_error_key: true
                    - drop_fields:
                        fields: 
                          - "kubernetes.node.labels"
                          - "kubernetes.namespace_labels.kubernetes_io/metadata_name"
                        ignore_missing: true
            # ---------- ↑ json格式日志 ↑ ----------
            

            # ---------- ↓ java语言的服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↓ ----------
            - condition:
                and:
                  - equals:
                      kubernetes.namespace: sit
                  - or:
                    - equals:
                        kubernetes.labels.app: "flymoon-admin"
                    - equals:
                        kubernetes.labels.app: "flymoon-agent"
                    - equals:
                        kubernetes.labels.app: "flymoon-payment"
              config:
                - type: filestream
                  id: "container-${data.kubernetes.container.id}"
                  prospector.scanner.symlinks: true
                  close.on_state_change.removed: false
                  parsers:
                    - container: ~
                    - multiline:
                        type: pattern
                        pattern: '^\d{4}-\d{2}-\d{2}-\d{2}:\d{2}:\d{2}\.\d{3}'
                        negate: true
                        match: after
                  paths:
                    - /var/log/containers/*-${data.kubernetes.container.id}.log
                  processors:
                    - add_kubernetes_metadata:
                        host: ${NODE_NAME}
                    - dissect:
                        tokenizer: '%{timestamp}  %{level}  %{pid}  ---  [%{thread}]  %{class}  :  [%{app_name->}]  %{log_message}'
                        field: "message"
                        target_prefix: ""
                        ignore_missing: true
                        overwrite_keys: true
                    - drop_fields:
                        fields: ["kubernetes.node.labels", "kubernetes.annotations"]
                        ignore_missing: true

            # ---------- ↑ java语言的服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↑ ----------


            # ---------- ↓ java语言的服务的Pod, email 项目自由文本格式日志 ↓ ----------
            - condition:
                and:
                  - equals:
                      kubernetes.namespace: sit
                  - equals:
                      kubernetes.labels.app: "flymoon-email"
              config:
                - type: filestream
                  id: "container-${data.kubernetes.container.id}"
                  prospector.scanner.symlinks: true
                  close.on_state_change.removed: false
                  parsers:
                    - container: ~
                    - multiline:
                        type: pattern
                        pattern: '^\d{4}-\d{2}-\d{2}'
                        negate: true
                        match: after
                  paths:
                    - /var/log/containers/*-${data.kubernetes.container.id}.log
                  processors:
                    - add_kubernetes_metadata:
                        host: ${NODE_NAME}
                    - dissect:
                        tokenizer: '%{timestamp}  %{level} %{pid} --- [%{thread}] %{class} : %{log_message}'
                        field: "message"
                        target_prefix: ""
                        ignore_missing: true
                        overwrite_keys: true
                    - drop_fields:
                        fields: ["kubernetes.node.labels", "kubernetes.annotations"]
                        ignore_missing: true
            # ---------- ↑ java语言的服务的Pod, email 项目自由文本格式日志 ↑ ----------


            # ---------- ↓ python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↓ ----------
            - condition:
                and:
                  - equals:
                      kubernetes.namespace: sit
                  - equals:
                      kubernetes.labels.app: "lessie-agents"
              config:
                - type: filestream
                  id: "container-${data.kubernetes.container.id}"
                  prospector.scanner.symlinks: true
                  close.on_state_change.removed: false
                  parsers:
                    - container: ~
                  paths:
                    - /var/log/containers/*-${data.kubernetes.container.id}.log
                  processors:
                    - add_kubernetes_metadata:
                        host: ${NODE_NAME}
                    # 第一层：仅解析符合时间戳开头的日志行(for业务告警的日志格式)
                    - dissect:
                        when:
                          regexp:
                            message: '^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}.*'
                        tokenizer: '%{timestamp} - %{level} - %{module} - %{function} - %{msg_body}'
                        field: "message"
                        target_prefix: "mylog"
                        ignore_missing: true
                        overwrite_keys: true
                    # 第二层：针对带有 [level: | event: | msg: | context:] 的日志，再做一次 dissect
                    - dissect:
                        when:
                          contains:
                            mylog.msg_body: "[level:"
                        tokenizer: '[level: %{event_level} | event: %{event} | msg: %{msg} | context: %{ctx_raw}]'
                        field: "mylog.msg_body"
                        target_prefix: "mylog"
                        ignore_missing: true
                        overwrite_keys: true
                    # 第三层：把 ctx_raw 再拆成独立字段
                    - script:
                        lang: javascript
                        id: parse_context
                        source: >
                          function process(event) {
                            var ctx = event.Get("mylog.ctx_raw");
                            if (!ctx) return;
                            var parts = ctx.trim().split(",");
                            for (var i = 0; i < parts.length; i++) {
                              var pair = parts[i].split(":");
                              if (pair.length === 2) {
                                event.Put("mylog." + pair[0].trim(), pair[1].trim());
                              }
                            }
                          }
                    # 第四层: 去除大量不需要的k8s元数据字段
                    - drop_fields:
                        fields: 
                          - "kubernetes.node.labels"
                          - "kubernetes.annotations"
                        ignore_missing: true
            # ---------- ↑ python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↑ ----------


            # ---------- ↓ apex 动态创建的 python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↓ ----------
            - condition:
                and:
                  - equals:
                      kubernetes.namespace: apex-evaluation
                  - equals:
                      kubernetes.labels.apex: "lessie-agents"
              config:
                - type: filestream
                  id: "container-${data.kubernetes.container.id}"
                  prospector.scanner.symlinks: true
                  close.on_state_change.removed: false
                  parsers:
                    - container: ~
                  paths:
                    - /var/log/containers/*-${data.kubernetes.container.id}.log
                  processors:
                    - drop_fields:
                        fields: 
                          - "kubernetes.node.labels"
                          - "kubernetes.annotations"
                        ignore_missing: true
            # ---------- ↑ apex 动态创建的 python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↑ ----------



    # ---- 输出到 Elasticsearch ----
    output.elasticsearch:
      hosts: ["http://10.0.0.38:9200"]
      username: "admin"
      password: "G7ZSKFM4AQwHQpwA"

      indices:
        - index: "k8s-%{[kubernetes.labels.environment]}-%{[kubernetes.labels.app]}-%{+yyyy.MM}"
          when:
            regexp:
              kubernetes.labels.app: "(lessie-go-api|flymoon-admin|flymoon-agent|flymoon-payment|flymoon-email|lessie-agents|apex|lessie-review-service|lessie-search-service)"

        - index: "apex-python-%{+yyyy.MM}"
          when:
            equals:
              kubernetes.labels.apex: "lessie-agents"

    logging.level: info
    logging.selectors:  ["*"]