jenkins-pipeline/k8s_yaml/ELK/filebast/02-filebeat-configmap.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: filebeat-config
  namespace: kube-system
data:
  filebeat.yml: |
    setup.ilm.enabled: false
    setup.template.enabled: false

    filebeat.autodiscover:
      providers:
        - type: kubernetes
          node: ${NODE_NAME}
          hints.enabled: false

          templates:
            # ---------- Template 1: java语言的admin、agent、payment Pod, java21项目多行堆栈文本日志 ----------
            # - condition:
            #     # 匹配 sit 命名空间下的 3个 flymoon 应用
            #     and:
            #       - equals:
            #           kubernetes.namespace: "sit"
            #       - regexp: 
            #           kubernetes.labels.app: "(flymoon-admin|flymoon-agent|flymoon-payment)"
            #   config:
            #     - type: filestream
            #       id: "k8s-java-log-${data.kubernetes.container.id}"
            #       prospector.scanner.symlinks: true
            #       parsers:
            #         - container: ~
            #       paths:
            #         - /var/log/containers/*-${data.kubernetes.container.id}.log
            #       multiline:
            #         pattern: '^\d{4}-\d{2}-\d{2}-\d{2}:\d{2}:\d{2}\.\d{3}'
            #         negate: true
            #         match: after
            #       ignore_older: 24h
            #       scan_frequency: 10s
            #       clean_inactive: 25h
            #       close_inactive: 5m
            #       close_renamed: true
            #       start_position: beginning
            #       fields:
            #         application: ${data.kubernetes.labels.app}
            #         log_type: ${data.kubernetes.labels.log_type}
            #         environment: ${data.kubernetes.labels.environment}
            #         instance: ${data.kubernetes.host}

            #       processors:
            #         - add_kubernetes_metadata:
            #               host: ${NODE_NAME}
            #         - add_fields:
            #             fields:
            #               log_source: k8s
            #             target: 'mylog'
            #         - dissect:
            #             tokenizer: "%{timestamp} [%{thread}] %{level} %{class} - [%{method},%{line}] - %{message}"
            #             field: "message"
            #             target_prefix: "mylog"
            #             ignore_missing: true
            #             overwrite_keys: true

            # ---------- java语言的email服务的Pod, java1.8项目自由文本格式日志, java21项目格式不太一样, 但也有堆栈信息----------




            # ---------- go语言的中转服务的Pod, go项目json格式日志 ----------
            - condition:
                # 匹配 sit 命名空间下的 lessie-go-api 应用
                and:
                  - equals:
                      kubernetes.namespace: "sit"
                  - equals:
                      kubernetes.labels.app: "lessie-go-api"
              config:
                - type: filestream
                  id: "k8s-go-json-log-${data.kubernetes.container.id}"
                  prospector.scanner.symlinks: true
                  parsers:
                    - container: ~
                  paths:
                    - /var/log/containers/*-${data.kubernetes.container.id}.log
                  fields:
                    application: ${data.kubernetes.labels.app}
                    log_type: "go.log" # 显式设置 log_type 字段，用于后续 processors 中的 'when' 条件
                    environment: ${data.kubernetes.labels.environment}
                    instance: ${data.kubernetes.host}

                  ignore_older: 24h
                  scan_frequency: 10s
                  clean_inactive: 25h
                  close_inactive: 5m
                  close_renamed: true
                  start_position: beginning

                  processors:
                    - add_kubernetes_metadata:
                          host: ${NODE_NAME}
                    - add_fields:
                        fields:
                          log_source: k8s
                        target: 'mylog'
                        
                    # 核心处理器：解析 JSON 格式日志
                    - decode_json_fields:
                        fields: ["message"]
                        target: ""
                        overwrite_keys: true
                        add_error_key: true
                        

            # ---------- python语言的lessie-agent的Pod, python项目只有文本格式日志, 需排除掉一些不采集的日志 ----------
            # - condition:
            #     # 匹配 sit 命名空间下的 lessie-agent 应用
            #     and:
            #       - equals:
            #           kubernetes.namespace: "sit"
            #       - equals:
            #           kubernetes.labels.app: "lessie-agent"
            #   config:
            #     - type: filestream
            #       id: "k8s-python-log-${data.kubernetes.container.id}"
            #       prospector.scanner.symlinks: true
            #       parsers:
            #         - container: ~
            #       paths:
            #         - /var/log/containers/*-${data.kubernetes.container.id}.log
                  
            #       # 核心采集配置：只包含以时间戳开头的行
            #       include_lines: ['^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}'] 

            #       ignore_older: 24h
            #       scan_frequency: 10s
            #       clean_inactive: 25h
            #       close_inactive: 5m
            #       close_renamed: true
            #       start_position: beginning

            #       fields:
            #         application: ${data.kubernetes.labels.app} # lessie-agent
            #         log_type: "lessie_search.log" # 保持与处理器 when 条件一致
            #         environment: ${data.kubernetes.labels.environment}
            #         instance: ${data.kubernetes.host}

            #       processors:
            #         - add_kubernetes_metadata:
            #               host: ${NODE_NAME}
            #         - add_fields:
            #             fields:
            #               log_source: k8s
            #             target: 'mylog'
                    
            #         # 1. 基础 Dissect 解析
            #         - dissect:
            #             when:
            #               equals:
            #                 log_type: lessie_search.log
            #             tokenizer: '%{timestamp} - %{level} - %{module} - %{function} - %{message}'
            #             field: "message"
            #             target_prefix: "mylog"
            #             ignore_missing: true
            #             overwrite_keys: true

            #         # 2. 针对带有 [level: | event: | msg: | context:] 的日志，再做一次 dissect
            #         - dissect:
            #             when:
            #               regexp:
            #                 mylog.message: '^\[level:.*\]'
            #             tokenizer: '[level: %{event_level} | event: %{event} | msg: %{msg} | context: %{context}]'
            #             field: "mylog.message"
            #             target_prefix: "mylog"
            #             ignore_missing: true
            #             overwrite_keys: true
                        
            #         # 3. 把 context 再拆成独立字段 (JavaScript 脚本处理器)
            #         - script:
            #             lang: javascript
            #             id: parse_context
            #             source: >
            #               function process(event) {
            #                 var ctx = event.Get("mylog.context");
            #                 if (ctx) {
            #                   var parts = ctx.split(",");
            #                   parts.forEach(function(p) {
            #                     var kv = p.split(":");
            #                     if (kv.length == 2) {
            #                       // 确保 kv[0] 是有效的字段名
            #                       event.Put("mylog." + kv[0].trim(), kv[1].trim());
            #                     }
            #                   });
            #                 }
            #               }

            # ---------- python语言的apex的Pod, python项目json格式日志 ----------


            # ---------- 前端存储静态资源的nginx pod, nginx 格式日志 ----------


    # ---- 输出到 Elasticsearch ----
    output.elasticsearch:
      hosts: ["http://10.0.0.38:9200"]
      username: "admin"
      password: "G7ZSKFM4AQwHQpwA"

      # 动态索引命名：k8s-环境-应用-日期
      index: "k8s-%{[kubernetes.labels.environment]}-%{[kubernetes.labels.app]}-%{+yyyy.MM.dd}"


    logging.level: debug
    logging.selectors: ["*"]