jenkins-pipeline/k8s_yaml/ELK/filebast/02-filebeat-configmap.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: filebeat-config
  namespace: kube-system
data:
  filebeat.yml: |
    setup.ilm.enabled: false
    setup.template.enabled: false

    filebeat.autodiscover:
      providers:
        - type: kubernetes
          node: ${NODE_NAME}
          hints.enabled: true

          templates:
            # ---------- Template 1: java语言的admin、agent、payment Pod, java21项目多行堆栈文本日志 ----------
            - condition:
                # 匹配 sit 命名空间下的 3个 flymoon 应用
                and:
                  - equals:
                      kubernetes.namespace: "sit"
                  - regexp: 
                      kubernetes.labels.app: "(flymoon-admin|flymoon-agent|flymoon-payment)"
              config:
                - type: filestream
                  id: "k8s-java-log-${data.kubernetes.container.id}"
                  prospector.scanner.symlinks: true
                  parsers:
                    - container: ~
                  paths:
                    - /var/log/containers/*-${data.kubernetes.container.id}.log
                  multiline:
                    pattern: '^\d{4}-\d{2}-\d{2}-\d{2}:\d{2}:\d{2}\.\d{3}'
                    negate: true
                    match: after
                  ignore_older: 24h
                  scan_frequency: 10s
                  clean_inactive: 25h
                  close_inactive: 5m
                  close_renamed: true
                  start_position: beginning
                  fields:
                    application: ${data.kubernetes.labels.app}
                    log_type: ${data.kubernetes.labels.log_type}
                    environment: ${data.kubernetes.labels.environment}
                    instance: ${data.kubernetes.host}

                  processors:
                    - add_kubernetes_metadata:
                          host: ${NODE_NAME}
                    - add_fields:
                        fields:
                          log_source: k8s
                        target: 'mylog'
                    - dissect:
                        tokenizer: "%{timestamp} [%{thread}] %{level} %{class} - [%{method},%{line}] - %{message}"
                        field: "message"
                        target_prefix: "mylog"
                        ignore_missing: true
                        overwrite_keys: true

            # ---------- java语言的email服务的Pod, java1.8项目自由文本格式日志, java21项目格式不太一样, 但也有堆栈信息----------




            # ---------- go语言的中转服务的Pod, go项目json格式日志 ----------
            - condition:
                # 匹配 sit 命名空间下的 lessie-go-api 应用
                and:
                  - equals:
                      kubernetes.namespace: "sit"
                  - equals:
                      kubernetes.labels.app: "lessie-go-api"
              config:
                - type: filestream
                  id: "k8s-go-json-log-${data.kubernetes.container.id}"
                  prospector.scanner.symlinks: true
                  parsers:
                    - container: ~
                  paths:
                    - /var/log/containers/*-${data.kubernetes.container.id}.log
                  fields:
                    application: ${data.kubernetes.labels.app}
                    log_type: "go.log" # 显式设置 log_type 字段，用于后续 processors 中的 'when' 条件
                    environment: ${data.kubernetes.labels.environment}
                    instance: ${data.kubernetes.host}

                  ignore_older: 24h
                  scan_frequency: 10s
                  clean_inactive: 25h
                  close_inactive: 5m
                  close_renamed: true
                  start_position: beginning

                  processors:
                    - add_kubernetes_metadata:
                          host: ${NODE_NAME}
                    - add_fields:
                        fields:
                          log_source: k8s
                        target: 'mylog'
                        
                    # 核心处理器：解析 JSON 格式日志
                    - decode_json_fields:
                        # 仅在 log_type 字段等于 go.log 时执行解析
                        when:
                          equals:
                            log_type: go.log
                        fields: ["message"]
                        target: ""
                        overwrite_keys: true
                        add_error_key: true
                        

            # ---------- python语言的lessie-agent的Pod, python项目只有文本格式日志, 需排除掉一些不采集的日志 ----------
            - condition:
                # 匹配 sit 命名空间下的 lessie-agent 应用
                and:
                  - equals:
                      kubernetes.namespace: "sit"
                  - equals:
                      kubernetes.labels.app: "lessie-agent"
              config:
                - type: filestream
                  id: "k8s-python-log-${data.kubernetes.container.id}"
                  prospector.scanner.symlinks: true
                  parsers:
                    - container: ~
                  paths:
                    - /var/log/containers/*-${data.kubernetes.container.id}.log
                  
                  # 核心采集配置：只包含以时间戳开头的行
                  include_lines: ['^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}'] 

                  ignore_older: 24h
                  scan_frequency: 10s
                  clean_inactive: 25h
                  close_inactive: 5m
                  close_renamed: true
                  start_position: beginning

                  fields:
                    application: ${data.kubernetes.labels.app} # lessie-agent
                    log_type: "lessie_search.log" # 保持与处理器 when 条件一致
                    environment: ${data.kubernetes.labels.environment}
                    instance: ${data.kubernetes.host}

                  processors:
                    - add_kubernetes_metadata:
                          host: ${NODE_NAME}
                    - add_fields:
                        fields:
                          log_source: k8s
                        target: 'mylog'
                        
                    # --- 处理器部分：移植您非 K8s 环境的逻辑 ---
                    
                    # 1. 基础 Dissect 解析
                    - dissect:
                        when:
                          equals:
                            log_type: lessie_search.log
                        tokenizer: '%{timestamp} - %{level} - %{module} - %{function} - %{message}'
                        field: "message"
                        target_prefix: "mylog"
                        ignore_missing: true
                        overwrite_keys: true

                    # 2. 针对带有 [level: | event: | msg: | context:] 的日志，再做一次 dissect
                    - dissect:
                        when:
                          regexp:
                            mylog.message: '^\[level:.*\]'
                        tokenizer: '[level: %{event_level} | event: %{event} | msg: %{msg} | context: %{context}]'
                        field: "mylog.message"
                        target_prefix: "mylog"
                        ignore_missing: true
                        overwrite_keys: true
                        
                    # 3. 把 context 再拆成独立字段 (JavaScript 脚本处理器)
                    - script:
                        lang: javascript
                        id: parse_context
                        source: >
                          function process(event) {
                            var ctx = event.Get("mylog.context");
                            if (ctx) {
                              var parts = ctx.split(",");
                              parts.forEach(function(p) {
                                var kv = p.split(":");
                                if (kv.length == 2) {
                                  // 确保 kv[0] 是有效的字段名
                                  event.Put("mylog." + kv[0].trim(), kv[1].trim());
                                }
                              });
                            }
                          }

            # ---------- python语言的apex的Pod, python项目json格式日志 ----------


            # ---------- 前端存储静态资源的nginx pod, nginx 格式日志 ----------


    # ---- 输出到 Elasticsearch ----
    output.elasticsearch:
      hosts: ["http://10.0.0.38:9200"]
      username: "admin"
      password: "G7ZSKFM4AQwHQpwA"

      # 动态索引命名：k8s-环境-应用-日期
      index: "k8s-%{[kubernetes.labels.environment]}-%{[kubernetes.labels.app]}-%{+yyyy.MM.dd}"


    logging.level: debug
    logging.selectors: ["*"]