更新filebast的配置文件

k8s_yaml/ELK/filebast/02-filebeat-configmap.yaml

@@ -12,38 +12,35 @@ data:
       providers:
         - type: kubernetes
           node: ${NODE_NAME}
-          hints.enabled: false
+          hints.enabled: true
 
           templates:
             # ---------- Template 1: java语言的admin、agent、payment Pod, java21项目多行堆栈文本日志 ----------
             - condition:
-                equals:
+                # 匹配 sit 命名空间下的 3个 flymoon 应用
-                  kubernetes.namespace: "sit" # 假设你的业务 pod 在 sit 命名空间
+                and:
-                # or:
+                  - equals:
-                #   - equals:
+                      kubernetes.namespace: "sit"
-                #       kubernetes.labels.app: "flymoon-admin"
+                  - regexp: 
-                #   - equals:
+                      kubernetes.labels.app: "(flymoon-admin|flymoon-agent|flymoon-payment)"
-                #       kubernetes.labels.app: "flymoon-agent"
-                #   - equals:
-                #       kubernetes.labels.app: "flymoon-payment"
               config:
                 - type: filestream
-                  id: "k8s-log-${data.kubernetes.container.id}"
+                  id: "k8s-java-log-${data.kubernetes.container.id}"
                   prospector.scanner.symlinks: true
                   parsers:
                     - container: ~
                   paths:
                     - /var/log/containers/*-${data.kubernetes.container.id}.log
-                  # multiline:
+                  multiline:
-                  #   pattern: '^\d{4}-\d{2}-\d{2}-\d{2}:\d{2}:\d{2}\.\d{3}'
+                    pattern: '^\d{4}-\d{2}-\d{2}-\d{2}:\d{2}:\d{2}\.\d{3}'
-                  #   negate: true
+                    negate: true
-                  #   match: after
+                    match: after
-                  # ignore_older: 24h
+                  ignore_older: 24h
-                  # scan_frequency: 10s
+                  scan_frequency: 10s
-                  # clean_inactive: 25h
+                  clean_inactive: 25h
-                  # close_inactive: 5m
+                  close_inactive: 5m
-                  # close_renamed: true
+                  close_renamed: true
-                  # start_position: beginning
+                  start_position: beginning
                   fields:
                     application: ${data.kubernetes.labels.app}
                     log_type: ${data.kubernetes.labels.log_type}
@@ -67,11 +64,140 @@ data:
             # ---------- java语言的email服务的Pod, java1.8项目自由文本格式日志, java21项目格式不太一样, 但也有堆栈信息----------
 
 
-            # ---------- go语言的中转服务的Pod, go项目json格式日志 ----------
 
 
+            # ---------- go语言的中转服务的Pod, go项目json格式日志 ----------
+            - condition:
+                # 匹配 sit 命名空间下的 lessie-go-api 应用
+                and:
+                  - equals:
+                      kubernetes.namespace: "sit"
+                  - equals:
+                      kubernetes.labels.app: "lessie-go-api"
+              config:
+                - type: filestream
+                  id: "k8s-go-json-log-${data.kubernetes.container.id}"
+                  prospector.scanner.symlinks: true
+                  parsers:
+                    - container: ~
+                  paths:
+                    - /var/log/containers/*-${data.kubernetes.container.id}.log
+                  fields:
+                    application: ${data.kubernetes.labels.app}
+                    log_type: "go.log" # 显式设置 log_type 字段，用于后续 processors 中的 'when' 条件
+                    environment: ${data.kubernetes.labels.environment}
+                    instance: ${data.kubernetes.host}
+
+                  ignore_older: 24h
+                  scan_frequency: 10s
+                  clean_inactive: 25h
+                  close_inactive: 5m
+                  close_renamed: true
+                  start_position: beginning
+
+                  processors:
+                    - add_kubernetes_metadata:
+                          host: ${NODE_NAME}
+                    - add_fields:
+                        fields:
+                          log_source: k8s
+                        target: 'mylog'
+                        
+                    # 核心处理器：解析 JSON 格式日志
+                    - decode_json_fields:
+                        # 仅在 log_type 字段等于 go.log 时执行解析
+                        when:
+                          equals:
+                            log_type: go.log
+                        fields: ["message"]
+                        target: ""
+                        overwrite_keys: true
+                        add_error_key: true
+                        
+
             # ---------- python语言的lessie-agent的Pod, python项目只有文本格式日志, 需排除掉一些不采集的日志 ----------
+            - condition:
+                # 匹配 sit 命名空间下的 lessie-agent 应用
+                and:
+                  - equals:
+                      kubernetes.namespace: "sit"
+                  - equals:
+                      kubernetes.labels.app: "lessie-agent"
+              config:
+                - type: filestream
+                  id: "k8s-python-log-${data.kubernetes.container.id}"
+                  prospector.scanner.symlinks: true
+                  parsers:
+                    - container: ~
+                  paths:
+                    - /var/log/containers/*-${data.kubernetes.container.id}.log
+                  
+                  # 核心采集配置：只包含以时间戳开头的行
+                  include_lines: ['^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}'] 
 
+                  ignore_older: 24h
+                  scan_frequency: 10s
+                  clean_inactive: 25h
+                  close_inactive: 5m
+                  close_renamed: true
+                  start_position: beginning
+
+                  fields:
+                    application: ${data.kubernetes.labels.app} # lessie-agent
+                    log_type: "lessie_search.log" # 保持与处理器 when 条件一致
+                    environment: ${data.kubernetes.labels.environment}
+                    instance: ${data.kubernetes.host}
+
+                  processors:
+                    - add_kubernetes_metadata:
+                          host: ${NODE_NAME}
+                    - add_fields:
+                        fields:
+                          log_source: k8s
+                        target: 'mylog'
+                        
+                    # --- 处理器部分：移植您非 K8s 环境的逻辑 ---
+                    
+                    # 1. 基础 Dissect 解析
+                    - dissect:
+                        when:
+                          equals:
+                            log_type: lessie_search.log
+                        tokenizer: '%{timestamp} - %{level} - %{module} - %{function} - %{message}'
+                        field: "message"
+                        target_prefix: "mylog"
+                        ignore_missing: true
+                        overwrite_keys: true
+
+                    # 2. 针对带有 [level: | event: | msg: | context:] 的日志，再做一次 dissect
+                    - dissect:
+                        when:
+                          regexp:
+                            mylog.message: '^\[level:.*\]'
+                        tokenizer: '[level: %{event_level} | event: %{event} | msg: %{msg} | context: %{context}]'
+                        field: "mylog.message"
+                        target_prefix: "mylog"
+                        ignore_missing: true
+                        overwrite_keys: true
+                        
+                    # 3. 把 context 再拆成独立字段 (JavaScript 脚本处理器)
+                    - script:
+                        lang: javascript
+                        id: parse_context
+                        source: >
+                          function process(event) {
+                            var ctx = event.Get("mylog.context");
+                            if (ctx) {
+                              var parts = ctx.split(",");
+                              parts.forEach(function(p) {
+                                var kv = p.split(":");
+                                if (kv.length == 2) {
+                                  // 确保 kv[0] 是有效的字段名
+                                  event.Put("mylog." + kv[0].trim(), kv[1].trim());
+                                }
+                              });
+                            }
+                          }
 
             # ---------- python语言的apex的Pod, python项目json格式日志 ----------
 

k8s_yaml/ELK/filebast/021-filebeat-configmap.yaml

@@ -22,18 +22,6 @@ data:
             paths:
               - /var/log/containers/*-${data.kubernetes.container.id}.log
 
-          # templates:
-          #   - condition:
-          #       exists: ['kubernetes.pod.name'] 
-          #     config:
-          #       - type: container
-          #         id: "debug"
-          #         paths:
-          #           - /var/log/containers/*.log
-          #         # follow_symlinks: true
-          #         # parsers:
-          #         #   - container: ~
-
 
     # ---- 输出到 Elasticsearch ----
     output.elasticsearch:
@@ -48,6 +36,3 @@ data:
     logging.selectors: ["*"]
 
 
-
-
-