更改filebast采集配置文件

k8s_yaml/ELK/filebast/02-filebeat-configmap.yaml

@@ -87,11 +87,70 @@ data:
 
             # ---------- ↑ java语言的中转服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↑ ----------
 
-            # ---------- ↓ python语言的中转服务的Pod, lessie agent 项目自由文本格式日志 ↓ ----------
+            # ---------- ↓ python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↓ ----------
+            - condition:
+                and:
+                  - equals:
+                      kubernetes.namespace: sit
+                  - equals:
+                      kubernetes.labels.app: "lessie-agents"
+              config:
+                - type: filestream
+                  id: "container-${data.kubernetes.container.id}"
+                  prospector.scanner.symlinks: true
+                  close.on_state_change.removed: false
+                  parsers:
+                    - container: ~
+                  paths:
+                    - /var/log/containers/*-${data.kubernetes.container.id}.log
 
+                  processors:
+                    - add_kubernetes_metadata:
+                        host: ${NODE_NAME}
+                    # 第一层：仅解析符合时间戳开头的日志行(for业务告警的日志格式)
+                    - dissect:
+                        when:
+                          regexp:
+                            message: '^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}.*'
+                        tokenizer: '%{timestamp} - %{level} - %{module} - %{function} - %{msg_body}'
+                        field: "message"
+                        target_prefix: "mylog"
+                        ignore_missing: true
+                        overwrite_keys: true
+                    # 第二层：针对带有 [level: | event: | msg: | context:] 的日志，再做一次 dissect
+                    - dissect:
+                        when:
+                          contains:
+                            mylog.msg_body: "[level:"
+                        tokenizer: '[level: %{event_level} | event: %{event} | msg: %{msg} | context: %{ctx_raw}]'
+                        field: "mylog.msg_body"
+                        target_prefix: "mylog"
+                        ignore_missing: true
+                        overwrite_keys: true
+                    # 第三层：把 ctx_raw 再拆成独立字段
+                    - script:
+                        lang: javascript
+                        id: parse_context
+                        source: >
+                          function process(event) {
+                            var ctx = event.Get("mylog.ctx_raw");
+                            if (!ctx) return;
+                            var parts = ctx.trim().split(",");
+                            for (var i = 0; i < parts.length; i++) {
+                              var pair = parts[i].split(":");
+                              if (pair.length === 2) {
+                                event.Put("mylog." + pair[0].trim(), pair[1].trim());
+                              }
+                            }
+                          }
+                    # 第四层: 去除大量不需要的k8s元数据字段
+                    - drop_fields:
+                        fields: 
+                          - "kubernetes.node.labels"
+                          - "kubernetes.annotations"
+                        ignore_missing: true
 
-
-            # ---------- ↑ python语言的中转服务的Pod, lessie agent 项目自由文本格式日志 ↑ ----------
+            # ---------- ↑ python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↑ ----------
 
 
     # ---- 输出到 Elasticsearch ----