更改采集日志模板

k8s_yaml/ELK/filebast/02-filebeat-configmap.yaml

@@ -11,192 +11,87 @@ data:
     filebeat.autodiscover:
       providers:
         - type: kubernetes
-          node: ${NODE_NAME}
-          hints.enabled: false
-
           templates:
-            # ---------- Template 1: java语言的admin、agent、payment Pod, java21项目多行堆栈文本日志 ----------
+            # ---------- ↓ go语言的中转服务的Pod, go项目json格式日志 ↓ ----------
-            # - condition:
-            #     # 匹配 sit 命名空间下的 3个 flymoon 应用
-            #     and:
-            #       - equals:
-            #           kubernetes.namespace: "sit"
-            #       - regexp: 
-            #           kubernetes.labels.app: "(flymoon-admin|flymoon-agent|flymoon-payment)"
-            #   config:
-            #     - type: filestream
-            #       id: "k8s-java-log-${data.kubernetes.container.id}"
-            #       prospector.scanner.symlinks: true
-            #       parsers:
-            #         - container: ~
-            #       paths:
-            #         - /var/log/containers/*-${data.kubernetes.container.id}.log
-            #       multiline:
-            #         pattern: '^\d{4}-\d{2}-\d{2}-\d{2}:\d{2}:\d{2}\.\d{3}'
-            #         negate: true
-            #         match: after
-            #       ignore_older: 24h
-            #       scan_frequency: 10s
-            #       clean_inactive: 25h
-            #       close_inactive: 5m
-            #       close_renamed: true
-            #       start_position: beginning
-            #       fields:
-            #         application: ${data.kubernetes.labels.app}
-            #         log_type: ${data.kubernetes.labels.log_type}
-            #         environment: ${data.kubernetes.labels.environment}
-            #         instance: ${data.kubernetes.host}
-
-            #       processors:
-            #         - add_kubernetes_metadata:
-            #               host: ${NODE_NAME}
-            #         - add_fields:
-            #             fields:
-            #               log_source: k8s
-            #             target: 'mylog'
-            #         - dissect:
-            #             tokenizer: "%{timestamp} [%{thread}] %{level} %{class} - [%{method},%{line}] - %{message}"
-            #             field: "message"
-            #             target_prefix: "mylog"
-            #             ignore_missing: true
-            #             overwrite_keys: true
-
-            # ---------- java语言的email服务的Pod, java1.8项目自由文本格式日志, java21项目格式不太一样, 但也有堆栈信息----------
-
-
-
-
-            # ---------- go语言的中转服务的Pod, go项目json格式日志 ----------
             - condition:
-                # 匹配 sit 命名空间下的 lessie-go-api 应用
                 and:
                   - equals:
-                      kubernetes.namespace: "sit"
+                      kubernetes.namespace: sit
                   - equals:
                       kubernetes.labels.app: "lessie-go-api"
               config:
                 - type: filestream
-                  id: "k8s-go-json-log-${data.kubernetes.container.id}"
+                  id: "container-${data.kubernetes.container.id}"
                   prospector.scanner.symlinks: true
+                  close.on_state_change.removed: false
                   parsers:
                     - container: ~
                   paths:
                     - /var/log/containers/*-${data.kubernetes.container.id}.log
-                  fields:
-                    application: ${data.kubernetes.labels.app}
-                    log_type: "go.log" 
-                    environment: ${data.kubernetes.labels.environment}
-                    instance: ${data.kubernetes.host}
-
-                  ignore_older: 24h
-                  scan_frequency: 10s
-                  clean_inactive: 25h
-                  close_inactive: 5m
-                  close_renamed: true
-                  start_position: beginning
 
                   processors:
                     - add_kubernetes_metadata:
-                          host: ${NODE_NAME}
+                        host: ${NODE_NAME}
-                    - add_fields:
-                        fields:
-                          log_source: k8s
-                        target: 'mylog'
-                        
-                    # 核心处理器：解析 JSON 格式日志
                     - decode_json_fields:
                         fields: ["message"]
-                        target: ""
+                        target: "mylog"
                         overwrite_keys: true
                         add_error_key: true
+                    - drop_fields:
+                        fields: 
+                          - "kubernetes.node.labels"
+                          - "kubernetes.namespace_labels.kubernetes_io/metadata_name"
+                        ignore_missing: true
+
+            # ---------- ↑ go语言的中转服务的Pod, go项目json格式日志 ↑ ----------
+            
+            # ---------- ↓ java语言的中转服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↓ ----------
+            - condition:
+                and:
+                  - equals:
+                      kubernetes.namespace: sit
+                  - or:
+                    - equals:
+                        kubernetes.labels.app: "flymoon-admin"
+                    - equals:
+                        kubernetes.labels.app: "flymoon-agent"
+                    - equals:
+                        kubernetes.labels.app: "flymoon-payment"
+              config:
+                - type: filestream
+                  id: "container-${data.kubernetes.container.id}"
+                  prospector.scanner.symlinks: true
+                  close.on_state_change.removed: false
+                  parsers:
+                    - container: ~
+                    - multiline:
+                        type: pattern
+                        pattern: '^\d{4}-\d{2}-\d{2}-\d{2}:\d{2}:\d{2}\.\d{3}'
+                        negate: true
+                        match: after
+                  paths:
+                    - /var/log/containers/*-${data.kubernetes.container.id}.log
+
+                  processors:
+                    - add_kubernetes_metadata:
+                        host: ${NODE_NAME}
+                    - dissect:
+                        tokenizer: '%{timestamp}  %{level}  %{pid}  ---  [%{thread}]  %{class}  :  [%{app_name->}]  %{message}'
+                        field: "message"
+                        target_prefix: "mylog"
+                        ignore_missing: true
+                        overwrite_keys: true
+                    - drop_fields:
+                        fields: ["kubernetes.node.labels", "kubernetes.annotations"]
+                        ignore_missing: true
+
+            # ---------- ↑ java语言的中转服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↑ ----------
+
+            # ---------- ↓ python语言的中转服务的Pod, lessie agent 项目自由文本格式日志 ↓ ----------
 
 
-            # ---------- python语言的lessie-agent的Pod, python项目只有文本格式日志, 需排除掉一些不采集的日志 ----------
-            # - condition:
-            #     # 匹配 sit 命名空间下的 lessie-agent 应用
-            #     and:
-            #       - equals:
-            #           kubernetes.namespace: "sit"
-            #       - equals:
-            #           kubernetes.labels.app: "lessie-agent"
-            #   config:
-            #     - type: filestream
-            #       id: "k8s-python-log-${data.kubernetes.container.id}"
-            #       prospector.scanner.symlinks: true
-            #       parsers:
-            #         - container: ~
-            #       paths:
-            #         - /var/log/containers/*-${data.kubernetes.container.id}.log
 
-            #       # 核心采集配置：只包含以时间戳开头的行
+            # ---------- ↑ python语言的中转服务的Pod, lessie agent 项目自由文本格式日志 ↑ ----------
-            #       include_lines: ['^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}'] 
-
-            #       ignore_older: 24h
-            #       scan_frequency: 10s
-            #       clean_inactive: 25h
-            #       close_inactive: 5m
-            #       close_renamed: true
-            #       start_position: beginning
-
-            #       fields:
-            #         application: ${data.kubernetes.labels.app} # lessie-agent
-            #         log_type: "lessie_search.log" # 保持与处理器 when 条件一致
-            #         environment: ${data.kubernetes.labels.environment}
-            #         instance: ${data.kubernetes.host}
-
-            #       processors:
-            #         - add_kubernetes_metadata:
-            #               host: ${NODE_NAME}
-            #         - add_fields:
-            #             fields:
-            #               log_source: k8s
-            #             target: 'mylog'
-                    
-            #         # 1. 基础 Dissect 解析
-            #         - dissect:
-            #             when:
-            #               equals:
-            #                 log_type: lessie_search.log
-            #             tokenizer: '%{timestamp} - %{level} - %{module} - %{function} - %{message}'
-            #             field: "message"
-            #             target_prefix: "mylog"
-            #             ignore_missing: true
-            #             overwrite_keys: true
-
-            #         # 2. 针对带有 [level: | event: | msg: | context:] 的日志，再做一次 dissect
-            #         - dissect:
-            #             when:
-            #               regexp:
-            #                 mylog.message: '^\[level:.*\]'
-            #             tokenizer: '[level: %{event_level} | event: %{event} | msg: %{msg} | context: %{context}]'
-            #             field: "mylog.message"
-            #             target_prefix: "mylog"
-            #             ignore_missing: true
-            #             overwrite_keys: true
-                        
-            #         # 3. 把 context 再拆成独立字段 (JavaScript 脚本处理器)
-            #         - script:
-            #             lang: javascript
-            #             id: parse_context
-            #             source: >
-            #               function process(event) {
-            #                 var ctx = event.Get("mylog.context");
-            #                 if (ctx) {
-            #                   var parts = ctx.split(",");
-            #                   parts.forEach(function(p) {
-            #                     var kv = p.split(":");
-            #                     if (kv.length == 2) {
-            #                       // 确保 kv[0] 是有效的字段名
-            #                       event.Put("mylog." + kv[0].trim(), kv[1].trim());
-            #                     }
-            #                   });
-            #                 }
-            #               }
-
-            # ---------- python语言的apex的Pod, python项目json格式日志 ----------
-
-
-            # ---------- 前端存储静态资源的nginx pod, nginx 格式日志 ----------
 
 
     # ---- 输出到 Elasticsearch ----
@@ -204,10 +99,8 @@ data:
       hosts: ["http://10.0.0.38:9200"]
       username: "admin"
       password: "G7ZSKFM4AQwHQpwA"
-
-      # 动态索引命名：k8s-环境-应用-日期
       index: "k8s-%{[kubernetes.labels.environment]}-%{[kubernetes.labels.app]}-%{+yyyy.MM.dd}"
-
+      # index: "k8s-%{[kubernetes.labels.app]}-%{+yyyy.MM.dd}"
 
     logging.level: debug
-    logging.selectors: ["*"]
+    logging.selectors:  ["*"]

k8s_yaml/ELK/filebast/021-filebeat-configmap.yaml

@@ -1,38 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: filebeat-config
-  namespace: kube-system
-data:
-  filebeat.yml: |
-    setup.ilm.enabled: false
-    setup.template.enabled: false
-
-    filebeat.autodiscover:
-      providers:
-        - type: kubernetes
-          node: ${NODE_NAME}
-          hints.enabled: true
-          hints.default_config:
-            type: filestream
-            id: container-${data.kubernetes.container.id}
-            prospector.scanner.symlinks: true
-            parsers:
-              - container: ~
-            paths:
-              - /var/log/containers/*-${data.kubernetes.container.id}.log
-
-
-    # ---- 输出到 Elasticsearch ----
-    output.elasticsearch:
-      hosts: ["http://10.0.0.38:9200"]
-      username: "admin"
-      password: "G7ZSKFM4AQwHQpwA"
-
-      # 动态索引命名：k8s-环境-应用-日期
-      index: "k8s-%{[kubernetes.labels.environment]}-%{[kubernetes.labels.app]}-%{+yyyy.MM.dd}"
-
-    logging.level: debug
-    logging.selectors: ["*"]
-
-

k8s_yaml/ELK/filebast/022-filebeat-configmap.yaml

@@ -32,26 +32,125 @@ data:
                   processors:
                     - add_kubernetes_metadata:
                         host: ${NODE_NAME}
-                        include_fields:
-                          - "kubernetes.node.hostname"
-                          - "kubernetes.container.name"
-                          - "kubernetes.pod.name"
-                          - "kubernetes.pod.ip"
-                          - "kubernetes.namespace"
-                          - "kubernetes.labels.app"
-                          - "kubernetes.labels.environment"
-                          - "kubernetes.labels.project"
-
                     - decode_json_fields:
                         fields: ["message"]
                         target: "mylog"
                         overwrite_keys: true
                         add_error_key: true
-
                     - drop_fields:
-                      fields:
+                        fields: 
-                        - ""
+                          - "kubernetes.node.labels"
+                          - "kubernetes.namespace_labels.kubernetes_io/metadata_name"
+                        ignore_missing: true
 
+            # ---------- ↑ go语言的中转服务的Pod, go项目json格式日志 ↑ ----------
+            
+            # ---------- ↓ java语言的中转服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↓ ----------
+            - condition:
+                and:
+                  - equals:
+                      kubernetes.namespace: sit
+                  - or:
+                    - equals:
+                        kubernetes.labels.app: "flymoon-admin"
+                    - equals:
+                        kubernetes.labels.app: "flymoon-agent"
+                    - equals:
+                        kubernetes.labels.app: "flymoon-payment"
+              config:
+                - type: filestream
+                  id: "container-${data.kubernetes.container.id}"
+                  prospector.scanner.symlinks: true
+                  close.on_state_change.removed: false
+                  parsers:
+                    - container: ~
+                    - multiline:
+                        type: pattern
+                        pattern: '^\d{4}-\d{2}-\d{2}-\d{2}:\d{2}:\d{2}\.\d{3}'
+                        negate: true
+                        match: after
+                  paths:
+                    - /var/log/containers/*-${data.kubernetes.container.id}.log
+
+                  processors:
+                    - add_kubernetes_metadata:
+                        host: ${NODE_NAME}
+                    - dissect:
+                        tokenizer: '%{timestamp}  %{level}  %{pid}  ---  [%{thread}]  %{class}  :  [%{app_name->}]  %{message}'
+                        field: "message"
+                        target_prefix: "mylog"
+                        ignore_missing: true
+                        overwrite_keys: true
+                    - drop_fields:
+                        fields: ["kubernetes.node.labels", "kubernetes.annotations"]
+                        ignore_missing: true
+
+            # ---------- ↑ java语言的中转服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↑ ----------
+
+            # ---------- ↓ python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↓ ----------
+            - condition:
+                and:
+                  - equals:
+                      kubernetes.namespace: sit
+                  - equals:
+                      kubernetes.labels.app: "lessie-agents"
+              config:
+                - type: filestream
+                  id: "container-${data.kubernetes.container.id}"
+                  prospector.scanner.symlinks: true
+                  close.on_state_change.removed: false
+                  parsers:
+                    - container: ~
+                  paths:
+                    - /var/log/containers/*-${data.kubernetes.container.id}.log
+
+                  processors:
+                    - add_kubernetes_metadata:
+                        host: ${NODE_NAME}
+                    # 第一层：仅解析符合时间戳开头的日志行(for业务告警的日志格式)
+                    - dissect:
+                        when:
+                          regexp:
+                            message: '^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}.*'
+                        tokenizer: '%{timestamp} - %{level} - %{module} - %{function} - %{msg_body}'
+                        field: "message"
+                        target_prefix: "mylog"
+                        ignore_missing: true
+                        overwrite_keys: true
+                    # 第二层：针对带有 [level: | event: | msg: | context:] 的日志，再做一次 dissect
+                    - dissect:
+                        when:
+                          contains:
+                            mylog.msg_body: "[level:"
+                        tokenizer: '[level: %{event_level} | event: %{event} | msg: %{msg} | context: %{ctx_raw}]'
+                        field: "mylog.msg_body"
+                        target_prefix: "mylog"
+                        ignore_missing: true
+                        overwrite_keys: true
+                    # 第三层：把 ctx_raw 再拆成独立字段
+                    - script:
+                        lang: javascript
+                        id: parse_context
+                        source: >
+                          function process(event) {
+                            var ctx = event.Get("mylog.ctx_raw");
+                            if (!ctx) return;
+                            var parts = ctx.trim().split(",");
+                            for (var i = 0; i < parts.length; i++) {
+                              var pair = parts[i].split(":");
+                              if (pair.length === 2) {
+                                event.Put("mylog." + pair[0].trim(), pair[1].trim());
+                              }
+                            }
+                          }
+                    # 第四层: 去除大量不需要的k8s元数据字段
+                    - drop_fields:
+                        fields: 
+                          - "kubernetes.node.labels"
+                          - "kubernetes.annotations"
+                        ignore_missing: true
+
+            # ---------- ↑ python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↑ ----------
 
 
     # ---- 输出到 Elasticsearch ----

k8s_yaml/ELK/filebast/03-filebeat-daemonset.yaml

@@ -22,6 +22,8 @@ spec:
         args:
           - "-e"
         env:
+          - name: TZ
+            value: Asia/Shanghai
           - name: NODE_NAME
             valueFrom:
               fieldRef: