更改filebast采集配置文件

SCM/构建镜像/v3/build_image_apex_v3.groovy

@@ -43,12 +43,12 @@ pipeline {
         booleanParam(
             name: 'DEPLOY_TO_TEST',
             defaultValue: false,
-            description: '构建成功后自动部署到 test 环境 (触发 job: DM_test_apex)'
+            description: '可选：构建成功后自动部署到 test 环境 (触发 job: DM_test_apex)'
         )
         booleanParam(
             name: 'DEPLOY_TO_PROD',
             defaultValue: false,
-            description: '构建成功后自动部署到 prod 环境 (触发 job: DM_prod_apex)'
+            description: '可选：构建成功后自动部署到 prod 环境 (触发 job: DM_prod_apex)'
         )
     }
     environment {

SCM/构建镜像/v3/build_image_apex_web_v3.groovy

@@ -43,12 +43,12 @@ pipeline {
         booleanParam(
             name: 'DEPLOY_TO_TEST',
             defaultValue: false,
-            description: '构建成功后自动部署到 test 环境 (触发 job: DM_test_apex_web)'
+            description: '可选：构建成功后自动部署到 test 环境 (触发 job: DM_test_apex_web)'
         )
         booleanParam(
             name: 'DEPLOY_TO_PROD',
             defaultValue: false,
-            description: '构建成功后自动部署到 prod 环境 (触发 job: DM_prod_apex_web)'
+            description: '可选：构建成功后自动部署到 prod 环境 (触发 job: DM_prod_apex_web)'
         )
     }
     environment {

k8s_yaml/ELK/filebast/02-filebeat-configmap.yaml

@@ -87,11 +87,70 @@ data:
 
             # ---------- ↑ java语言的中转服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↑ ----------
 
-            # ---------- ↓ python语言的中转服务的Pod, lessie agent 项目自由文本格式日志 ↓ ----------
+            # ---------- ↓ python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↓ ----------
+            - condition:
+                and:
+                  - equals:
+                      kubernetes.namespace: sit
+                  - equals:
+                      kubernetes.labels.app: "lessie-agents"
+              config:
+                - type: filestream
+                  id: "container-${data.kubernetes.container.id}"
+                  prospector.scanner.symlinks: true
+                  close.on_state_change.removed: false
+                  parsers:
+                    - container: ~
+                  paths:
+                    - /var/log/containers/*-${data.kubernetes.container.id}.log
 
+                  processors:
+                    - add_kubernetes_metadata:
+                        host: ${NODE_NAME}
+                    # 第一层：仅解析符合时间戳开头的日志行(for业务告警的日志格式)
+                    - dissect:
+                        when:
+                          regexp:
+                            message: '^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}.*'
+                        tokenizer: '%{timestamp} - %{level} - %{module} - %{function} - %{msg_body}'
+                        field: "message"
+                        target_prefix: "mylog"
+                        ignore_missing: true
+                        overwrite_keys: true
+                    # 第二层：针对带有 [level: | event: | msg: | context:] 的日志，再做一次 dissect
+                    - dissect:
+                        when:
+                          contains:
+                            mylog.msg_body: "[level:"
+                        tokenizer: '[level: %{event_level} | event: %{event} | msg: %{msg} | context: %{ctx_raw}]'
+                        field: "mylog.msg_body"
+                        target_prefix: "mylog"
+                        ignore_missing: true
+                        overwrite_keys: true
+                    # 第三层：把 ctx_raw 再拆成独立字段
+                    - script:
+                        lang: javascript
+                        id: parse_context
+                        source: >
+                          function process(event) {
+                            var ctx = event.Get("mylog.ctx_raw");
+                            if (!ctx) return;
+                            var parts = ctx.trim().split(",");
+                            for (var i = 0; i < parts.length; i++) {
+                              var pair = parts[i].split(":");
+                              if (pair.length === 2) {
+                                event.Put("mylog." + pair[0].trim(), pair[1].trim());
+                              }
+                            }
+                          }
+                    # 第四层: 去除大量不需要的k8s元数据字段
+                    - drop_fields:
+                        fields: 
+                          - "kubernetes.node.labels"
+                          - "kubernetes.annotations"
+                        ignore_missing: true
 
-
+            # ---------- ↑ python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↑ ----------
-            # ---------- ↑ python语言的中转服务的Pod, lessie agent 项目自由文本格式日志 ↑ ----------
 
 
     # ---- 输出到 Elasticsearch ----

k8s_yaml/ELK/filebast/023-filebeat-configmap.yaml

@@ -0,0 +1,233 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: filebeat-config
+  namespace: kube-system
+data:
+  filebeat.yml: |
+    setup.ilm.enabled: false
+    setup.template.enabled: false
+
+    filebeat.autodiscover:
+      providers:
+        - type: kubernetes
+          templates:
+            # ---------- ↓ json格式日志 ↓ ----------
+            - condition:
+                and:
+                  - regexp:
+                      kubernetes.namespace: "^(sit|apex-evaluation)$"
+                  - regexp:
+                      kubernetes.labels.app: "^(lessie-go-api|apex)$"
+              config:
+                - type: filestream
+                  id: "container-${data.kubernetes.container.id}"
+                  prospector.scanner.symlinks: true
+                  close.on_state_change.removed: false
+                  parsers:
+                    - container: ~
+                  paths:
+                    - /var/log/containers/*-${data.kubernetes.container.id}.log
+                  processors:
+                    - add_kubernetes_metadata:
+                        host: ${NODE_NAME}
+                    - decode_json_fields:
+                        fields: ["message"]
+                        target: "mylog"
+                        overwrite_keys: true
+                        add_error_key: true
+                    - drop_fields:
+                        fields: 
+                          - "kubernetes.node.labels"
+                          - "kubernetes.namespace_labels.kubernetes_io/metadata_name"
+                        ignore_missing: true
+            # ---------- ↑ json格式日志 ↑ ----------
+            
+
+            # ---------- ↓ java语言的服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↓ ----------
+            - condition:
+                and:
+                  - equals:
+                      kubernetes.namespace: sit
+                  - or:
+                    - equals:
+                        kubernetes.labels.app: "flymoon-admin"
+                    - equals:
+                        kubernetes.labels.app: "flymoon-agent"
+                    - equals:
+                        kubernetes.labels.app: "flymoon-payment"
+              config:
+                - type: filestream
+                  id: "container-${data.kubernetes.container.id}"
+                  prospector.scanner.symlinks: true
+                  close.on_state_change.removed: false
+                  parsers:
+                    - container: ~
+                    - multiline:
+                        type: pattern
+                        pattern: '^\d{4}-\d{2}-\d{2}-\d{2}:\d{2}:\d{2}\.\d{3}'
+                        negate: true
+                        match: after
+                  paths:
+                    - /var/log/containers/*-${data.kubernetes.container.id}.log
+                  processors:
+                    - add_kubernetes_metadata:
+                        host: ${NODE_NAME}
+                    - dissect:
+                        tokenizer: '%{timestamp}  %{level}  %{pid}  ---  [%{thread}]  %{class}  :  [%{app_name->}]  %{message}'
+                        field: "message"
+                        target_prefix: "mylog"
+                        ignore_missing: true
+                        overwrite_keys: true
+                    - drop_fields:
+                        fields: ["kubernetes.node.labels", "kubernetes.annotations"]
+                        ignore_missing: true
+
+            # ---------- ↑ java语言的服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↑ ----------
+
+
+            # ---------- ↓ java语言的服务的Pod, email 项目自由文本格式日志 ↓ ----------
+            - condition:
+                and:
+                  - equals:
+                      kubernetes.namespace: sit
+                  - equals:
+                      kubernetes.labels.app: "flymoon-email"
+              config:
+                - type: filestream
+                  id: "container-${data.kubernetes.container.id}"
+                  prospector.scanner.symlinks: true
+                  close.on_state_change.removed: false
+                  parsers:
+                    - container: ~
+                    - multiline:
+                        type: pattern
+                        pattern: '^\d{4}-\d{2}-\d{2}'
+                        negate: true
+                        match: after
+                  paths:
+                    - /var/log/containers/*-${data.kubernetes.container.id}.log
+                  processors:
+                    - add_kubernetes_metadata:
+                        host: ${NODE_NAME}
+                    - dissect:
+                        tokenizer: '%{timestamp}  %{level} %{pid} --- [%{thread}] %{class} : %{message}'
+                        field: "message"
+                        target_prefix: "mylog"
+                        ignore_missing: true
+                        overwrite_keys: true
+                    - drop_fields:
+                        fields: ["kubernetes.node.labels", "kubernetes.annotations"]
+                        ignore_missing: true
+            # ---------- ↑ java语言的服务的Pod, email 项目自由文本格式日志 ↑ ----------
+
+
+            # ---------- ↓ python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↓ ----------
+            - condition:
+                and:
+                  - equals:
+                      kubernetes.namespace: sit
+                  - equals:
+                      kubernetes.labels.app: "lessie-agents"
+              config:
+                - type: filestream
+                  id: "container-${data.kubernetes.container.id}"
+                  prospector.scanner.symlinks: true
+                  close.on_state_change.removed: false
+                  parsers:
+                    - container: ~
+                  paths:
+                    - /var/log/containers/*-${data.kubernetes.container.id}.log
+                  processors:
+                    - add_kubernetes_metadata:
+                        host: ${NODE_NAME}
+                    # 第一层：仅解析符合时间戳开头的日志行(for业务告警的日志格式)
+                    - dissect:
+                        when:
+                          regexp:
+                            message: '^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}.*'
+                        tokenizer: '%{timestamp} - %{level} - %{module} - %{function} - %{msg_body}'
+                        field: "message"
+                        target_prefix: "mylog"
+                        ignore_missing: true
+                        overwrite_keys: true
+                    # 第二层：针对带有 [level: | event: | msg: | context:] 的日志，再做一次 dissect
+                    - dissect:
+                        when:
+                          contains:
+                            mylog.msg_body: "[level:"
+                        tokenizer: '[level: %{event_level} | event: %{event} | msg: %{msg} | context: %{ctx_raw}]'
+                        field: "mylog.msg_body"
+                        target_prefix: "mylog"
+                        ignore_missing: true
+                        overwrite_keys: true
+                    # 第三层：把 ctx_raw 再拆成独立字段
+                    - script:
+                        lang: javascript
+                        id: parse_context
+                        source: >
+                          function process(event) {
+                            var ctx = event.Get("mylog.ctx_raw");
+                            if (!ctx) return;
+                            var parts = ctx.trim().split(",");
+                            for (var i = 0; i < parts.length; i++) {
+                              var pair = parts[i].split(":");
+                              if (pair.length === 2) {
+                                event.Put("mylog." + pair[0].trim(), pair[1].trim());
+                              }
+                            }
+                          }
+                    # 第四层: 去除大量不需要的k8s元数据字段
+                    - drop_fields:
+                        fields: 
+                          - "kubernetes.node.labels"
+                          - "kubernetes.annotations"
+                        ignore_missing: true
+            # ---------- ↑ python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↑ ----------
+
+
+            # ---------- ↓ apex 动态创建的 python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↓ ----------
+            - condition:
+                and:
+                  - equals:
+                      kubernetes.namespace: apex-evaluation
+                  - equals:
+                      kubernetes.labels.apex: "lessie-agents"
+              config:
+                - type: filestream
+                  id: "container-${data.kubernetes.container.id}"
+                  prospector.scanner.symlinks: true
+                  close.on_state_change.removed: false
+                  parsers:
+                    - container: ~
+                  paths:
+                    - /var/log/containers/*-${data.kubernetes.container.id}.log
+                  processors:
+                    - drop_fields:
+                        fields: 
+                          - "kubernetes.node.labels"
+                          - "kubernetes.annotations"
+                        ignore_missing: true
+            # ---------- ↑ apex 动态创建的 python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↑ ----------
+
+
+
+    # ---- 输出到 Elasticsearch ----
+    output.elasticsearch:
+      hosts: ["http://10.0.0.38:9200"]
+      username: "admin"
+      password: "G7ZSKFM4AQwHQpwA"
+
+      indices:
+        - index: "k8s-%{[kubernetes.labels.environment]}-%{[kubernetes.labels.app]}-%{+yyyy.MM.dd}"
+          when:
+            regexp:
+              kubernetes.labels.app: "(lessie-go-api|flymoon-admin|flymoon-agent|flymoon-payment|flymoon-email|lessie-agents|apex)"
+
+        - index: "apex-python-%{[kubernetes.pod.name]}"
+          when:
+            equals:
+              kubernetes.labels.apex: "lessie-agents"
+
+    logging.level: info
+    logging.selectors:  ["*"]