From dd374ca771e58e778ed9082edf49b7f20139c1ae Mon Sep 17 00:00:00 2001 From: dxin Date: Fri, 26 Dec 2025 14:19:36 +0800 Subject: [PATCH] =?UTF-8?q?=E6=9B=B4=E6=94=B9filebast=E9=87=87=E9=9B=86?= =?UTF-8?q?=E9=85=8D=E7=BD=AE=E6=96=87=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- SCM/构建镜像/v3/build_image_apex_v3.groovy | 4 +- .../v3/build_image_apex_web_v3.groovy | 4 +- .../ELK/filebast/02-filebeat-configmap.yaml | 65 ++++- .../ELK/filebast/023-filebeat-configmap.yaml | 233 ++++++++++++++++++ 4 files changed, 299 insertions(+), 7 deletions(-) create mode 100644 k8s_yaml/ELK/filebast/023-filebeat-configmap.yaml diff --git a/SCM/构建镜像/v3/build_image_apex_v3.groovy b/SCM/构建镜像/v3/build_image_apex_v3.groovy index c964bd8..961e878 100644 --- a/SCM/构建镜像/v3/build_image_apex_v3.groovy +++ b/SCM/构建镜像/v3/build_image_apex_v3.groovy @@ -43,12 +43,12 @@ pipeline { booleanParam( name: 'DEPLOY_TO_TEST', defaultValue: false, - description: '构建成功后自动部署到 test 环境 (触发 job: DM_test_apex)' + description: '可选：构建成功后自动部署到 test 环境 (触发 job: DM_test_apex)' ) booleanParam( name: 'DEPLOY_TO_PROD', defaultValue: false, - description: '构建成功后自动部署到 prod 环境 (触发 job: DM_prod_apex)' + description: '可选：构建成功后自动部署到 prod 环境 (触发 job: DM_prod_apex)' ) } environment { diff --git a/SCM/构建镜像/v3/build_image_apex_web_v3.groovy b/SCM/构建镜像/v3/build_image_apex_web_v3.groovy index 46ad00a..85afb22 100644 --- a/SCM/构建镜像/v3/build_image_apex_web_v3.groovy +++ b/SCM/构建镜像/v3/build_image_apex_web_v3.groovy @@ -43,12 +43,12 @@ pipeline { booleanParam( name: 'DEPLOY_TO_TEST', defaultValue: false, - description: '构建成功后自动部署到 test 环境 (触发 job: DM_test_apex_web)' + description: '可选：构建成功后自动部署到 test 环境 (触发 job: DM_test_apex_web)' ) booleanParam( name: 'DEPLOY_TO_PROD', defaultValue: false, - description: '构建成功后自动部署到 prod 环境 (触发 job: DM_prod_apex_web)' + description: '可选：构建成功后自动部署到 prod 环境 (触发 job: DM_prod_apex_web)' ) } environment { diff --git a/k8s_yaml/ELK/filebast/02-filebeat-configmap.yaml b/k8s_yaml/ELK/filebast/02-filebeat-configmap.yaml index 65da690..0ce9be0 100644 --- a/k8s_yaml/ELK/filebast/02-filebeat-configmap.yaml +++ b/k8s_yaml/ELK/filebast/02-filebeat-configmap.yaml @@ -87,11 +87,70 @@ data: # ---------- ↑ java语言的中转服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↑ ---------- - # ---------- ↓ python语言的中转服务的Pod, lessie agent 项目自由文本格式日志 ↓ ---------- + # ---------- ↓ python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↓ ---------- + - condition: + and: + - equals: + kubernetes.namespace: sit + - equals: + kubernetes.labels.app: "lessie-agents" + config: + - type: filestream + id: "container-${data.kubernetes.container.id}" + prospector.scanner.symlinks: true + close.on_state_change.removed: false + parsers: + - container: ~ + paths: + - /var/log/containers/*-${data.kubernetes.container.id}.log + processors: + - add_kubernetes_metadata: + host: ${NODE_NAME} + # 第一层：仅解析符合时间戳开头的日志行(for业务告警的日志格式) + - dissect: + when: + regexp: + message: '^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}.*' + tokenizer: '%{timestamp} - %{level} - %{module} - %{function} - %{msg_body}' + field: "message" + target_prefix: "mylog" + ignore_missing: true + overwrite_keys: true + # 第二层：针对带有 [level: | event: | msg: | context:] 的日志，再做一次 dissect + - dissect: + when: + contains: + mylog.msg_body: "[level:" + tokenizer: '[level: %{event_level} | event: %{event} | msg: %{msg} | context: %{ctx_raw}]' + field: "mylog.msg_body" + target_prefix: "mylog" + ignore_missing: true + overwrite_keys: true + # 第三层：把 ctx_raw 再拆成独立字段 + - script: + lang: javascript + id: parse_context + source: > + function process(event) { + var ctx = event.Get("mylog.ctx_raw"); + if (!ctx) return; + var parts = ctx.trim().split(","); + for (var i = 0; i < parts.length; i++) { + var pair = parts[i].split(":"); + if (pair.length === 2) { + event.Put("mylog." + pair[0].trim(), pair[1].trim()); + } + } + } + # 第四层: 去除大量不需要的k8s元数据字段 + - drop_fields: + fields: + - "kubernetes.node.labels" + - "kubernetes.annotations" + ignore_missing: true - - # ---------- ↑ python语言的中转服务的Pod, lessie agent 项目自由文本格式日志 ↑ ---------- + # ---------- ↑ python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↑ ---------- # ---- 输出到 Elasticsearch ---- diff --git a/k8s_yaml/ELK/filebast/023-filebeat-configmap.yaml b/k8s_yaml/ELK/filebast/023-filebeat-configmap.yaml new file mode 100644 index 0000000..e6eeb60 --- /dev/null +++ b/k8s_yaml/ELK/filebast/023-filebeat-configmap.yaml @@ -0,0 +1,233 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: filebeat-config + namespace: kube-system +data: + filebeat.yml: | + setup.ilm.enabled: false + setup.template.enabled: false + + filebeat.autodiscover: + providers: + - type: kubernetes + templates: + # ---------- ↓ json格式日志 ↓ ---------- + - condition: + and: + - regexp: + kubernetes.namespace: "^(sit|apex-evaluation)$" + - regexp: + kubernetes.labels.app: "^(lessie-go-api|apex)$" + config: + - type: filestream + id: "container-${data.kubernetes.container.id}" + prospector.scanner.symlinks: true + close.on_state_change.removed: false + parsers: + - container: ~ + paths: + - /var/log/containers/*-${data.kubernetes.container.id}.log + processors: + - add_kubernetes_metadata: + host: ${NODE_NAME} + - decode_json_fields: + fields: ["message"] + target: "mylog" + overwrite_keys: true + add_error_key: true + - drop_fields: + fields: + - "kubernetes.node.labels" + - "kubernetes.namespace_labels.kubernetes_io/metadata_name" + ignore_missing: true + # ---------- ↑ json格式日志 ↑ ---------- + + + # ---------- ↓ java语言的服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↓ ---------- + - condition: + and: + - equals: + kubernetes.namespace: sit + - or: + - equals: + kubernetes.labels.app: "flymoon-admin" + - equals: + kubernetes.labels.app: "flymoon-agent" + - equals: + kubernetes.labels.app: "flymoon-payment" + config: + - type: filestream + id: "container-${data.kubernetes.container.id}" + prospector.scanner.symlinks: true + close.on_state_change.removed: false + parsers: + - container: ~ + - multiline: + type: pattern + pattern: '^\d{4}-\d{2}-\d{2}-\d{2}:\d{2}:\d{2}\.\d{3}' + negate: true + match: after + paths: + - /var/log/containers/*-${data.kubernetes.container.id}.log + processors: + - add_kubernetes_metadata: + host: ${NODE_NAME} + - dissect: + tokenizer: '%{timestamp} %{level} %{pid} --- [%{thread}] %{class} : [%{app_name->}] %{message}' + field: "message" + target_prefix: "mylog" + ignore_missing: true + overwrite_keys: true + - drop_fields: + fields: ["kubernetes.node.labels", "kubernetes.annotations"] + ignore_missing: true + + # ---------- ↑ java语言的服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↑ ---------- + + + # ---------- ↓ java语言的服务的Pod, email 项目自由文本格式日志 ↓ ---------- + - condition: + and: + - equals: + kubernetes.namespace: sit + - equals: + kubernetes.labels.app: "flymoon-email" + config: + - type: filestream + id: "container-${data.kubernetes.container.id}" + prospector.scanner.symlinks: true + close.on_state_change.removed: false + parsers: + - container: ~ + - multiline: + type: pattern + pattern: '^\d{4}-\d{2}-\d{2}' + negate: true + match: after + paths: + - /var/log/containers/*-${data.kubernetes.container.id}.log + processors: + - add_kubernetes_metadata: + host: ${NODE_NAME} + - dissect: + tokenizer: '%{timestamp} %{level} %{pid} --- [%{thread}] %{class} : %{message}' + field: "message" + target_prefix: "mylog" + ignore_missing: true + overwrite_keys: true + - drop_fields: + fields: ["kubernetes.node.labels", "kubernetes.annotations"] + ignore_missing: true + # ---------- ↑ java语言的服务的Pod, email 项目自由文本格式日志 ↑ ---------- + + + # ---------- ↓ python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↓ ---------- + - condition: + and: + - equals: + kubernetes.namespace: sit + - equals: + kubernetes.labels.app: "lessie-agents" + config: + - type: filestream + id: "container-${data.kubernetes.container.id}" + prospector.scanner.symlinks: true + close.on_state_change.removed: false + parsers: + - container: ~ + paths: + - /var/log/containers/*-${data.kubernetes.container.id}.log + processors: + - add_kubernetes_metadata: + host: ${NODE_NAME} + # 第一层：仅解析符合时间戳开头的日志行(for业务告警的日志格式) + - dissect: + when: + regexp: + message: '^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}.*' + tokenizer: '%{timestamp} - %{level} - %{module} - %{function} - %{msg_body}' + field: "message" + target_prefix: "mylog" + ignore_missing: true + overwrite_keys: true + # 第二层：针对带有 [level: | event: | msg: | context:] 的日志，再做一次 dissect + - dissect: + when: + contains: + mylog.msg_body: "[level:" + tokenizer: '[level: %{event_level} | event: %{event} | msg: %{msg} | context: %{ctx_raw}]' + field: "mylog.msg_body" + target_prefix: "mylog" + ignore_missing: true + overwrite_keys: true + # 第三层：把 ctx_raw 再拆成独立字段 + - script: + lang: javascript + id: parse_context + source: > + function process(event) { + var ctx = event.Get("mylog.ctx_raw"); + if (!ctx) return; + var parts = ctx.trim().split(","); + for (var i = 0; i < parts.length; i++) { + var pair = parts[i].split(":"); + if (pair.length === 2) { + event.Put("mylog." + pair[0].trim(), pair[1].trim()); + } + } + } + # 第四层: 去除大量不需要的k8s元数据字段 + - drop_fields: + fields: + - "kubernetes.node.labels" + - "kubernetes.annotations" + ignore_missing: true + # ---------- ↑ python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↑ ---------- + + + # ---------- ↓ apex 动态创建的 python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↓ ---------- + - condition: + and: + - equals: + kubernetes.namespace: apex-evaluation + - equals: + kubernetes.labels.apex: "lessie-agents" + config: + - type: filestream + id: "container-${data.kubernetes.container.id}" + prospector.scanner.symlinks: true + close.on_state_change.removed: false + parsers: + - container: ~ + paths: + - /var/log/containers/*-${data.kubernetes.container.id}.log + processors: + - drop_fields: + fields: + - "kubernetes.node.labels" + - "kubernetes.annotations" + ignore_missing: true + # ---------- ↑ apex 动态创建的 python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↑ ---------- + + + + # ---- 输出到 Elasticsearch ---- + output.elasticsearch: + hosts: ["http://10.0.0.38:9200"] + username: "admin" + password: "G7ZSKFM4AQwHQpwA" + + indices: + - index: "k8s-%{[kubernetes.labels.environment]}-%{[kubernetes.labels.app]}-%{+yyyy.MM.dd}" + when: + regexp: + kubernetes.labels.app: "(lessie-go-api|flymoon-admin|flymoon-agent|flymoon-payment|flymoon-email|lessie-agents|apex)" + + - index: "apex-python-%{[kubernetes.pod.name]}" + when: + equals: + kubernetes.labels.apex: "lessie-agents" + + logging.level: info + logging.selectors: ["*"] \ No newline at end of file