diff --git a/k8s_yaml/ELK/filebast/02-filebeat-configmap.yaml b/k8s_yaml/ELK/filebast/02-filebeat-configmap.yaml index 7943fa3..65da690 100644 --- a/k8s_yaml/ELK/filebast/02-filebeat-configmap.yaml +++ b/k8s_yaml/ELK/filebast/02-filebeat-configmap.yaml @@ -11,192 +11,87 @@ data: filebeat.autodiscover: providers: - type: kubernetes - node: ${NODE_NAME} - hints.enabled: false - templates: - # ---------- Template 1: java语言的admin、agent、payment Pod, java21项目多行堆栈文本日志 ---------- - # - condition: - # # 匹配 sit 命名空间下的 3个 flymoon 应用 - # and: - # - equals: - # kubernetes.namespace: "sit" - # - regexp: - # kubernetes.labels.app: "(flymoon-admin|flymoon-agent|flymoon-payment)" - # config: - # - type: filestream - # id: "k8s-java-log-${data.kubernetes.container.id}" - # prospector.scanner.symlinks: true - # parsers: - # - container: ~ - # paths: - # - /var/log/containers/*-${data.kubernetes.container.id}.log - # multiline: - # pattern: '^\d{4}-\d{2}-\d{2}-\d{2}:\d{2}:\d{2}\.\d{3}' - # negate: true - # match: after - # ignore_older: 24h - # scan_frequency: 10s - # clean_inactive: 25h - # close_inactive: 5m - # close_renamed: true - # start_position: beginning - # fields: - # application: ${data.kubernetes.labels.app} - # log_type: ${data.kubernetes.labels.log_type} - # environment: ${data.kubernetes.labels.environment} - # instance: ${data.kubernetes.host} - - # processors: - # - add_kubernetes_metadata: - # host: ${NODE_NAME} - # - add_fields: - # fields: - # log_source: k8s - # target: 'mylog' - # - dissect: - # tokenizer: "%{timestamp} [%{thread}] %{level} %{class} - [%{method},%{line}] - %{message}" - # field: "message" - # target_prefix: "mylog" - # ignore_missing: true - # overwrite_keys: true - - # ---------- java语言的email服务的Pod, java1.8项目自由文本格式日志, java21项目格式不太一样, 但也有堆栈信息---------- - - - - - # ---------- go语言的中转服务的Pod, go项目json格式日志 ---------- + # ---------- ↓ go语言的中转服务的Pod, go项目json格式日志 ↓ ---------- - condition: - # 匹配 sit 命名空间下的 lessie-go-api 应用 and: - equals: - kubernetes.namespace: "sit" + kubernetes.namespace: sit - equals: kubernetes.labels.app: "lessie-go-api" config: - type: filestream - id: "k8s-go-json-log-${data.kubernetes.container.id}" + id: "container-${data.kubernetes.container.id}" prospector.scanner.symlinks: true + close.on_state_change.removed: false parsers: - container: ~ paths: - /var/log/containers/*-${data.kubernetes.container.id}.log - fields: - application: ${data.kubernetes.labels.app} - log_type: "go.log" - environment: ${data.kubernetes.labels.environment} - instance: ${data.kubernetes.host} - - ignore_older: 24h - scan_frequency: 10s - clean_inactive: 25h - close_inactive: 5m - close_renamed: true - start_position: beginning processors: - add_kubernetes_metadata: - host: ${NODE_NAME} - - add_fields: - fields: - log_source: k8s - target: 'mylog' - - # 核心处理器：解析 JSON 格式日志 + host: ${NODE_NAME} - decode_json_fields: fields: ["message"] - target: "" + target: "mylog" overwrite_keys: true add_error_key: true - + - drop_fields: + fields: + - "kubernetes.node.labels" + - "kubernetes.namespace_labels.kubernetes_io/metadata_name" + ignore_missing: true - # ---------- python语言的lessie-agent的Pod, python项目只有文本格式日志, 需排除掉一些不采集的日志 ---------- - # - condition: - # # 匹配 sit 命名空间下的 lessie-agent 应用 - # and: - # - equals: - # kubernetes.namespace: "sit" - # - equals: - # kubernetes.labels.app: "lessie-agent" - # config: - # - type: filestream - # id: "k8s-python-log-${data.kubernetes.container.id}" - # prospector.scanner.symlinks: true - # parsers: - # - container: ~ - # paths: - # - /var/log/containers/*-${data.kubernetes.container.id}.log - - # # 核心采集配置：只包含以时间戳开头的行 - # include_lines: ['^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}'] + # ---------- ↑ go语言的中转服务的Pod, go项目json格式日志 ↑ ---------- + + # ---------- ↓ java语言的中转服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↓ ---------- + - condition: + and: + - equals: + kubernetes.namespace: sit + - or: + - equals: + kubernetes.labels.app: "flymoon-admin" + - equals: + kubernetes.labels.app: "flymoon-agent" + - equals: + kubernetes.labels.app: "flymoon-payment" + config: + - type: filestream + id: "container-${data.kubernetes.container.id}" + prospector.scanner.symlinks: true + close.on_state_change.removed: false + parsers: + - container: ~ + - multiline: + type: pattern + pattern: '^\d{4}-\d{2}-\d{2}-\d{2}:\d{2}:\d{2}\.\d{3}' + negate: true + match: after + paths: + - /var/log/containers/*-${data.kubernetes.container.id}.log - # ignore_older: 24h - # scan_frequency: 10s - # clean_inactive: 25h - # close_inactive: 5m - # close_renamed: true - # start_position: beginning + processors: + - add_kubernetes_metadata: + host: ${NODE_NAME} + - dissect: + tokenizer: '%{timestamp} %{level} %{pid} --- [%{thread}] %{class} : [%{app_name->}] %{message}' + field: "message" + target_prefix: "mylog" + ignore_missing: true + overwrite_keys: true + - drop_fields: + fields: ["kubernetes.node.labels", "kubernetes.annotations"] + ignore_missing: true - # fields: - # application: ${data.kubernetes.labels.app} # lessie-agent - # log_type: "lessie_search.log" # 保持与处理器 when 条件一致 - # environment: ${data.kubernetes.labels.environment} - # instance: ${data.kubernetes.host} + # ---------- ↑ java语言的中转服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↑ ---------- - # processors: - # - add_kubernetes_metadata: - # host: ${NODE_NAME} - # - add_fields: - # fields: - # log_source: k8s - # target: 'mylog' - - # # 1. 基础 Dissect 解析 - # - dissect: - # when: - # equals: - # log_type: lessie_search.log - # tokenizer: '%{timestamp} - %{level} - %{module} - %{function} - %{message}' - # field: "message" - # target_prefix: "mylog" - # ignore_missing: true - # overwrite_keys: true - - # # 2. 针对带有 [level: | event: | msg: | context:] 的日志，再做一次 dissect - # - dissect: - # when: - # regexp: - # mylog.message: '^\[level:.*\]' - # tokenizer: '[level: %{event_level} | event: %{event} | msg: %{msg} | context: %{context}]' - # field: "mylog.message" - # target_prefix: "mylog" - # ignore_missing: true - # overwrite_keys: true - - # # 3. 把 context 再拆成独立字段 (JavaScript 脚本处理器) - # - script: - # lang: javascript - # id: parse_context - # source: > - # function process(event) { - # var ctx = event.Get("mylog.context"); - # if (ctx) { - # var parts = ctx.split(","); - # parts.forEach(function(p) { - # var kv = p.split(":"); - # if (kv.length == 2) { - # // 确保 kv[0] 是有效的字段名 - # event.Put("mylog." + kv[0].trim(), kv[1].trim()); - # } - # }); - # } - # } - - # ---------- python语言的apex的Pod, python项目json格式日志 ---------- + # ---------- ↓ python语言的中转服务的Pod, lessie agent 项目自由文本格式日志 ↓ ---------- - # ---------- 前端存储静态资源的nginx pod, nginx 格式日志 ---------- + + # ---------- ↑ python语言的中转服务的Pod, lessie agent 项目自由文本格式日志 ↑ ---------- # ---- 输出到 Elasticsearch ---- @@ -204,10 +99,8 @@ data: hosts: ["http://10.0.0.38:9200"] username: "admin" password: "G7ZSKFM4AQwHQpwA" - - # 动态索引命名：k8s-环境-应用-日期 index: "k8s-%{[kubernetes.labels.environment]}-%{[kubernetes.labels.app]}-%{+yyyy.MM.dd}" - + # index: "k8s-%{[kubernetes.labels.app]}-%{+yyyy.MM.dd}" logging.level: debug - logging.selectors: ["*"] + logging.selectors: ["*"] \ No newline at end of file diff --git a/k8s_yaml/ELK/filebast/021-filebeat-configmap.yaml b/k8s_yaml/ELK/filebast/021-filebeat-configmap.yaml deleted file mode 100644 index d40e3d2..0000000 --- a/k8s_yaml/ELK/filebast/021-filebeat-configmap.yaml +++ /dev/null @@ -1,38 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: filebeat-config - namespace: kube-system -data: - filebeat.yml: | - setup.ilm.enabled: false - setup.template.enabled: false - - filebeat.autodiscover: - providers: - - type: kubernetes - node: ${NODE_NAME} - hints.enabled: true - hints.default_config: - type: filestream - id: container-${data.kubernetes.container.id} - prospector.scanner.symlinks: true - parsers: - - container: ~ - paths: - - /var/log/containers/*-${data.kubernetes.container.id}.log - - - # ---- 输出到 Elasticsearch ---- - output.elasticsearch: - hosts: ["http://10.0.0.38:9200"] - username: "admin" - password: "G7ZSKFM4AQwHQpwA" - - # 动态索引命名：k8s-环境-应用-日期 - index: "k8s-%{[kubernetes.labels.environment]}-%{[kubernetes.labels.app]}-%{+yyyy.MM.dd}" - - logging.level: debug - logging.selectors: ["*"] - - diff --git a/k8s_yaml/ELK/filebast/022-filebeat-configmap.yaml b/k8s_yaml/ELK/filebast/022-filebeat-configmap.yaml index 430cf76..0ce9be0 100644 --- a/k8s_yaml/ELK/filebast/022-filebeat-configmap.yaml +++ b/k8s_yaml/ELK/filebast/022-filebeat-configmap.yaml @@ -32,26 +32,125 @@ data: processors: - add_kubernetes_metadata: host: ${NODE_NAME} - include_fields: - - "kubernetes.node.hostname" - - "kubernetes.container.name" - - "kubernetes.pod.name" - - "kubernetes.pod.ip" - - "kubernetes.namespace" - - "kubernetes.labels.app" - - "kubernetes.labels.environment" - - "kubernetes.labels.project" - - decode_json_fields: fields: ["message"] target: "mylog" overwrite_keys: true add_error_key: true - - drop_fields: - fields: - - "" + fields: + - "kubernetes.node.labels" + - "kubernetes.namespace_labels.kubernetes_io/metadata_name" + ignore_missing: true + # ---------- ↑ go语言的中转服务的Pod, go项目json格式日志 ↑ ---------- + + # ---------- ↓ java语言的中转服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↓ ---------- + - condition: + and: + - equals: + kubernetes.namespace: sit + - or: + - equals: + kubernetes.labels.app: "flymoon-admin" + - equals: + kubernetes.labels.app: "flymoon-agent" + - equals: + kubernetes.labels.app: "flymoon-payment" + config: + - type: filestream + id: "container-${data.kubernetes.container.id}" + prospector.scanner.symlinks: true + close.on_state_change.removed: false + parsers: + - container: ~ + - multiline: + type: pattern + pattern: '^\d{4}-\d{2}-\d{2}-\d{2}:\d{2}:\d{2}\.\d{3}' + negate: true + match: after + paths: + - /var/log/containers/*-${data.kubernetes.container.id}.log + + processors: + - add_kubernetes_metadata: + host: ${NODE_NAME} + - dissect: + tokenizer: '%{timestamp} %{level} %{pid} --- [%{thread}] %{class} : [%{app_name->}] %{message}' + field: "message" + target_prefix: "mylog" + ignore_missing: true + overwrite_keys: true + - drop_fields: + fields: ["kubernetes.node.labels", "kubernetes.annotations"] + ignore_missing: true + + # ---------- ↑ java语言的中转服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↑ ---------- + + # ---------- ↓ python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↓ ---------- + - condition: + and: + - equals: + kubernetes.namespace: sit + - equals: + kubernetes.labels.app: "lessie-agents" + config: + - type: filestream + id: "container-${data.kubernetes.container.id}" + prospector.scanner.symlinks: true + close.on_state_change.removed: false + parsers: + - container: ~ + paths: + - /var/log/containers/*-${data.kubernetes.container.id}.log + + processors: + - add_kubernetes_metadata: + host: ${NODE_NAME} + # 第一层：仅解析符合时间戳开头的日志行(for业务告警的日志格式) + - dissect: + when: + regexp: + message: '^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}.*' + tokenizer: '%{timestamp} - %{level} - %{module} - %{function} - %{msg_body}' + field: "message" + target_prefix: "mylog" + ignore_missing: true + overwrite_keys: true + # 第二层：针对带有 [level: | event: | msg: | context:] 的日志，再做一次 dissect + - dissect: + when: + contains: + mylog.msg_body: "[level:" + tokenizer: '[level: %{event_level} | event: %{event} | msg: %{msg} | context: %{ctx_raw}]' + field: "mylog.msg_body" + target_prefix: "mylog" + ignore_missing: true + overwrite_keys: true + # 第三层：把 ctx_raw 再拆成独立字段 + - script: + lang: javascript + id: parse_context + source: > + function process(event) { + var ctx = event.Get("mylog.ctx_raw"); + if (!ctx) return; + var parts = ctx.trim().split(","); + for (var i = 0; i < parts.length; i++) { + var pair = parts[i].split(":"); + if (pair.length === 2) { + event.Put("mylog." + pair[0].trim(), pair[1].trim()); + } + } + } + # 第四层: 去除大量不需要的k8s元数据字段 + - drop_fields: + fields: + - "kubernetes.node.labels" + - "kubernetes.annotations" + ignore_missing: true + + # ---------- ↑ python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↑ ---------- # ---- 输出到 Elasticsearch ---- diff --git a/k8s_yaml/ELK/filebast/03-filebeat-daemonset.yaml b/k8s_yaml/ELK/filebast/03-filebeat-daemonset.yaml index 492e2e6..ca1adc3 100644 --- a/k8s_yaml/ELK/filebast/03-filebeat-daemonset.yaml +++ b/k8s_yaml/ELK/filebast/03-filebeat-daemonset.yaml @@ -22,6 +22,8 @@ spec: args: - "-e" env: + - name: TZ + value: Asia/Shanghai - name: NODE_NAME valueFrom: fieldRef: