diff --git a/k8s_yaml/ELK/filebast/02-filebeat-configmap.yaml b/k8s_yaml/ELK/filebast/02-filebeat-configmap.yaml index e8c7bda..8a8f0fa 100644 --- a/k8s_yaml/ELK/filebast/02-filebeat-configmap.yaml +++ b/k8s_yaml/ELK/filebast/02-filebeat-configmap.yaml @@ -12,54 +12,54 @@ data: providers: - type: kubernetes node: ${NODE_NAME} - hints.enabled: true + hints.enabled: false templates: # ---------- Template 1: java语言的admin、agent、payment Pod, java21项目多行堆栈文本日志 ---------- - - condition: - # 匹配 sit 命名空间下的 3个 flymoon 应用 - and: - - equals: - kubernetes.namespace: "sit" - - regexp: - kubernetes.labels.app: "(flymoon-admin|flymoon-agent|flymoon-payment)" - config: - - type: filestream - id: "k8s-java-log-${data.kubernetes.container.id}" - prospector.scanner.symlinks: true - parsers: - - container: ~ - paths: - - /var/log/containers/*-${data.kubernetes.container.id}.log - multiline: - pattern: '^\d{4}-\d{2}-\d{2}-\d{2}:\d{2}:\d{2}\.\d{3}' - negate: true - match: after - ignore_older: 24h - scan_frequency: 10s - clean_inactive: 25h - close_inactive: 5m - close_renamed: true - start_position: beginning - fields: - application: ${data.kubernetes.labels.app} - log_type: ${data.kubernetes.labels.log_type} - environment: ${data.kubernetes.labels.environment} - instance: ${data.kubernetes.host} + # - condition: + # # 匹配 sit 命名空间下的 3个 flymoon 应用 + # and: + # - equals: + # kubernetes.namespace: "sit" + # - regexp: + # kubernetes.labels.app: "(flymoon-admin|flymoon-agent|flymoon-payment)" + # config: + # - type: filestream + # id: "k8s-java-log-${data.kubernetes.container.id}" + # prospector.scanner.symlinks: true + # parsers: + # - container: ~ + # paths: + # - /var/log/containers/*-${data.kubernetes.container.id}.log + # multiline: + # pattern: '^\d{4}-\d{2}-\d{2}-\d{2}:\d{2}:\d{2}\.\d{3}' + # negate: true + # match: after + # ignore_older: 24h + # scan_frequency: 10s + # clean_inactive: 25h + # close_inactive: 5m + # close_renamed: true + # start_position: beginning + # fields: + # application: ${data.kubernetes.labels.app} + # log_type: ${data.kubernetes.labels.log_type} + # environment: ${data.kubernetes.labels.environment} + # instance: ${data.kubernetes.host} - processors: - - add_kubernetes_metadata: - host: ${NODE_NAME} - - add_fields: - fields: - log_source: k8s - target: 'mylog' - - dissect: - tokenizer: "%{timestamp} [%{thread}] %{level} %{class} - [%{method},%{line}] - %{message}" - field: "message" - target_prefix: "mylog" - ignore_missing: true - overwrite_keys: true + # processors: + # - add_kubernetes_metadata: + # host: ${NODE_NAME} + # - add_fields: + # fields: + # log_source: k8s + # target: 'mylog' + # - dissect: + # tokenizer: "%{timestamp} [%{thread}] %{level} %{class} - [%{method},%{line}] - %{message}" + # field: "message" + # target_prefix: "mylog" + # ignore_missing: true + # overwrite_keys: true # ---------- java语言的email服务的Pod, java1.8项目自由文本格式日志, java21项目格式不太一样, 但也有堆栈信息---------- @@ -105,10 +105,6 @@ data: # 核心处理器：解析 JSON 格式日志 - decode_json_fields: - # 仅在 log_type 字段等于 go.log 时执行解析 - when: - equals: - log_type: go.log fields: ["message"] target: "" overwrite_keys: true @@ -116,88 +112,86 @@ data: # ---------- python语言的lessie-agent的Pod, python项目只有文本格式日志, 需排除掉一些不采集的日志 ---------- - - condition: - # 匹配 sit 命名空间下的 lessie-agent 应用 - and: - - equals: - kubernetes.namespace: "sit" - - equals: - kubernetes.labels.app: "lessie-agent" - config: - - type: filestream - id: "k8s-python-log-${data.kubernetes.container.id}" - prospector.scanner.symlinks: true - parsers: - - container: ~ - paths: - - /var/log/containers/*-${data.kubernetes.container.id}.log + # - condition: + # # 匹配 sit 命名空间下的 lessie-agent 应用 + # and: + # - equals: + # kubernetes.namespace: "sit" + # - equals: + # kubernetes.labels.app: "lessie-agent" + # config: + # - type: filestream + # id: "k8s-python-log-${data.kubernetes.container.id}" + # prospector.scanner.symlinks: true + # parsers: + # - container: ~ + # paths: + # - /var/log/containers/*-${data.kubernetes.container.id}.log - # 核心采集配置：只包含以时间戳开头的行 - include_lines: ['^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}'] + # # 核心采集配置：只包含以时间戳开头的行 + # include_lines: ['^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}'] - ignore_older: 24h - scan_frequency: 10s - clean_inactive: 25h - close_inactive: 5m - close_renamed: true - start_position: beginning + # ignore_older: 24h + # scan_frequency: 10s + # clean_inactive: 25h + # close_inactive: 5m + # close_renamed: true + # start_position: beginning - fields: - application: ${data.kubernetes.labels.app} # lessie-agent - log_type: "lessie_search.log" # 保持与处理器 when 条件一致 - environment: ${data.kubernetes.labels.environment} - instance: ${data.kubernetes.host} + # fields: + # application: ${data.kubernetes.labels.app} # lessie-agent + # log_type: "lessie_search.log" # 保持与处理器 when 条件一致 + # environment: ${data.kubernetes.labels.environment} + # instance: ${data.kubernetes.host} - processors: - - add_kubernetes_metadata: - host: ${NODE_NAME} - - add_fields: - fields: - log_source: k8s - target: 'mylog' - - # --- 处理器部分：移植您非 K8s 环境的逻辑 --- + # processors: + # - add_kubernetes_metadata: + # host: ${NODE_NAME} + # - add_fields: + # fields: + # log_source: k8s + # target: 'mylog' - # 1. 基础 Dissect 解析 - - dissect: - when: - equals: - log_type: lessie_search.log - tokenizer: '%{timestamp} - %{level} - %{module} - %{function} - %{message}' - field: "message" - target_prefix: "mylog" - ignore_missing: true - overwrite_keys: true + # # 1. 基础 Dissect 解析 + # - dissect: + # when: + # equals: + # log_type: lessie_search.log + # tokenizer: '%{timestamp} - %{level} - %{module} - %{function} - %{message}' + # field: "message" + # target_prefix: "mylog" + # ignore_missing: true + # overwrite_keys: true - # 2. 针对带有 [level: | event: | msg: | context:] 的日志，再做一次 dissect - - dissect: - when: - regexp: - mylog.message: '^\[level:.*\]' - tokenizer: '[level: %{event_level} | event: %{event} | msg: %{msg} | context: %{context}]' - field: "mylog.message" - target_prefix: "mylog" - ignore_missing: true - overwrite_keys: true + # # 2. 针对带有 [level: | event: | msg: | context:] 的日志，再做一次 dissect + # - dissect: + # when: + # regexp: + # mylog.message: '^\[level:.*\]' + # tokenizer: '[level: %{event_level} | event: %{event} | msg: %{msg} | context: %{context}]' + # field: "mylog.message" + # target_prefix: "mylog" + # ignore_missing: true + # overwrite_keys: true - # 3. 把 context 再拆成独立字段 (JavaScript 脚本处理器) - - script: - lang: javascript - id: parse_context - source: > - function process(event) { - var ctx = event.Get("mylog.context"); - if (ctx) { - var parts = ctx.split(","); - parts.forEach(function(p) { - var kv = p.split(":"); - if (kv.length == 2) { - // 确保 kv[0] 是有效的字段名 - event.Put("mylog." + kv[0].trim(), kv[1].trim()); - } - }); - } - } + # # 3. 把 context 再拆成独立字段 (JavaScript 脚本处理器) + # - script: + # lang: javascript + # id: parse_context + # source: > + # function process(event) { + # var ctx = event.Get("mylog.context"); + # if (ctx) { + # var parts = ctx.split(","); + # parts.forEach(function(p) { + # var kv = p.split(":"); + # if (kv.length == 2) { + # // 确保 kv[0] 是有效的字段名 + # event.Put("mylog." + kv[0].trim(), kv[1].trim()); + # } + # }); + # } + # } # ---------- python语言的apex的Pod, python项目json格式日志 ---------- diff --git a/k8s_yaml/ELK/filebast/022-filebeat-configmap.yaml b/k8s_yaml/ELK/filebast/022-filebeat-configmap.yaml new file mode 100644 index 0000000..404a8b9 --- /dev/null +++ b/k8s_yaml/ELK/filebast/022-filebeat-configmap.yaml @@ -0,0 +1,46 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: filebeat-config + namespace: kube-system +data: + filebeat.yml: | + setup.ilm.enabled: false + setup.template.enabled: false + + filebeat.autodiscover: + providers: + # 配置 Provider + - type: kubernetes + node: ${NODE_NAME} + hints.enabled: false + + templates: + # ---------- go语言的中转服务的Pod, go项目json格式日志 ---------- + - condition: + equals: + kubernetes.namespace: kube-system + config: + - type: filestream + id: "k8s-go-json-log-${data.kubernetes.container.id}" + prospector.scanner.symlinks: true + parsers: + - container: ~ + paths: + - /var/log/containers/*-${data.kubernetes.container.id}.log + fields: + application: ${data.kubernetes.labels.app} + log_type: "goho.log" + environment: ${data.kubernetes.labels.environment} + instance: ${data.kubernetes.host} + + + # ---- 输出到 Elasticsearch ---- + output.elasticsearch: + hosts: ["http://10.0.0.38:9200"] + username: "admin" + password: "G7ZSKFM4AQwHQpwA" + index: "k8s-%{[kubernetes.labels.environment]}-%{[kubernetes.labels.app]}-%{+yyyy.MM.dd}" + + logging.level: debug + logging.selectors: ["*"]