diff --git a/k8s_yaml/ELK/filebast/02-filebeat-configmap.yaml b/k8s_yaml/ELK/filebast/02-filebeat-configmap.yaml index 771e2f9..e8c7bda 100644 --- a/k8s_yaml/ELK/filebast/02-filebeat-configmap.yaml +++ b/k8s_yaml/ELK/filebast/02-filebeat-configmap.yaml @@ -12,38 +12,35 @@ data: providers: - type: kubernetes node: ${NODE_NAME} - hints.enabled: false + hints.enabled: true templates: # ---------- Template 1: java语言的admin、agent、payment Pod, java21项目多行堆栈文本日志 ---------- - condition: - equals: - kubernetes.namespace: "sit" # 假设你的业务 pod 在 sit 命名空间 - # or: - # - equals: - # kubernetes.labels.app: "flymoon-admin" - # - equals: - # kubernetes.labels.app: "flymoon-agent" - # - equals: - # kubernetes.labels.app: "flymoon-payment" + # 匹配 sit 命名空间下的 3个 flymoon 应用 + and: + - equals: + kubernetes.namespace: "sit" + - regexp: + kubernetes.labels.app: "(flymoon-admin|flymoon-agent|flymoon-payment)" config: - type: filestream - id: "k8s-log-${data.kubernetes.container.id}" + id: "k8s-java-log-${data.kubernetes.container.id}" prospector.scanner.symlinks: true parsers: - container: ~ paths: - /var/log/containers/*-${data.kubernetes.container.id}.log - # multiline: - # pattern: '^\d{4}-\d{2}-\d{2}-\d{2}:\d{2}:\d{2}\.\d{3}' - # negate: true - # match: after - # ignore_older: 24h - # scan_frequency: 10s - # clean_inactive: 25h - # close_inactive: 5m - # close_renamed: true - # start_position: beginning + multiline: + pattern: '^\d{4}-\d{2}-\d{2}-\d{2}:\d{2}:\d{2}\.\d{3}' + negate: true + match: after + ignore_older: 24h + scan_frequency: 10s + clean_inactive: 25h + close_inactive: 5m + close_renamed: true + start_position: beginning fields: application: ${data.kubernetes.labels.app} log_type: ${data.kubernetes.labels.log_type} @@ -67,11 +64,140 @@ data: # ---------- java语言的email服务的Pod, java1.8项目自由文本格式日志, java21项目格式不太一样, 但也有堆栈信息---------- - # ---------- go语言的中转服务的Pod, go项目json格式日志 ---------- + # ---------- go语言的中转服务的Pod, go项目json格式日志 ---------- + - condition: + # 匹配 sit 命名空间下的 lessie-go-api 应用 + and: + - equals: + kubernetes.namespace: "sit" + - equals: + kubernetes.labels.app: "lessie-go-api" + config: + - type: filestream + id: "k8s-go-json-log-${data.kubernetes.container.id}" + prospector.scanner.symlinks: true + parsers: + - container: ~ + paths: + - /var/log/containers/*-${data.kubernetes.container.id}.log + fields: + application: ${data.kubernetes.labels.app} + log_type: "go.log" # 显式设置 log_type 字段，用于后续 processors 中的 'when' 条件 + environment: ${data.kubernetes.labels.environment} + instance: ${data.kubernetes.host} + + ignore_older: 24h + scan_frequency: 10s + clean_inactive: 25h + close_inactive: 5m + close_renamed: true + start_position: beginning + + processors: + - add_kubernetes_metadata: + host: ${NODE_NAME} + - add_fields: + fields: + log_source: k8s + target: 'mylog' + + # 核心处理器：解析 JSON 格式日志 + - decode_json_fields: + # 仅在 log_type 字段等于 go.log 时执行解析 + when: + equals: + log_type: go.log + fields: ["message"] + target: "" + overwrite_keys: true + add_error_key: true + + # ---------- python语言的lessie-agent的Pod, python项目只有文本格式日志, 需排除掉一些不采集的日志 ---------- + - condition: + # 匹配 sit 命名空间下的 lessie-agent 应用 + and: + - equals: + kubernetes.namespace: "sit" + - equals: + kubernetes.labels.app: "lessie-agent" + config: + - type: filestream + id: "k8s-python-log-${data.kubernetes.container.id}" + prospector.scanner.symlinks: true + parsers: + - container: ~ + paths: + - /var/log/containers/*-${data.kubernetes.container.id}.log + + # 核心采集配置：只包含以时间戳开头的行 + include_lines: ['^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}'] + ignore_older: 24h + scan_frequency: 10s + clean_inactive: 25h + close_inactive: 5m + close_renamed: true + start_position: beginning + + fields: + application: ${data.kubernetes.labels.app} # lessie-agent + log_type: "lessie_search.log" # 保持与处理器 when 条件一致 + environment: ${data.kubernetes.labels.environment} + instance: ${data.kubernetes.host} + + processors: + - add_kubernetes_metadata: + host: ${NODE_NAME} + - add_fields: + fields: + log_source: k8s + target: 'mylog' + + # --- 处理器部分：移植您非 K8s 环境的逻辑 --- + + # 1. 基础 Dissect 解析 + - dissect: + when: + equals: + log_type: lessie_search.log + tokenizer: '%{timestamp} - %{level} - %{module} - %{function} - %{message}' + field: "message" + target_prefix: "mylog" + ignore_missing: true + overwrite_keys: true + + # 2. 针对带有 [level: | event: | msg: | context:] 的日志，再做一次 dissect + - dissect: + when: + regexp: + mylog.message: '^\[level:.*\]' + tokenizer: '[level: %{event_level} | event: %{event} | msg: %{msg} | context: %{context}]' + field: "mylog.message" + target_prefix: "mylog" + ignore_missing: true + overwrite_keys: true + + # 3. 把 context 再拆成独立字段 (JavaScript 脚本处理器) + - script: + lang: javascript + id: parse_context + source: > + function process(event) { + var ctx = event.Get("mylog.context"); + if (ctx) { + var parts = ctx.split(","); + parts.forEach(function(p) { + var kv = p.split(":"); + if (kv.length == 2) { + // 确保 kv[0] 是有效的字段名 + event.Put("mylog." + kv[0].trim(), kv[1].trim()); + } + }); + } + } # ---------- python语言的apex的Pod, python项目json格式日志 ---------- diff --git a/k8s_yaml/ELK/filebast/021-filebeat-configmap.yaml b/k8s_yaml/ELK/filebast/021-filebeat-configmap.yaml index 930338b..d40e3d2 100644 --- a/k8s_yaml/ELK/filebast/021-filebeat-configmap.yaml +++ b/k8s_yaml/ELK/filebast/021-filebeat-configmap.yaml @@ -22,18 +22,6 @@ data: paths: - /var/log/containers/*-${data.kubernetes.container.id}.log - # templates: - # - condition: - # exists: ['kubernetes.pod.name'] - # config: - # - type: container - # id: "debug" - # paths: - # - /var/log/containers/*.log - # # follow_symlinks: true - # # parsers: - # # - container: ~ - # ---- 输出到 Elasticsearch ---- output.elasticsearch: @@ -48,6 +36,3 @@ data: logging.selectors: ["*"] - - -