220 lines
8.8 KiB
YAML
220 lines
8.8 KiB
YAML
apiVersion: v1
|
||
kind: ConfigMap
|
||
metadata:
|
||
name: filebeat-config
|
||
namespace: kube-system
|
||
data:
|
||
filebeat.yml: |
|
||
setup.ilm.enabled: false
|
||
setup.template.enabled: false
|
||
|
||
filebeat.autodiscover:
|
||
providers:
|
||
- type: kubernetes
|
||
node: ${NODE_NAME}
|
||
hints.enabled: true
|
||
|
||
templates:
|
||
# ---------- Template 1: java语言的admin、agent、payment Pod, java21项目多行堆栈文本日志 ----------
|
||
- condition:
|
||
# 匹配 sit 命名空间下的 3个 flymoon 应用
|
||
and:
|
||
- equals:
|
||
kubernetes.namespace: "sit"
|
||
- regexp:
|
||
kubernetes.labels.app: "(flymoon-admin|flymoon-agent|flymoon-payment)"
|
||
config:
|
||
- type: filestream
|
||
id: "k8s-java-log-${data.kubernetes.container.id}"
|
||
prospector.scanner.symlinks: true
|
||
parsers:
|
||
- container: ~
|
||
paths:
|
||
- /var/log/containers/*-${data.kubernetes.container.id}.log
|
||
multiline:
|
||
pattern: '^\d{4}-\d{2}-\d{2}-\d{2}:\d{2}:\d{2}\.\d{3}'
|
||
negate: true
|
||
match: after
|
||
ignore_older: 24h
|
||
scan_frequency: 10s
|
||
clean_inactive: 25h
|
||
close_inactive: 5m
|
||
close_renamed: true
|
||
start_position: beginning
|
||
fields:
|
||
application: ${data.kubernetes.labels.app}
|
||
log_type: ${data.kubernetes.labels.log_type}
|
||
environment: ${data.kubernetes.labels.environment}
|
||
instance: ${data.kubernetes.host}
|
||
|
||
processors:
|
||
- add_kubernetes_metadata:
|
||
host: ${NODE_NAME}
|
||
- add_fields:
|
||
fields:
|
||
log_source: k8s
|
||
target: 'mylog'
|
||
- dissect:
|
||
tokenizer: "%{timestamp} [%{thread}] %{level} %{class} - [%{method},%{line}] - %{message}"
|
||
field: "message"
|
||
target_prefix: "mylog"
|
||
ignore_missing: true
|
||
overwrite_keys: true
|
||
|
||
# ---------- java语言的email服务的Pod, java1.8项目自由文本格式日志, java21项目格式不太一样, 但也有堆栈信息----------
|
||
|
||
|
||
|
||
|
||
# ---------- go语言的中转服务的Pod, go项目json格式日志 ----------
|
||
- condition:
|
||
# 匹配 sit 命名空间下的 lessie-go-api 应用
|
||
and:
|
||
- equals:
|
||
kubernetes.namespace: "sit"
|
||
- equals:
|
||
kubernetes.labels.app: "lessie-go-api"
|
||
config:
|
||
- type: filestream
|
||
id: "k8s-go-json-log-${data.kubernetes.container.id}"
|
||
prospector.scanner.symlinks: true
|
||
parsers:
|
||
- container: ~
|
||
paths:
|
||
- /var/log/containers/*-${data.kubernetes.container.id}.log
|
||
fields:
|
||
application: ${data.kubernetes.labels.app}
|
||
log_type: "go.log" # 显式设置 log_type 字段，用于后续 processors 中的 'when' 条件
|
||
environment: ${data.kubernetes.labels.environment}
|
||
instance: ${data.kubernetes.host}
|
||
|
||
ignore_older: 24h
|
||
scan_frequency: 10s
|
||
clean_inactive: 25h
|
||
close_inactive: 5m
|
||
close_renamed: true
|
||
start_position: beginning
|
||
|
||
processors:
|
||
- add_kubernetes_metadata:
|
||
host: ${NODE_NAME}
|
||
- add_fields:
|
||
fields:
|
||
log_source: k8s
|
||
target: 'mylog'
|
||
|
||
# 核心处理器：解析 JSON 格式日志
|
||
- decode_json_fields:
|
||
# 仅在 log_type 字段等于 go.log 时执行解析
|
||
when:
|
||
equals:
|
||
log_type: go.log
|
||
fields: ["message"]
|
||
target: ""
|
||
overwrite_keys: true
|
||
add_error_key: true
|
||
|
||
|
||
# ---------- python语言的lessie-agent的Pod, python项目只有文本格式日志, 需排除掉一些不采集的日志 ----------
|
||
- condition:
|
||
# 匹配 sit 命名空间下的 lessie-agent 应用
|
||
and:
|
||
- equals:
|
||
kubernetes.namespace: "sit"
|
||
- equals:
|
||
kubernetes.labels.app: "lessie-agent"
|
||
config:
|
||
- type: filestream
|
||
id: "k8s-python-log-${data.kubernetes.container.id}"
|
||
prospector.scanner.symlinks: true
|
||
parsers:
|
||
- container: ~
|
||
paths:
|
||
- /var/log/containers/*-${data.kubernetes.container.id}.log
|
||
|
||
# 核心采集配置：只包含以时间戳开头的行
|
||
include_lines: ['^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}']
|
||
|
||
ignore_older: 24h
|
||
scan_frequency: 10s
|
||
clean_inactive: 25h
|
||
close_inactive: 5m
|
||
close_renamed: true
|
||
start_position: beginning
|
||
|
||
fields:
|
||
application: ${data.kubernetes.labels.app} # lessie-agent
|
||
log_type: "lessie_search.log" # 保持与处理器 when 条件一致
|
||
environment: ${data.kubernetes.labels.environment}
|
||
instance: ${data.kubernetes.host}
|
||
|
||
processors:
|
||
- add_kubernetes_metadata:
|
||
host: ${NODE_NAME}
|
||
- add_fields:
|
||
fields:
|
||
log_source: k8s
|
||
target: 'mylog'
|
||
|
||
# --- 处理器部分：移植您非 K8s 环境的逻辑 ---
|
||
|
||
# 1. 基础 Dissect 解析
|
||
- dissect:
|
||
when:
|
||
equals:
|
||
log_type: lessie_search.log
|
||
tokenizer: '%{timestamp} - %{level} - %{module} - %{function} - %{message}'
|
||
field: "message"
|
||
target_prefix: "mylog"
|
||
ignore_missing: true
|
||
overwrite_keys: true
|
||
|
||
# 2. 针对带有 [level: | event: | msg: | context:] 的日志，再做一次 dissect
|
||
- dissect:
|
||
when:
|
||
regexp:
|
||
mylog.message: '^\[level:.*\]'
|
||
tokenizer: '[level: %{event_level} | event: %{event} | msg: %{msg} | context: %{context}]'
|
||
field: "mylog.message"
|
||
target_prefix: "mylog"
|
||
ignore_missing: true
|
||
overwrite_keys: true
|
||
|
||
# 3. 把 context 再拆成独立字段 (JavaScript 脚本处理器)
|
||
- script:
|
||
lang: javascript
|
||
id: parse_context
|
||
source: >
|
||
function process(event) {
|
||
var ctx = event.Get("mylog.context");
|
||
if (ctx) {
|
||
var parts = ctx.split(",");
|
||
parts.forEach(function(p) {
|
||
var kv = p.split(":");
|
||
if (kv.length == 2) {
|
||
// 确保 kv[0] 是有效的字段名
|
||
event.Put("mylog." + kv[0].trim(), kv[1].trim());
|
||
}
|
||
});
|
||
}
|
||
}
|
||
|
||
# ---------- python语言的apex的Pod, python项目json格式日志 ----------
|
||
|
||
|
||
# ---------- 前端存储静态资源的nginx pod, nginx 格式日志 ----------
|
||
|
||
|
||
# ---- 输出到 Elasticsearch ----
|
||
output.elasticsearch:
|
||
hosts: ["http://10.0.0.38:9200"]
|
||
username: "admin"
|
||
password: "G7ZSKFM4AQwHQpwA"
|
||
|
||
# 动态索引命名：k8s-环境-应用-日期
|
||
index: "k8s-%{[kubernetes.labels.environment]}-%{[kubernetes.labels.app]}-%{+yyyy.MM.dd}"
|
||
|
||
|
||
logging.level: debug
|
||
logging.selectors: ["*"]
|