2025-12-17 17:32:04 +08:00
|
|
|
setup.ilm.enabled: false
|
|
|
|
|
setup.template.enabled: false
|
|
|
|
|
|
|
|
|
|
filebeat.autodiscover:
|
|
|
|
|
providers:
|
|
|
|
|
- type: kubernetes
|
|
|
|
|
templates:
|
2026-01-09 17:52:16 +08:00
|
|
|
# ---------- ↓ json格式日志 ↓ ----------
|
2025-12-17 17:32:04 +08:00
|
|
|
- condition:
|
2026-01-09 17:52:16 +08:00
|
|
|
and:
|
|
|
|
|
- regexp:
|
|
|
|
|
kubernetes.namespace: "^(sit|apex-evaluation)$"
|
|
|
|
|
- regexp:
|
2026-02-11 14:55:11 +08:00
|
|
|
kubernetes.labels.app: "^(lessie-go-api|apex)$"
|
2025-12-17 17:32:04 +08:00
|
|
|
config:
|
|
|
|
|
- type: filestream
|
2026-01-09 17:52:16 +08:00
|
|
|
id: "container-${data.kubernetes.container.id}"
|
2025-12-17 17:32:04 +08:00
|
|
|
prospector.scanner.symlinks: true
|
|
|
|
|
close.on_state_change.removed: false
|
|
|
|
|
parsers:
|
|
|
|
|
- container: ~
|
|
|
|
|
paths:
|
|
|
|
|
- /var/log/containers/*-${data.kubernetes.container.id}.log
|
2026-01-09 17:52:16 +08:00
|
|
|
processors:
|
|
|
|
|
- add_kubernetes_metadata:
|
|
|
|
|
host: ${NODE_NAME}
|
|
|
|
|
- decode_json_fields:
|
|
|
|
|
fields: ["message"]
|
|
|
|
|
target: "mylog"
|
|
|
|
|
overwrite_keys: true
|
|
|
|
|
add_error_key: true
|
|
|
|
|
- drop_fields:
|
|
|
|
|
fields:
|
|
|
|
|
- "kubernetes.node.labels"
|
|
|
|
|
- "kubernetes.namespace_labels.kubernetes_io/metadata_name"
|
|
|
|
|
ignore_missing: true
|
|
|
|
|
# ---------- ↑ json格式日志 ↑ ----------
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# ---------- ↓ java语言的服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↓ ----------
|
|
|
|
|
- condition:
|
|
|
|
|
and:
|
|
|
|
|
- equals:
|
|
|
|
|
kubernetes.namespace: sit
|
|
|
|
|
- or:
|
|
|
|
|
- equals:
|
|
|
|
|
kubernetes.labels.app: "flymoon-admin"
|
|
|
|
|
- equals:
|
|
|
|
|
kubernetes.labels.app: "flymoon-agent"
|
|
|
|
|
- equals:
|
|
|
|
|
kubernetes.labels.app: "flymoon-payment"
|
|
|
|
|
config:
|
|
|
|
|
- type: filestream
|
|
|
|
|
id: "container-${data.kubernetes.container.id}"
|
|
|
|
|
prospector.scanner.symlinks: true
|
|
|
|
|
close.on_state_change.removed: false
|
|
|
|
|
parsers:
|
|
|
|
|
- container: ~
|
|
|
|
|
- multiline:
|
|
|
|
|
type: pattern
|
|
|
|
|
pattern: '^\d{4}-\d{2}-\d{2}-\d{2}:\d{2}:\d{2}\.\d{3}'
|
|
|
|
|
negate: true
|
|
|
|
|
match: after
|
|
|
|
|
paths:
|
|
|
|
|
- /var/log/containers/*-${data.kubernetes.container.id}.log
|
|
|
|
|
processors:
|
|
|
|
|
- add_kubernetes_metadata:
|
|
|
|
|
host: ${NODE_NAME}
|
|
|
|
|
- dissect:
|
|
|
|
|
tokenizer: '%{timestamp} %{level} %{pid} --- [%{thread}] %{class} : [%{app_name->}] %{message}'
|
|
|
|
|
field: "message"
|
|
|
|
|
target_prefix: "mylog"
|
|
|
|
|
ignore_missing: true
|
|
|
|
|
overwrite_keys: true
|
|
|
|
|
- drop_fields:
|
|
|
|
|
fields: ["kubernetes.node.labels", "kubernetes.annotations"]
|
|
|
|
|
ignore_missing: true
|
|
|
|
|
|
|
|
|
|
# ---------- ↑ java语言的服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↑ ----------
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# ---------- ↓ java语言的服务的Pod, email 项目自由文本格式日志 ↓ ----------
|
|
|
|
|
- condition:
|
|
|
|
|
and:
|
|
|
|
|
- equals:
|
|
|
|
|
kubernetes.namespace: sit
|
|
|
|
|
- equals:
|
|
|
|
|
kubernetes.labels.app: "flymoon-email"
|
|
|
|
|
config:
|
|
|
|
|
- type: filestream
|
|
|
|
|
id: "container-${data.kubernetes.container.id}"
|
|
|
|
|
prospector.scanner.symlinks: true
|
|
|
|
|
close.on_state_change.removed: false
|
|
|
|
|
parsers:
|
|
|
|
|
- container: ~
|
|
|
|
|
- multiline:
|
|
|
|
|
type: pattern
|
|
|
|
|
pattern: '^\d{4}-\d{2}-\d{2}'
|
|
|
|
|
negate: true
|
|
|
|
|
match: after
|
|
|
|
|
paths:
|
|
|
|
|
- /var/log/containers/*-${data.kubernetes.container.id}.log
|
|
|
|
|
processors:
|
|
|
|
|
- add_kubernetes_metadata:
|
|
|
|
|
host: ${NODE_NAME}
|
|
|
|
|
- dissect:
|
|
|
|
|
tokenizer: '%{timestamp} %{level} %{pid} --- [%{thread}] %{class} : %{message}'
|
|
|
|
|
field: "message"
|
|
|
|
|
target_prefix: "mylog"
|
|
|
|
|
ignore_missing: true
|
|
|
|
|
overwrite_keys: true
|
|
|
|
|
- drop_fields:
|
|
|
|
|
fields: ["kubernetes.node.labels", "kubernetes.annotations"]
|
|
|
|
|
ignore_missing: true
|
|
|
|
|
# ---------- ↑ java语言的服务的Pod, email 项目自由文本格式日志 ↑ ----------
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# ---------- ↓ python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↓ ----------
|
|
|
|
|
- condition:
|
|
|
|
|
and:
|
|
|
|
|
- equals:
|
|
|
|
|
kubernetes.namespace: sit
|
|
|
|
|
- equals:
|
|
|
|
|
kubernetes.labels.app: "lessie-agents"
|
|
|
|
|
config:
|
|
|
|
|
- type: filestream
|
|
|
|
|
id: "container-${data.kubernetes.container.id}"
|
|
|
|
|
prospector.scanner.symlinks: true
|
|
|
|
|
close.on_state_change.removed: false
|
|
|
|
|
parsers:
|
|
|
|
|
- container: ~
|
|
|
|
|
paths:
|
|
|
|
|
- /var/log/containers/*-${data.kubernetes.container.id}.log
|
|
|
|
|
processors:
|
|
|
|
|
- add_kubernetes_metadata:
|
|
|
|
|
host: ${NODE_NAME}
|
|
|
|
|
# 第一层：仅解析符合时间戳开头的日志行(for业务告警的日志格式)
|
|
|
|
|
- dissect:
|
|
|
|
|
when:
|
|
|
|
|
regexp:
|
|
|
|
|
message: '^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}.*'
|
|
|
|
|
tokenizer: '%{timestamp} - %{level} - %{module} - %{function} - %{msg_body}'
|
|
|
|
|
field: "message"
|
|
|
|
|
target_prefix: "mylog"
|
|
|
|
|
ignore_missing: true
|
|
|
|
|
overwrite_keys: true
|
|
|
|
|
# 第二层：针对带有 [level: | event: | msg: | context:] 的日志，再做一次 dissect
|
|
|
|
|
- dissect:
|
|
|
|
|
when:
|
|
|
|
|
contains:
|
|
|
|
|
mylog.msg_body: "[level:"
|
|
|
|
|
tokenizer: '[level: %{event_level} | event: %{event} | msg: %{msg} | context: %{ctx_raw}]'
|
|
|
|
|
field: "mylog.msg_body"
|
|
|
|
|
target_prefix: "mylog"
|
|
|
|
|
ignore_missing: true
|
|
|
|
|
overwrite_keys: true
|
|
|
|
|
# 第三层：把 ctx_raw 再拆成独立字段
|
|
|
|
|
- script:
|
|
|
|
|
lang: javascript
|
|
|
|
|
id: parse_context
|
|
|
|
|
source: >
|
|
|
|
|
function process(event) {
|
|
|
|
|
var ctx = event.Get("mylog.ctx_raw");
|
|
|
|
|
if (!ctx) return;
|
|
|
|
|
var parts = ctx.trim().split(",");
|
|
|
|
|
for (var i = 0; i < parts.length; i++) {
|
|
|
|
|
var pair = parts[i].split(":");
|
|
|
|
|
if (pair.length === 2) {
|
|
|
|
|
event.Put("mylog." + pair[0].trim(), pair[1].trim());
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
# 第四层: 去除大量不需要的k8s元数据字段
|
|
|
|
|
- drop_fields:
|
|
|
|
|
fields:
|
|
|
|
|
- "kubernetes.node.labels"
|
|
|
|
|
- "kubernetes.annotations"
|
|
|
|
|
ignore_missing: true
|
|
|
|
|
# ---------- ↑ python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↑ ----------
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# ---------- ↓ apex 动态创建的 python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↓ ----------
|
|
|
|
|
- condition:
|
|
|
|
|
and:
|
|
|
|
|
- equals:
|
|
|
|
|
kubernetes.namespace: apex-evaluation
|
|
|
|
|
- equals:
|
|
|
|
|
kubernetes.labels.apex: "lessie-agents"
|
|
|
|
|
config:
|
|
|
|
|
- type: filestream
|
|
|
|
|
id: "container-${data.kubernetes.container.id}"
|
|
|
|
|
prospector.scanner.symlinks: true
|
|
|
|
|
close.on_state_change.removed: false
|
|
|
|
|
parsers:
|
|
|
|
|
- container: ~
|
|
|
|
|
paths:
|
|
|
|
|
- /var/log/containers/*-${data.kubernetes.container.id}.log
|
|
|
|
|
processors:
|
|
|
|
|
- drop_fields:
|
|
|
|
|
fields:
|
|
|
|
|
- "kubernetes.node.labels"
|
|
|
|
|
- "kubernetes.annotations"
|
|
|
|
|
ignore_missing: true
|
|
|
|
|
# ---------- ↑ apex 动态创建的 python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↑ ----------
|
|
|
|
|
|
2025-12-17 17:32:04 +08:00
|
|
|
|
|
|
|
|
|
|
|
|
|
# ---- 输出到 Elasticsearch ----
|
|
|
|
|
output.elasticsearch:
|
|
|
|
|
hosts: ["http://10.0.0.38:9200"]
|
|
|
|
|
username: "admin"
|
|
|
|
|
password: "G7ZSKFM4AQwHQpwA"
|
|
|
|
|
|
2026-01-09 17:52:16 +08:00
|
|
|
indices:
|
2026-01-27 18:21:54 +08:00
|
|
|
- index: "k8s-%{[kubernetes.labels.environment]}-%{[kubernetes.labels.app]}-%{+yyyy.MM}"
|
2026-01-09 17:52:16 +08:00
|
|
|
when:
|
|
|
|
|
regexp:
|
|
|
|
|
kubernetes.labels.app: "(lessie-go-api|flymoon-admin|flymoon-agent|flymoon-payment|flymoon-email|lessie-agents|apex)"
|
|
|
|
|
|
2026-01-27 18:21:54 +08:00
|
|
|
- index: "apex-python-%{+yyyy.MM}"
|
2026-01-09 17:52:16 +08:00
|
|
|
when:
|
|
|
|
|
equals:
|
|
|
|
|
kubernetes.labels.apex: "lessie-agents"
|
|
|
|
|
|
|
|
|
|
logging.level: info
|
|
|
|
|
logging.selectors: ["*"]
|
