dxin/jenkins-pipeline
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

更改采集日志模板

Browse Source
This commit is contained in:
dxin
2025-12-24 14:41:42 +08:00
parent a81013a32a
commit c5d60f6aec
4 changed files with 171 additions and 215 deletions
Download Patch File Download Diff File Expand all files Collapse all files

125
k8s_yaml/ELK/filebast/022-filebeat-configmap.yaml
View File

@@ -32,26 +32,125 @@ data:
processors:
- add_kubernetes_metadata:
host: ${NODE_NAME}
include_fields:
- "kubernetes.node.hostname"
- "kubernetes.container.name"
- "kubernetes.pod.name"
- "kubernetes.pod.ip"
- "kubernetes.namespace"
- "kubernetes.labels.app"
- "kubernetes.labels.environment"
- "kubernetes.labels.project"
- decode_json_fields:
fields: ["message"]
target: "mylog"
overwrite_keys: true
add_error_key: true
- drop_fields:
fields:
- ""
fields:
- "kubernetes.node.labels"
- "kubernetes.namespace_labels.kubernetes_io/metadata_name"
ignore_missing: true
# ---------- ↑ go语言的中转服务的Pod, go项目json格式日志 ↑ ----------
# ---------- ↓ java语言的中转服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↓ ----------
- condition:
and:
- equals:
kubernetes.namespace: sit
- or:
- equals:
kubernetes.labels.app: "flymoon-admin"
- equals:
kubernetes.labels.app: "flymoon-agent"
- equals:
kubernetes.labels.app: "flymoon-payment"
config:
- type: filestream
id: "container-${data.kubernetes.container.id}"
prospector.scanner.symlinks: true
close.on_state_change.removed: false
parsers:
- container: ~
- multiline:
type: pattern
pattern: '^\d{4}-\d{2}-\d{2}-\d{2}:\d{2}:\d{2}\.\d{3}'
negate: true
match: after
paths:
- /var/log/containers/*-${data.kubernetes.container.id}.log
processors:
- add_kubernetes_metadata:
host: ${NODE_NAME}
- dissect:
tokenizer: '%{timestamp} %{level} %{pid} --- [%{thread}] %{class} : [%{app_name->}] %{message}'
field: "message"
target_prefix: "mylog"
ignore_missing: true
overwrite_keys: true
- drop_fields:
fields: ["kubernetes.node.labels", "kubernetes.annotations"]
ignore_missing: true
# ---------- ↑ java语言的中转服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↑ ----------
# ---------- ↓ python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↓ ----------
- condition:
and:
- equals:
kubernetes.namespace: sit
- equals:
kubernetes.labels.app: "lessie-agents"
config:
- type: filestream
id: "container-${data.kubernetes.container.id}"
prospector.scanner.symlinks: true
close.on_state_change.removed: false
parsers:
- container: ~
paths:
- /var/log/containers/*-${data.kubernetes.container.id}.log
processors:
- add_kubernetes_metadata:
host: ${NODE_NAME}
# 第一层：仅解析符合时间戳开头的日志行(for业务告警的日志格式)
- dissect:
when:
regexp:
message: '^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}.*'
tokenizer: '%{timestamp} - %{level} - %{module} - %{function} - %{msg_body}'
field: "message"
target_prefix: "mylog"
ignore_missing: true
overwrite_keys: true
# 第二层：针对带有 [level: | event: | msg: | context:] 的日志，再做一次 dissect
- dissect:
when:
contains:
mylog.msg_body: "[level:"
tokenizer: '[level: %{event_level} | event: %{event} | msg: %{msg} | context: %{ctx_raw}]'
field: "mylog.msg_body"
target_prefix: "mylog"
ignore_missing: true
overwrite_keys: true
# 第三层：把 ctx_raw 再拆成独立字段
- script:
lang: javascript
id: parse_context
source: >
function process(event) {
var ctx = event.Get("mylog.ctx_raw");
if (!ctx) return;
var parts = ctx.trim().split(",");
for (var i = 0; i < parts.length; i++) {
var pair = parts[i].split(":");
if (pair.length === 2) {
event.Put("mylog." + pair[0].trim(), pair[1].trim());
}
}
}
# 第四层: 去除大量不需要的k8s元数据字段
- drop_fields:
fields:
- "kubernetes.node.labels"
- "kubernetes.annotations"
ignore_missing: true
# ---------- ↑ python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↑ ----------
# ---- 输出到 Elasticsearch ----