更改采集日志模板
This commit is contained in:
dxin
@@ -11,192 +11,87 @@ data:
|
||||
filebeat.autodiscover:
|
||||
providers:
|
||||
- type: kubernetes
|
||||
node: ${NODE_NAME}
|
||||
hints.enabled: false
|
||||
|
||||
templates:
|
||||
# ---------- Template 1: java语言的admin、agent、payment Pod, java21项目多行堆栈文本日志 ----------
|
||||
# - condition:
|
||||
# # 匹配 sit 命名空间下的 3个 flymoon 应用
|
||||
# and:
|
||||
# - equals:
|
||||
# kubernetes.namespace: "sit"
|
||||
# - regexp:
|
||||
# kubernetes.labels.app: "(flymoon-admin|flymoon-agent|flymoon-payment)"
|
||||
# config:
|
||||
# - type: filestream
|
||||
# id: "k8s-java-log-${data.kubernetes.container.id}"
|
||||
# prospector.scanner.symlinks: true
|
||||
# parsers:
|
||||
# - container: ~
|
||||
# paths:
|
||||
# - /var/log/containers/*-${data.kubernetes.container.id}.log
|
||||
# multiline:
|
||||
# pattern: '^\d{4}-\d{2}-\d{2}-\d{2}:\d{2}:\d{2}\.\d{3}'
|
||||
# negate: true
|
||||
# match: after
|
||||
# ignore_older: 24h
|
||||
# scan_frequency: 10s
|
||||
# clean_inactive: 25h
|
||||
# close_inactive: 5m
|
||||
# close_renamed: true
|
||||
# start_position: beginning
|
||||
# fields:
|
||||
# application: ${data.kubernetes.labels.app}
|
||||
# log_type: ${data.kubernetes.labels.log_type}
|
||||
# environment: ${data.kubernetes.labels.environment}
|
||||
# instance: ${data.kubernetes.host}
|
||||
|
||||
# processors:
|
||||
# - add_kubernetes_metadata:
|
||||
# host: ${NODE_NAME}
|
||||
# - add_fields:
|
||||
# fields:
|
||||
# log_source: k8s
|
||||
# target: 'mylog'
|
||||
# - dissect:
|
||||
# tokenizer: "%{timestamp} [%{thread}] %{level} %{class} - [%{method},%{line}] - %{message}"
|
||||
# field: "message"
|
||||
# target_prefix: "mylog"
|
||||
# ignore_missing: true
|
||||
# overwrite_keys: true
|
||||
|
||||
# ---------- java语言的email服务的Pod, java1.8项目自由文本格式日志, java21项目格式不太一样, 但也有堆栈信息----------
|
||||
|
||||
|
||||
|
||||
|
||||
# ---------- go语言的中转服务的Pod, go项目json格式日志 ----------
|
||||
# ---------- ↓ go语言的中转服务的Pod, go项目json格式日志 ↓ ----------
|
||||
- condition:
|
||||
# 匹配 sit 命名空间下的 lessie-go-api 应用
|
||||
and:
|
||||
- equals:
|
||||
kubernetes.namespace: "sit"
|
||||
kubernetes.namespace: sit
|
||||
- equals:
|
||||
kubernetes.labels.app: "lessie-go-api"
|
||||
config:
|
||||
- type: filestream
|
||||
id: "k8s-go-json-log-${data.kubernetes.container.id}"
|
||||
id: "container-${data.kubernetes.container.id}"
|
||||
prospector.scanner.symlinks: true
|
||||
close.on_state_change.removed: false
|
||||
parsers:
|
||||
- container: ~
|
||||
paths:
|
||||
- /var/log/containers/*-${data.kubernetes.container.id}.log
|
||||
fields:
|
||||
application: ${data.kubernetes.labels.app}
|
||||
log_type: "go.log"
|
||||
environment: ${data.kubernetes.labels.environment}
|
||||
instance: ${data.kubernetes.host}
|
||||
|
||||
ignore_older: 24h
|
||||
scan_frequency: 10s
|
||||
clean_inactive: 25h
|
||||
close_inactive: 5m
|
||||
close_renamed: true
|
||||
start_position: beginning
|
||||
|
||||
processors:
|
||||
- add_kubernetes_metadata:
|
||||
host: ${NODE_NAME}
|
||||
- add_fields:
|
||||
fields:
|
||||
log_source: k8s
|
||||
target: 'mylog'
|
||||
|
||||
# 核心处理器：解析 JSON 格式日志
|
||||
host: ${NODE_NAME}
|
||||
- decode_json_fields:
|
||||
fields: ["message"]
|
||||
target: ""
|
||||
target: "mylog"
|
||||
overwrite_keys: true
|
||||
add_error_key: true
|
||||
- drop_fields:
|
||||
fields:
|
||||
- "kubernetes.node.labels"
|
||||
- "kubernetes.namespace_labels.kubernetes_io/metadata_name"
|
||||
ignore_missing: true
|
||||
|
||||
# ---------- ↑ go语言的中转服务的Pod, go项目json格式日志 ↑ ----------
|
||||
|
||||
# ---------- ↓ java语言的中转服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↓ ----------
|
||||
- condition:
|
||||
and:
|
||||
- equals:
|
||||
kubernetes.namespace: sit
|
||||
- or:
|
||||
- equals:
|
||||
kubernetes.labels.app: "flymoon-admin"
|
||||
- equals:
|
||||
kubernetes.labels.app: "flymoon-agent"
|
||||
- equals:
|
||||
kubernetes.labels.app: "flymoon-payment"
|
||||
config:
|
||||
- type: filestream
|
||||
id: "container-${data.kubernetes.container.id}"
|
||||
prospector.scanner.symlinks: true
|
||||
close.on_state_change.removed: false
|
||||
parsers:
|
||||
- container: ~
|
||||
- multiline:
|
||||
type: pattern
|
||||
pattern: '^\d{4}-\d{2}-\d{2}-\d{2}:\d{2}:\d{2}\.\d{3}'
|
||||
negate: true
|
||||
match: after
|
||||
paths:
|
||||
- /var/log/containers/*-${data.kubernetes.container.id}.log
|
||||
|
||||
processors:
|
||||
- add_kubernetes_metadata:
|
||||
host: ${NODE_NAME}
|
||||
- dissect:
|
||||
tokenizer: '%{timestamp} %{level} %{pid} --- [%{thread}] %{class} : [%{app_name->}] %{message}'
|
||||
field: "message"
|
||||
target_prefix: "mylog"
|
||||
ignore_missing: true
|
||||
overwrite_keys: true
|
||||
- drop_fields:
|
||||
fields: ["kubernetes.node.labels", "kubernetes.annotations"]
|
||||
ignore_missing: true
|
||||
|
||||
# ---------- ↑ java语言的中转服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↑ ----------
|
||||
|
||||
# ---------- ↓ python语言的中转服务的Pod, lessie agent 项目自由文本格式日志 ↓ ----------
|
||||
|
||||
|
||||
# ---------- python语言的lessie-agent的Pod, python项目只有文本格式日志, 需排除掉一些不采集的日志 ----------
|
||||
# - condition:
|
||||
# # 匹配 sit 命名空间下的 lessie-agent 应用
|
||||
# and:
|
||||
# - equals:
|
||||
# kubernetes.namespace: "sit"
|
||||
# - equals:
|
||||
# kubernetes.labels.app: "lessie-agent"
|
||||
# config:
|
||||
# - type: filestream
|
||||
# id: "k8s-python-log-${data.kubernetes.container.id}"
|
||||
# prospector.scanner.symlinks: true
|
||||
# parsers:
|
||||
# - container: ~
|
||||
# paths:
|
||||
# - /var/log/containers/*-${data.kubernetes.container.id}.log
|
||||
|
||||
# # 核心采集配置：只包含以时间戳开头的行
|
||||
# include_lines: ['^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}']
|
||||
|
||||
# ignore_older: 24h
|
||||
# scan_frequency: 10s
|
||||
# clean_inactive: 25h
|
||||
# close_inactive: 5m
|
||||
# close_renamed: true
|
||||
# start_position: beginning
|
||||
|
||||
# fields:
|
||||
# application: ${data.kubernetes.labels.app} # lessie-agent
|
||||
# log_type: "lessie_search.log" # 保持与处理器 when 条件一致
|
||||
# environment: ${data.kubernetes.labels.environment}
|
||||
# instance: ${data.kubernetes.host}
|
||||
|
||||
# processors:
|
||||
# - add_kubernetes_metadata:
|
||||
# host: ${NODE_NAME}
|
||||
# - add_fields:
|
||||
# fields:
|
||||
# log_source: k8s
|
||||
# target: 'mylog'
|
||||
|
||||
# # 1. 基础 Dissect 解析
|
||||
# - dissect:
|
||||
# when:
|
||||
# equals:
|
||||
# log_type: lessie_search.log
|
||||
# tokenizer: '%{timestamp} - %{level} - %{module} - %{function} - %{message}'
|
||||
# field: "message"
|
||||
# target_prefix: "mylog"
|
||||
# ignore_missing: true
|
||||
# overwrite_keys: true
|
||||
|
||||
# # 2. 针对带有 [level: | event: | msg: | context:] 的日志，再做一次 dissect
|
||||
# - dissect:
|
||||
# when:
|
||||
# regexp:
|
||||
# mylog.message: '^\[level:.*\]'
|
||||
# tokenizer: '[level: %{event_level} | event: %{event} | msg: %{msg} | context: %{context}]'
|
||||
# field: "mylog.message"
|
||||
# target_prefix: "mylog"
|
||||
# ignore_missing: true
|
||||
# overwrite_keys: true
|
||||
|
||||
# # 3. 把 context 再拆成独立字段 (JavaScript 脚本处理器)
|
||||
# - script:
|
||||
# lang: javascript
|
||||
# id: parse_context
|
||||
# source: >
|
||||
# function process(event) {
|
||||
# var ctx = event.Get("mylog.context");
|
||||
# if (ctx) {
|
||||
# var parts = ctx.split(",");
|
||||
# parts.forEach(function(p) {
|
||||
# var kv = p.split(":");
|
||||
# if (kv.length == 2) {
|
||||
# // 确保 kv[0] 是有效的字段名
|
||||
# event.Put("mylog." + kv[0].trim(), kv[1].trim());
|
||||
# }
|
||||
# });
|
||||
# }
|
||||
# }
|
||||
|
||||
# ---------- python语言的apex的Pod, python项目json格式日志 ----------
|
||||
|
||||
|
||||
# ---------- 前端存储静态资源的nginx pod, nginx 格式日志 ----------
|
||||
# ---------- ↑ python语言的中转服务的Pod, lessie agent 项目自由文本格式日志 ↑ ----------
|
||||
|
||||
|
||||
# ---- 输出到 Elasticsearch ----
|
||||
@@ -204,10 +99,8 @@ data:
|
||||
hosts: ["http://10.0.0.38:9200"]
|
||||
username: "admin"
|
||||
password: "G7ZSKFM4AQwHQpwA"
|
||||
|
||||
# 动态索引命名：k8s-环境-应用-日期
|
||||
index: "k8s-%{[kubernetes.labels.environment]}-%{[kubernetes.labels.app]}-%{+yyyy.MM.dd}"
|
||||
|
||||
# index: "k8s-%{[kubernetes.labels.app]}-%{+yyyy.MM.dd}"
|
||||
|
||||
logging.level: debug
|
||||
logging.selectors: ["*"]
|
||||
logging.selectors: ["*"]
|
||||
@@ -1,38 +0,0 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: filebeat-config
|
||||
namespace: kube-system
|
||||
data:
|
||||
filebeat.yml: |
|
||||
setup.ilm.enabled: false
|
||||
setup.template.enabled: false
|
||||
|
||||
filebeat.autodiscover:
|
||||
providers:
|
||||
- type: kubernetes
|
||||
node: ${NODE_NAME}
|
||||
hints.enabled: true
|
||||
hints.default_config:
|
||||
type: filestream
|
||||
id: container-${data.kubernetes.container.id}
|
||||
prospector.scanner.symlinks: true
|
||||
parsers:
|
||||
- container: ~
|
||||
paths:
|
||||
- /var/log/containers/*-${data.kubernetes.container.id}.log
|
||||
|
||||
|
||||
# ---- 输出到 Elasticsearch ----
|
||||
output.elasticsearch:
|
||||
hosts: ["http://10.0.0.38:9200"]
|
||||
username: "admin"
|
||||
password: "G7ZSKFM4AQwHQpwA"
|
||||
|
||||
# 动态索引命名：k8s-环境-应用-日期
|
||||
index: "k8s-%{[kubernetes.labels.environment]}-%{[kubernetes.labels.app]}-%{+yyyy.MM.dd}"
|
||||
|
||||
logging.level: debug
|
||||
logging.selectors: ["*"]
|
||||
|
||||
|
||||
@@ -32,26 +32,125 @@ data:
|
||||
processors:
|
||||
- add_kubernetes_metadata:
|
||||
host: ${NODE_NAME}
|
||||
include_fields:
|
||||
- "kubernetes.node.hostname"
|
||||
- "kubernetes.container.name"
|
||||
- "kubernetes.pod.name"
|
||||
- "kubernetes.pod.ip"
|
||||
- "kubernetes.namespace"
|
||||
- "kubernetes.labels.app"
|
||||
- "kubernetes.labels.environment"
|
||||
- "kubernetes.labels.project"
|
||||
|
||||
- decode_json_fields:
|
||||
fields: ["message"]
|
||||
target: "mylog"
|
||||
overwrite_keys: true
|
||||
add_error_key: true
|
||||
|
||||
- drop_fields:
|
||||
fields:
|
||||
- ""
|
||||
fields:
|
||||
- "kubernetes.node.labels"
|
||||
- "kubernetes.namespace_labels.kubernetes_io/metadata_name"
|
||||
ignore_missing: true
|
||||
|
||||
# ---------- ↑ go语言的中转服务的Pod, go项目json格式日志 ↑ ----------
|
||||
|
||||
# ---------- ↓ java语言的中转服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↓ ----------
|
||||
- condition:
|
||||
and:
|
||||
- equals:
|
||||
kubernetes.namespace: sit
|
||||
- or:
|
||||
- equals:
|
||||
kubernetes.labels.app: "flymoon-admin"
|
||||
- equals:
|
||||
kubernetes.labels.app: "flymoon-agent"
|
||||
- equals:
|
||||
kubernetes.labels.app: "flymoon-payment"
|
||||
config:
|
||||
- type: filestream
|
||||
id: "container-${data.kubernetes.container.id}"
|
||||
prospector.scanner.symlinks: true
|
||||
close.on_state_change.removed: false
|
||||
parsers:
|
||||
- container: ~
|
||||
- multiline:
|
||||
type: pattern
|
||||
pattern: '^\d{4}-\d{2}-\d{2}-\d{2}:\d{2}:\d{2}\.\d{3}'
|
||||
negate: true
|
||||
match: after
|
||||
paths:
|
||||
- /var/log/containers/*-${data.kubernetes.container.id}.log
|
||||
|
||||
processors:
|
||||
- add_kubernetes_metadata:
|
||||
host: ${NODE_NAME}
|
||||
- dissect:
|
||||
tokenizer: '%{timestamp} %{level} %{pid} --- [%{thread}] %{class} : [%{app_name->}] %{message}'
|
||||
field: "message"
|
||||
target_prefix: "mylog"
|
||||
ignore_missing: true
|
||||
overwrite_keys: true
|
||||
- drop_fields:
|
||||
fields: ["kubernetes.node.labels", "kubernetes.annotations"]
|
||||
ignore_missing: true
|
||||
|
||||
# ---------- ↑ java语言的中转服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↑ ----------
|
||||
|
||||
# ---------- ↓ python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↓ ----------
|
||||
- condition:
|
||||
and:
|
||||
- equals:
|
||||
kubernetes.namespace: sit
|
||||
- equals:
|
||||
kubernetes.labels.app: "lessie-agents"
|
||||
config:
|
||||
- type: filestream
|
||||
id: "container-${data.kubernetes.container.id}"
|
||||
prospector.scanner.symlinks: true
|
||||
close.on_state_change.removed: false
|
||||
parsers:
|
||||
- container: ~
|
||||
paths:
|
||||
- /var/log/containers/*-${data.kubernetes.container.id}.log
|
||||
|
||||
processors:
|
||||
- add_kubernetes_metadata:
|
||||
host: ${NODE_NAME}
|
||||
# 第一层：仅解析符合时间戳开头的日志行(for业务告警的日志格式)
|
||||
- dissect:
|
||||
when:
|
||||
regexp:
|
||||
message: '^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}.*'
|
||||
tokenizer: '%{timestamp} - %{level} - %{module} - %{function} - %{msg_body}'
|
||||
field: "message"
|
||||
target_prefix: "mylog"
|
||||
ignore_missing: true
|
||||
overwrite_keys: true
|
||||
# 第二层：针对带有 [level: | event: | msg: | context:] 的日志，再做一次 dissect
|
||||
- dissect:
|
||||
when:
|
||||
contains:
|
||||
mylog.msg_body: "[level:"
|
||||
tokenizer: '[level: %{event_level} | event: %{event} | msg: %{msg} | context: %{ctx_raw}]'
|
||||
field: "mylog.msg_body"
|
||||
target_prefix: "mylog"
|
||||
ignore_missing: true
|
||||
overwrite_keys: true
|
||||
# 第三层：把 ctx_raw 再拆成独立字段
|
||||
- script:
|
||||
lang: javascript
|
||||
id: parse_context
|
||||
source: >
|
||||
function process(event) {
|
||||
var ctx = event.Get("mylog.ctx_raw");
|
||||
if (!ctx) return;
|
||||
var parts = ctx.trim().split(",");
|
||||
for (var i = 0; i < parts.length; i++) {
|
||||
var pair = parts[i].split(":");
|
||||
if (pair.length === 2) {
|
||||
event.Put("mylog." + pair[0].trim(), pair[1].trim());
|
||||
}
|
||||
}
|
||||
}
|
||||
# 第四层: 去除大量不需要的k8s元数据字段
|
||||
- drop_fields:
|
||||
fields:
|
||||
- "kubernetes.node.labels"
|
||||
- "kubernetes.annotations"
|
||||
ignore_missing: true
|
||||
|
||||
# ---------- ↑ python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↑ ----------
|
||||
|
||||
|
||||
# ---- 输出到 Elasticsearch ----
|
||||
|
||||
@@ -22,6 +22,8 @@ spec:
|
||||
args:
|
||||
- "-e"
|
||||
env:
|
||||
- name: TZ
|
||||
value: Asia/Shanghai
|
||||
- name: NODE_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
|
||||
Reference in New Issue
Block a user