改改改
This commit is contained in:
73
OpenTelemetry/ES/filebast/01-filebeat-serviceaccount.yaml
Normal file
73
OpenTelemetry/ES/filebast/01-filebeat-serviceaccount.yaml
Normal file
@@ -0,0 +1,73 @@
|
||||
# 定义 Filebeat 的服务账户(ServiceAccount)
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: filebeat # 服务账户名称
|
||||
namespace: kube-system # 所在命名空间
|
||||
labels:
|
||||
k8s-app: filebeat # 标签，标识这是 Filebeat 应用
|
||||
---
|
||||
# 定义 Filebeat 的集群角色(ClusterRole)，授予集群范围的权限
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: filebeat # 集群角色名称
|
||||
labels:
|
||||
k8s-app: filebeat # 标签
|
||||
rules:
|
||||
# 授予对 namespaces, pods, nodes 资源的 get, list, watch 权限
|
||||
- apiGroups: [""]
|
||||
resources: ["namespaces", "pods", "nodes"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
# 授予对 ReplicaSets 的 get, list, watch 权限
|
||||
- apiGroups: ["apps"]
|
||||
resources: ["replicasets"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
# 授予对 Jobs 的 get, list, watch 权限
|
||||
- apiGroups: ["batch"]
|
||||
resources: ["jobs"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
---
|
||||
# 定义 Filebeat 的角色(Role)，授予命名空间范围的权限
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: Role
|
||||
metadata:
|
||||
name: filebeat # 角色名称
|
||||
namespace: kube-system # 作用命名空间
|
||||
labels:
|
||||
k8s-app: filebeat # 标签
|
||||
rules:
|
||||
# 授予对 leases 资源的 get, create, update 权限
|
||||
# Leases 用于协调和领导者选举
|
||||
- apiGroups: ["coordination.k8s.io"]
|
||||
resources: ["leases"]
|
||||
verbs: ["get", "create", "update"]
|
||||
---
|
||||
# 将 Filebeat 的服务账户与集群角色绑定(ClusterRoleBinding)
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: filebeat # 绑定名称
|
||||
subjects:
|
||||
- kind: ServiceAccount # 主体类型为服务账户
|
||||
name: filebeat # 服务账户名称
|
||||
namespace: kube-system # 服务账户所在命名空间
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole # 引用的角色类型
|
||||
name: filebeat # 引用的角色名称
|
||||
---
|
||||
# 将 Filebeat 的服务账户与角色绑定(RoleBinding)
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: filebeat # 绑定名称
|
||||
namespace: kube-system # 作用命名空间
|
||||
subjects:
|
||||
- kind: ServiceAccount # 主体类型为服务账户
|
||||
name: filebeat # 服务账户名称
|
||||
namespace: kube-system # 服务账户所在命名空间
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: Role # 引用的角色类型
|
||||
name: filebeat # 引用的角色名称
|
||||
233
OpenTelemetry/ES/filebast/02-filebeat-configmap.yaml
Normal file
233
OpenTelemetry/ES/filebast/02-filebeat-configmap.yaml
Normal file
@@ -0,0 +1,233 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: filebeat-config
|
||||
namespace: kube-system
|
||||
data:
|
||||
filebeat.yml: |
|
||||
setup.ilm.enabled: false
|
||||
setup.template.enabled: false
|
||||
|
||||
filebeat.autodiscover:
|
||||
providers:
|
||||
- type: kubernetes
|
||||
templates:
|
||||
# ---------- ↓ json格式日志 ↓ ----------
|
||||
- condition:
|
||||
and:
|
||||
- regexp:
|
||||
kubernetes.namespace: "^(sit|apex-evaluation)$"
|
||||
- regexp:
|
||||
kubernetes.labels.app: "^(lessie-go-api|apex)$"
|
||||
config:
|
||||
- type: filestream
|
||||
id: "container-${data.kubernetes.container.id}"
|
||||
prospector.scanner.symlinks: true
|
||||
close.on_state_change.removed: false
|
||||
parsers:
|
||||
- container: ~
|
||||
paths:
|
||||
- /var/log/containers/*-${data.kubernetes.container.id}.log
|
||||
processors:
|
||||
- add_kubernetes_metadata:
|
||||
host: ${NODE_NAME}
|
||||
- decode_json_fields:
|
||||
fields: ["message"]
|
||||
target: "mylog"
|
||||
overwrite_keys: true
|
||||
add_error_key: true
|
||||
- drop_fields:
|
||||
fields:
|
||||
- "kubernetes.node.labels"
|
||||
- "kubernetes.namespace_labels.kubernetes_io/metadata_name"
|
||||
ignore_missing: true
|
||||
# ---------- ↑ json格式日志 ↑ ----------
|
||||
|
||||
|
||||
# ---------- ↓ java语言的服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↓ ----------
|
||||
- condition:
|
||||
and:
|
||||
- equals:
|
||||
kubernetes.namespace: sit
|
||||
- or:
|
||||
- equals:
|
||||
kubernetes.labels.app: "flymoon-admin"
|
||||
- equals:
|
||||
kubernetes.labels.app: "flymoon-agent"
|
||||
- equals:
|
||||
kubernetes.labels.app: "flymoon-payment"
|
||||
config:
|
||||
- type: filestream
|
||||
id: "container-${data.kubernetes.container.id}"
|
||||
prospector.scanner.symlinks: true
|
||||
close.on_state_change.removed: false
|
||||
parsers:
|
||||
- container: ~
|
||||
- multiline:
|
||||
type: pattern
|
||||
pattern: '^\d{4}-\d{2}-\d{2}-\d{2}:\d{2}:\d{2}\.\d{3}'
|
||||
negate: true
|
||||
match: after
|
||||
paths:
|
||||
- /var/log/containers/*-${data.kubernetes.container.id}.log
|
||||
processors:
|
||||
- add_kubernetes_metadata:
|
||||
host: ${NODE_NAME}
|
||||
- dissect:
|
||||
tokenizer: '%{timestamp} %{level} %{pid} --- [%{thread}] %{class} : [%{app_name->}] %{message}'
|
||||
field: "message"
|
||||
target_prefix: "mylog"
|
||||
ignore_missing: true
|
||||
overwrite_keys: true
|
||||
- drop_fields:
|
||||
fields: ["kubernetes.node.labels", "kubernetes.annotations"]
|
||||
ignore_missing: true
|
||||
|
||||
# ---------- ↑ java语言的服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↑ ----------
|
||||
|
||||
|
||||
# ---------- ↓ java语言的服务的Pod, email 项目自由文本格式日志 ↓ ----------
|
||||
- condition:
|
||||
and:
|
||||
- equals:
|
||||
kubernetes.namespace: sit
|
||||
- equals:
|
||||
kubernetes.labels.app: "flymoon-email"
|
||||
config:
|
||||
- type: filestream
|
||||
id: "container-${data.kubernetes.container.id}"
|
||||
prospector.scanner.symlinks: true
|
||||
close.on_state_change.removed: false
|
||||
parsers:
|
||||
- container: ~
|
||||
- multiline:
|
||||
type: pattern
|
||||
pattern: '^\d{4}-\d{2}-\d{2}'
|
||||
negate: true
|
||||
match: after
|
||||
paths:
|
||||
- /var/log/containers/*-${data.kubernetes.container.id}.log
|
||||
processors:
|
||||
- add_kubernetes_metadata:
|
||||
host: ${NODE_NAME}
|
||||
- dissect:
|
||||
tokenizer: '%{timestamp} %{level} %{pid} --- [%{thread}] %{class} : %{message}'
|
||||
field: "message"
|
||||
target_prefix: "mylog"
|
||||
ignore_missing: true
|
||||
overwrite_keys: true
|
||||
- drop_fields:
|
||||
fields: ["kubernetes.node.labels", "kubernetes.annotations"]
|
||||
ignore_missing: true
|
||||
# ---------- ↑ java语言的服务的Pod, email 项目自由文本格式日志 ↑ ----------
|
||||
|
||||
|
||||
# ---------- ↓ python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↓ ----------
|
||||
- condition:
|
||||
and:
|
||||
- equals:
|
||||
kubernetes.namespace: sit
|
||||
- equals:
|
||||
kubernetes.labels.app: "lessie-agents"
|
||||
config:
|
||||
- type: filestream
|
||||
id: "container-${data.kubernetes.container.id}"
|
||||
prospector.scanner.symlinks: true
|
||||
close.on_state_change.removed: false
|
||||
parsers:
|
||||
- container: ~
|
||||
paths:
|
||||
- /var/log/containers/*-${data.kubernetes.container.id}.log
|
||||
processors:
|
||||
- add_kubernetes_metadata:
|
||||
host: ${NODE_NAME}
|
||||
# 第一层：仅解析符合时间戳开头的日志行(for业务告警的日志格式)
|
||||
- dissect:
|
||||
when:
|
||||
regexp:
|
||||
message: '^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}.*'
|
||||
tokenizer: '%{timestamp} - %{level} - %{module} - %{function} - %{msg_body}'
|
||||
field: "message"
|
||||
target_prefix: "mylog"
|
||||
ignore_missing: true
|
||||
overwrite_keys: true
|
||||
# 第二层：针对带有 [level: | event: | msg: | context:] 的日志，再做一次 dissect
|
||||
- dissect:
|
||||
when:
|
||||
contains:
|
||||
mylog.msg_body: "[level:"
|
||||
tokenizer: '[level: %{event_level} | event: %{event} | msg: %{msg} | context: %{ctx_raw}]'
|
||||
field: "mylog.msg_body"
|
||||
target_prefix: "mylog"
|
||||
ignore_missing: true
|
||||
overwrite_keys: true
|
||||
# 第三层：把 ctx_raw 再拆成独立字段
|
||||
- script:
|
||||
lang: javascript
|
||||
id: parse_context
|
||||
source: >
|
||||
function process(event) {
|
||||
var ctx = event.Get("mylog.ctx_raw");
|
||||
if (!ctx) return;
|
||||
var parts = ctx.trim().split(",");
|
||||
for (var i = 0; i < parts.length; i++) {
|
||||
var pair = parts[i].split(":");
|
||||
if (pair.length === 2) {
|
||||
event.Put("mylog." + pair[0].trim(), pair[1].trim());
|
||||
}
|
||||
}
|
||||
}
|
||||
# 第四层: 去除大量不需要的k8s元数据字段
|
||||
- drop_fields:
|
||||
fields:
|
||||
- "kubernetes.node.labels"
|
||||
- "kubernetes.annotations"
|
||||
ignore_missing: true
|
||||
# ---------- ↑ python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↑ ----------
|
||||
|
||||
|
||||
# ---------- ↓ apex 动态创建的 python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↓ ----------
|
||||
- condition:
|
||||
and:
|
||||
- equals:
|
||||
kubernetes.namespace: apex-evaluation
|
||||
- equals:
|
||||
kubernetes.labels.apex: "lessie-agents"
|
||||
config:
|
||||
- type: filestream
|
||||
id: "container-${data.kubernetes.container.id}"
|
||||
prospector.scanner.symlinks: true
|
||||
close.on_state_change.removed: false
|
||||
parsers:
|
||||
- container: ~
|
||||
paths:
|
||||
- /var/log/containers/*-${data.kubernetes.container.id}.log
|
||||
processors:
|
||||
- drop_fields:
|
||||
fields:
|
||||
- "kubernetes.node.labels"
|
||||
- "kubernetes.annotations"
|
||||
ignore_missing: true
|
||||
# ---------- ↑ apex 动态创建的 python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↑ ----------
|
||||
|
||||
|
||||
|
||||
# ---- 输出到 Elasticsearch ----
|
||||
output.elasticsearch:
|
||||
hosts: ["http://10.0.0.38:9200"]
|
||||
username: "admin"
|
||||
password: "G7ZSKFM4AQwHQpwA"
|
||||
|
||||
indices:
|
||||
- index: "k8s-%{[kubernetes.labels.environment]}-%{[kubernetes.labels.app]}-%{+yyyy.MM}"
|
||||
when:
|
||||
regexp:
|
||||
kubernetes.labels.app: "(lessie-go-api|flymoon-admin|flymoon-agent|flymoon-payment|flymoon-email|lessie-agents|apex)"
|
||||
|
||||
- index: "apex-python-%{+yyyy.MM}"
|
||||
when:
|
||||
equals:
|
||||
kubernetes.labels.apex: "lessie-agents"
|
||||
|
||||
logging.level: info
|
||||
logging.selectors: ["*"]
|
||||
233
OpenTelemetry/ES/filebast/022-filebeat-configmap.yaml
Normal file
233
OpenTelemetry/ES/filebast/022-filebeat-configmap.yaml
Normal file
@@ -0,0 +1,233 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: filebeat-config
|
||||
namespace: kube-system
|
||||
data:
|
||||
filebeat.yml: |
|
||||
setup.ilm.enabled: false
|
||||
setup.template.enabled: false
|
||||
|
||||
filebeat.autodiscover:
|
||||
providers:
|
||||
- type: kubernetes
|
||||
templates:
|
||||
# ---------- ↓ json格式日志 ↓ ----------
|
||||
- condition:
|
||||
and:
|
||||
- regexp:
|
||||
kubernetes.namespace: "^(sit|apex-evaluation)$"
|
||||
- regexp:
|
||||
kubernetes.labels.app: "^(lessie-go-api|apex)$"
|
||||
config:
|
||||
- type: filestream
|
||||
id: "container-${data.kubernetes.container.id}"
|
||||
prospector.scanner.symlinks: true
|
||||
close.on_state_change.removed: false
|
||||
parsers:
|
||||
- container: ~
|
||||
paths:
|
||||
- /var/log/containers/*-${data.kubernetes.container.id}.log
|
||||
processors:
|
||||
- add_kubernetes_metadata:
|
||||
host: ${NODE_NAME}
|
||||
- decode_json_fields:
|
||||
fields: ["message"]
|
||||
target: "mylog"
|
||||
overwrite_keys: true
|
||||
add_error_key: true
|
||||
- drop_fields:
|
||||
fields:
|
||||
- "kubernetes.node.labels"
|
||||
- "kubernetes.namespace_labels.kubernetes_io/metadata_name"
|
||||
ignore_missing: true
|
||||
# ---------- ↑ json格式日志 ↑ ----------
|
||||
|
||||
|
||||
# ---------- ↓ java语言的服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↓ ----------
|
||||
- condition:
|
||||
and:
|
||||
- equals:
|
||||
kubernetes.namespace: sit
|
||||
- or:
|
||||
- equals:
|
||||
kubernetes.labels.app: "flymoon-admin"
|
||||
- equals:
|
||||
kubernetes.labels.app: "flymoon-agent"
|
||||
- equals:
|
||||
kubernetes.labels.app: "flymoon-payment"
|
||||
config:
|
||||
- type: filestream
|
||||
id: "container-${data.kubernetes.container.id}"
|
||||
prospector.scanner.symlinks: true
|
||||
close.on_state_change.removed: false
|
||||
parsers:
|
||||
- container: ~
|
||||
- multiline:
|
||||
type: pattern
|
||||
pattern: '^\d{4}-\d{2}-\d{2}-\d{2}:\d{2}:\d{2}\.\d{3}'
|
||||
negate: true
|
||||
match: after
|
||||
paths:
|
||||
- /var/log/containers/*-${data.kubernetes.container.id}.log
|
||||
processors:
|
||||
- add_kubernetes_metadata:
|
||||
host: ${NODE_NAME}
|
||||
- dissect:
|
||||
tokenizer: '%{timestamp} %{level} %{pid} --- [%{thread}] %{class} : [%{app_name->}] %{message}'
|
||||
field: "message"
|
||||
target_prefix: "mylog"
|
||||
ignore_missing: true
|
||||
overwrite_keys: true
|
||||
- drop_fields:
|
||||
fields: ["kubernetes.node.labels", "kubernetes.annotations"]
|
||||
ignore_missing: true
|
||||
|
||||
# ---------- ↑ java语言的服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↑ ----------
|
||||
|
||||
|
||||
# ---------- ↓ java语言的服务的Pod, email 项目自由文本格式日志 ↓ ----------
|
||||
- condition:
|
||||
and:
|
||||
- equals:
|
||||
kubernetes.namespace: sit
|
||||
- equals:
|
||||
kubernetes.labels.app: "flymoon-email"
|
||||
config:
|
||||
- type: filestream
|
||||
id: "container-${data.kubernetes.container.id}"
|
||||
prospector.scanner.symlinks: true
|
||||
close.on_state_change.removed: false
|
||||
parsers:
|
||||
- container: ~
|
||||
- multiline:
|
||||
type: pattern
|
||||
pattern: '^\d{4}-\d{2}-\d{2}'
|
||||
negate: true
|
||||
match: after
|
||||
paths:
|
||||
- /var/log/containers/*-${data.kubernetes.container.id}.log
|
||||
processors:
|
||||
- add_kubernetes_metadata:
|
||||
host: ${NODE_NAME}
|
||||
- dissect:
|
||||
tokenizer: '%{timestamp} %{level} %{pid} --- [%{thread}] %{class} : %{message}'
|
||||
field: "message"
|
||||
target_prefix: "mylog"
|
||||
ignore_missing: true
|
||||
overwrite_keys: true
|
||||
- drop_fields:
|
||||
fields: ["kubernetes.node.labels", "kubernetes.annotations"]
|
||||
ignore_missing: true
|
||||
# ---------- ↑ java语言的服务的Pod, email 项目自由文本格式日志 ↑ ----------
|
||||
|
||||
|
||||
# ---------- ↓ python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↓ ----------
|
||||
- condition:
|
||||
and:
|
||||
- equals:
|
||||
kubernetes.namespace: sit
|
||||
- equals:
|
||||
kubernetes.labels.app: "lessie-agents"
|
||||
config:
|
||||
- type: filestream
|
||||
id: "container-${data.kubernetes.container.id}"
|
||||
prospector.scanner.symlinks: true
|
||||
close.on_state_change.removed: false
|
||||
parsers:
|
||||
- container: ~
|
||||
paths:
|
||||
- /var/log/containers/*-${data.kubernetes.container.id}.log
|
||||
processors:
|
||||
- add_kubernetes_metadata:
|
||||
host: ${NODE_NAME}
|
||||
# 第一层：仅解析符合时间戳开头的日志行(for业务告警的日志格式)
|
||||
- dissect:
|
||||
when:
|
||||
regexp:
|
||||
message: '^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}.*'
|
||||
tokenizer: '%{timestamp} - %{level} - %{module} - %{function} - %{msg_body}'
|
||||
field: "message"
|
||||
target_prefix: "mylog"
|
||||
ignore_missing: true
|
||||
overwrite_keys: true
|
||||
# 第二层：针对带有 [level: | event: | msg: | context:] 的日志，再做一次 dissect
|
||||
- dissect:
|
||||
when:
|
||||
contains:
|
||||
mylog.msg_body: "[level:"
|
||||
tokenizer: '[level: %{event_level} | event: %{event} | msg: %{msg} | context: %{ctx_raw}]'
|
||||
field: "mylog.msg_body"
|
||||
target_prefix: "mylog"
|
||||
ignore_missing: true
|
||||
overwrite_keys: true
|
||||
# 第三层：把 ctx_raw 再拆成独立字段
|
||||
- script:
|
||||
lang: javascript
|
||||
id: parse_context
|
||||
source: >
|
||||
function process(event) {
|
||||
var ctx = event.Get("mylog.ctx_raw");
|
||||
if (!ctx) return;
|
||||
var parts = ctx.trim().split(",");
|
||||
for (var i = 0; i < parts.length; i++) {
|
||||
var pair = parts[i].split(":");
|
||||
if (pair.length === 2) {
|
||||
event.Put("mylog." + pair[0].trim(), pair[1].trim());
|
||||
}
|
||||
}
|
||||
}
|
||||
# 第四层: 去除大量不需要的k8s元数据字段
|
||||
- drop_fields:
|
||||
fields:
|
||||
- "kubernetes.node.labels"
|
||||
- "kubernetes.annotations"
|
||||
ignore_missing: true
|
||||
# ---------- ↑ python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↑ ----------
|
||||
|
||||
|
||||
# ---------- ↓ apex 动态创建的 python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↓ ----------
|
||||
- condition:
|
||||
and:
|
||||
- equals:
|
||||
kubernetes.namespace: apex-evaluation
|
||||
- equals:
|
||||
kubernetes.labels.apex: "lessie-agents"
|
||||
config:
|
||||
- type: filestream
|
||||
id: "container-${data.kubernetes.container.id}"
|
||||
prospector.scanner.symlinks: true
|
||||
close.on_state_change.removed: false
|
||||
parsers:
|
||||
- container: ~
|
||||
paths:
|
||||
- /var/log/containers/*-${data.kubernetes.container.id}.log
|
||||
processors:
|
||||
- drop_fields:
|
||||
fields:
|
||||
- "kubernetes.node.labels"
|
||||
- "kubernetes.annotations"
|
||||
ignore_missing: true
|
||||
# ---------- ↑ apex 动态创建的 python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↑ ----------
|
||||
|
||||
|
||||
|
||||
# ---- 输出到 Elasticsearch ----
|
||||
output.elasticsearch:
|
||||
hosts: ["http://10.0.0.38:9200"]
|
||||
username: "admin"
|
||||
password: "G7ZSKFM4AQwHQpwA"
|
||||
|
||||
indices:
|
||||
- index: "k8s-%{[kubernetes.labels.environment]}-%{[kubernetes.labels.app]}-%{+yyyy.MM}"
|
||||
when:
|
||||
regexp:
|
||||
kubernetes.labels.app: "(lessie-go-api|flymoon-admin|flymoon-agent|flymoon-payment|flymoon-email|lessie-agents|apex)"
|
||||
|
||||
- index: "apex-python-%{+yyyy.MM}"
|
||||
when:
|
||||
equals:
|
||||
kubernetes.labels.apex: "lessie-agents"
|
||||
|
||||
logging.level: info
|
||||
logging.selectors: ["*"]
|
||||
65
OpenTelemetry/ES/filebast/03-filebeat-daemonset.yaml
Normal file
65
OpenTelemetry/ES/filebast/03-filebeat-daemonset.yaml
Normal file
@@ -0,0 +1,65 @@
|
||||
# 滚动更新
|
||||
# kubectl rollout restart daemonset filebeat -n kube-system
|
||||
|
||||
apiVersion: apps/v1
|
||||
kind: DaemonSet
|
||||
metadata:
|
||||
name: filebeat
|
||||
namespace: kube-system
|
||||
labels:
|
||||
k8s-app: filebeat
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
k8s-app: filebeat
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
k8s-app: filebeat
|
||||
spec:
|
||||
serviceAccountName: filebeat
|
||||
terminationGracePeriodSeconds: 30
|
||||
containers:
|
||||
- name: filebeat
|
||||
image: docker.elastic.co/beats/filebeat:9.2.2
|
||||
args:
|
||||
- "-e"
|
||||
env:
|
||||
- name: TZ
|
||||
value: Asia/Shanghai
|
||||
- name: NODE_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: spec.nodeName
|
||||
resources:
|
||||
limits:
|
||||
memory: 300Mi
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 200Mi
|
||||
volumeMounts:
|
||||
- name: config
|
||||
mountPath: /usr/share/filebeat/filebeat.yml
|
||||
subPath: filebeat.yml
|
||||
- name: data
|
||||
mountPath: /var/lib/filebeat-data
|
||||
- name: containers
|
||||
mountPath: /var/log/containers
|
||||
readOnly: true
|
||||
- name: pods
|
||||
mountPath: /var/log/pods
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: config
|
||||
configMap:
|
||||
name: filebeat-config
|
||||
- name: data
|
||||
hostPath:
|
||||
path: /var/lib/filebeat-data
|
||||
type: DirectoryOrCreate
|
||||
- name: containers
|
||||
hostPath:
|
||||
path: /var/log/containers
|
||||
- name: pods
|
||||
hostPath:
|
||||
path: /var/log/pods
|
||||
226
OpenTelemetry/ES/filebast/filebeat.yaml
Normal file
226
OpenTelemetry/ES/filebast/filebeat.yaml
Normal file
@@ -0,0 +1,226 @@
|
||||
setup.ilm.enabled: false
|
||||
setup.template.enabled: false
|
||||
|
||||
filebeat.autodiscover:
|
||||
providers:
|
||||
- type: kubernetes
|
||||
templates:
|
||||
# ---------- ↓ json格式日志 ↓ ----------
|
||||
- condition:
|
||||
and:
|
||||
- regexp:
|
||||
kubernetes.namespace: "^(sit|apex-evaluation)$"
|
||||
- regexp:
|
||||
kubernetes.labels.app: "^(lessie-go-api|apex)$"
|
||||
config:
|
||||
- type: filestream
|
||||
id: "container-${data.kubernetes.container.id}"
|
||||
prospector.scanner.symlinks: true
|
||||
close.on_state_change.removed: false
|
||||
parsers:
|
||||
- container: ~
|
||||
paths:
|
||||
- /var/log/containers/*-${data.kubernetes.container.id}.log
|
||||
processors:
|
||||
- add_kubernetes_metadata:
|
||||
host: ${NODE_NAME}
|
||||
- decode_json_fields:
|
||||
fields: ["message"]
|
||||
target: "mylog"
|
||||
overwrite_keys: true
|
||||
add_error_key: true
|
||||
- drop_fields:
|
||||
fields:
|
||||
- "kubernetes.node.labels"
|
||||
- "kubernetes.namespace_labels.kubernetes_io/metadata_name"
|
||||
ignore_missing: true
|
||||
# ---------- ↑ json格式日志 ↑ ----------
|
||||
|
||||
|
||||
# ---------- ↓ java语言的服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↓ ----------
|
||||
- condition:
|
||||
and:
|
||||
- equals:
|
||||
kubernetes.namespace: sit
|
||||
- or:
|
||||
- equals:
|
||||
kubernetes.labels.app: "flymoon-admin"
|
||||
- equals:
|
||||
kubernetes.labels.app: "flymoon-agent"
|
||||
- equals:
|
||||
kubernetes.labels.app: "flymoon-payment"
|
||||
config:
|
||||
- type: filestream
|
||||
id: "container-${data.kubernetes.container.id}"
|
||||
prospector.scanner.symlinks: true
|
||||
close.on_state_change.removed: false
|
||||
parsers:
|
||||
- container: ~
|
||||
- multiline:
|
||||
type: pattern
|
||||
pattern: '^\d{4}-\d{2}-\d{2}-\d{2}:\d{2}:\d{2}\.\d{3}'
|
||||
negate: true
|
||||
match: after
|
||||
paths:
|
||||
- /var/log/containers/*-${data.kubernetes.container.id}.log
|
||||
processors:
|
||||
- add_kubernetes_metadata:
|
||||
host: ${NODE_NAME}
|
||||
- dissect:
|
||||
tokenizer: '%{timestamp} %{level} %{pid} --- [%{thread}] %{class} : [%{app_name->}] %{message}'
|
||||
field: "message"
|
||||
target_prefix: "mylog"
|
||||
ignore_missing: true
|
||||
overwrite_keys: true
|
||||
- drop_fields:
|
||||
fields: ["kubernetes.node.labels", "kubernetes.annotations"]
|
||||
ignore_missing: true
|
||||
|
||||
# ---------- ↑ java语言的服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↑ ----------
|
||||
|
||||
|
||||
# ---------- ↓ java语言的服务的Pod, email 项目自由文本格式日志 ↓ ----------
|
||||
- condition:
|
||||
and:
|
||||
- equals:
|
||||
kubernetes.namespace: sit
|
||||
- equals:
|
||||
kubernetes.labels.app: "flymoon-email"
|
||||
config:
|
||||
- type: filestream
|
||||
id: "container-${data.kubernetes.container.id}"
|
||||
prospector.scanner.symlinks: true
|
||||
close.on_state_change.removed: false
|
||||
parsers:
|
||||
- container: ~
|
||||
- multiline:
|
||||
type: pattern
|
||||
pattern: '^\d{4}-\d{2}-\d{2}'
|
||||
negate: true
|
||||
match: after
|
||||
paths:
|
||||
- /var/log/containers/*-${data.kubernetes.container.id}.log
|
||||
processors:
|
||||
- add_kubernetes_metadata:
|
||||
host: ${NODE_NAME}
|
||||
- dissect:
|
||||
tokenizer: '%{timestamp} %{level} %{pid} --- [%{thread}] %{class} : %{message}'
|
||||
field: "message"
|
||||
target_prefix: "mylog"
|
||||
ignore_missing: true
|
||||
overwrite_keys: true
|
||||
- drop_fields:
|
||||
fields: ["kubernetes.node.labels", "kubernetes.annotations"]
|
||||
ignore_missing: true
|
||||
# ---------- ↑ java语言的服务的Pod, email 项目自由文本格式日志 ↑ ----------
|
||||
|
||||
|
||||
# ---------- ↓ python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↓ ----------
|
||||
- condition:
|
||||
and:
|
||||
- equals:
|
||||
kubernetes.namespace: sit
|
||||
- equals:
|
||||
kubernetes.labels.app: "lessie-agents"
|
||||
config:
|
||||
- type: filestream
|
||||
id: "container-${data.kubernetes.container.id}"
|
||||
prospector.scanner.symlinks: true
|
||||
close.on_state_change.removed: false
|
||||
parsers:
|
||||
- container: ~
|
||||
paths:
|
||||
- /var/log/containers/*-${data.kubernetes.container.id}.log
|
||||
processors:
|
||||
- add_kubernetes_metadata:
|
||||
host: ${NODE_NAME}
|
||||
# 第一层：仅解析符合时间戳开头的日志行(for业务告警的日志格式)
|
||||
- dissect:
|
||||
when:
|
||||
regexp:
|
||||
message: '^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}.*'
|
||||
tokenizer: '%{timestamp} - %{level} - %{module} - %{function} - %{msg_body}'
|
||||
field: "message"
|
||||
target_prefix: "mylog"
|
||||
ignore_missing: true
|
||||
overwrite_keys: true
|
||||
# 第二层：针对带有 [level: | event: | msg: | context:] 的日志，再做一次 dissect
|
||||
- dissect:
|
||||
when:
|
||||
contains:
|
||||
mylog.msg_body: "[level:"
|
||||
tokenizer: '[level: %{event_level} | event: %{event} | msg: %{msg} | context: %{ctx_raw}]'
|
||||
field: "mylog.msg_body"
|
||||
target_prefix: "mylog"
|
||||
ignore_missing: true
|
||||
overwrite_keys: true
|
||||
# 第三层：把 ctx_raw 再拆成独立字段
|
||||
- script:
|
||||
lang: javascript
|
||||
id: parse_context
|
||||
source: >
|
||||
function process(event) {
|
||||
var ctx = event.Get("mylog.ctx_raw");
|
||||
if (!ctx) return;
|
||||
var parts = ctx.trim().split(",");
|
||||
for (var i = 0; i < parts.length; i++) {
|
||||
var pair = parts[i].split(":");
|
||||
if (pair.length === 2) {
|
||||
event.Put("mylog." + pair[0].trim(), pair[1].trim());
|
||||
}
|
||||
}
|
||||
}
|
||||
# 第四层: 去除大量不需要的k8s元数据字段
|
||||
- drop_fields:
|
||||
fields:
|
||||
- "kubernetes.node.labels"
|
||||
- "kubernetes.annotations"
|
||||
ignore_missing: true
|
||||
# ---------- ↑ python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↑ ----------
|
||||
|
||||
|
||||
# ---------- ↓ apex 动态创建的 python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↓ ----------
|
||||
- condition:
|
||||
and:
|
||||
- equals:
|
||||
kubernetes.namespace: apex-evaluation
|
||||
- equals:
|
||||
kubernetes.labels.apex: "lessie-agents"
|
||||
config:
|
||||
- type: filestream
|
||||
id: "container-${data.kubernetes.container.id}"
|
||||
prospector.scanner.symlinks: true
|
||||
close.on_state_change.removed: false
|
||||
parsers:
|
||||
- container: ~
|
||||
paths:
|
||||
- /var/log/containers/*-${data.kubernetes.container.id}.log
|
||||
processors:
|
||||
- drop_fields:
|
||||
fields:
|
||||
- "kubernetes.node.labels"
|
||||
- "kubernetes.annotations"
|
||||
ignore_missing: true
|
||||
# ---------- ↑ apex 动态创建的 python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↑ ----------
|
||||
|
||||
|
||||
|
||||
# ---- 输出到 Elasticsearch ----
|
||||
output.elasticsearch:
|
||||
hosts: ["http://10.0.0.38:9200"]
|
||||
username: "admin"
|
||||
password: "G7ZSKFM4AQwHQpwA"
|
||||
|
||||
indices:
|
||||
- index: "k8s-%{[kubernetes.labels.environment]}-%{[kubernetes.labels.app]}-%{+yyyy.MM.dd}"
|
||||
when:
|
||||
regexp:
|
||||
kubernetes.labels.app: "(lessie-go-api|flymoon-admin|flymoon-agent|flymoon-payment|flymoon-email|lessie-agents|apex)"
|
||||
|
||||
- index: "apex-python-%{+yyyy.MM.dd}"
|
||||
when:
|
||||
equals:
|
||||
kubernetes.labels.apex: "lessie-agents"
|
||||
|
||||
logging.level: info
|
||||
logging.selectors: ["*"]
|
||||
Reference in New Issue
Block a user