增加配置文件
This commit is contained in:
dxin
73
k8s_yaml/ELK/filebast/01-filebeat-serviceaccount.yaml
Normal file
73
k8s_yaml/ELK/filebast/01-filebeat-serviceaccount.yaml
Normal file
@@ -0,0 +1,73 @@
|
|||||||
|
# 定义 Filebeat 的服务账户(ServiceAccount)
|
||||||
|
apiVersion: v1
|
||||||
|
kind: ServiceAccount
|
||||||
|
metadata:
|
||||||
|
name: filebeat # 服务账户名称
|
||||||
|
namespace: kube-system # 所在命名空间
|
||||||
|
labels:
|
||||||
|
k8s-app: filebeat # 标签，标识这是 Filebeat 应用
|
||||||
|
---
|
||||||
|
# 定义 Filebeat 的集群角色(ClusterRole)，授予集群范围的权限
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: ClusterRole
|
||||||
|
metadata:
|
||||||
|
name: filebeat # 集群角色名称
|
||||||
|
labels:
|
||||||
|
k8s-app: filebeat # 标签
|
||||||
|
rules:
|
||||||
|
# 授予对 namespaces, pods, nodes 资源的 get, list, watch 权限
|
||||||
|
- apiGroups: [""]
|
||||||
|
resources: ["namespaces", "pods", "nodes"]
|
||||||
|
verbs: ["get", "list", "watch"]
|
||||||
|
# 授予对 ReplicaSets 的 get, list, watch 权限
|
||||||
|
- apiGroups: ["apps"]
|
||||||
|
resources: ["replicasets"]
|
||||||
|
verbs: ["get", "list", "watch"]
|
||||||
|
# 授予对 Jobs 的 get, list, watch 权限
|
||||||
|
- apiGroups: ["batch"]
|
||||||
|
resources: ["jobs"]
|
||||||
|
verbs: ["get", "list", "watch"]
|
||||||
|
---
|
||||||
|
# 定义 Filebeat 的角色(Role)，授予命名空间范围的权限
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: Role
|
||||||
|
metadata:
|
||||||
|
name: filebeat # 角色名称
|
||||||
|
namespace: kube-system # 作用命名空间
|
||||||
|
labels:
|
||||||
|
k8s-app: filebeat # 标签
|
||||||
|
rules:
|
||||||
|
# 授予对 leases 资源的 get, create, update 权限
|
||||||
|
# Leases 用于协调和领导者选举
|
||||||
|
- apiGroups: ["coordination.k8s.io"]
|
||||||
|
resources: ["leases"]
|
||||||
|
verbs: ["get", "create", "update"]
|
||||||
|
---
|
||||||
|
# 将 Filebeat 的服务账户与集群角色绑定(ClusterRoleBinding)
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: ClusterRoleBinding
|
||||||
|
metadata:
|
||||||
|
name: filebeat # 绑定名称
|
||||||
|
subjects:
|
||||||
|
- kind: ServiceAccount # 主体类型为服务账户
|
||||||
|
name: filebeat # 服务账户名称
|
||||||
|
namespace: kube-system # 服务账户所在命名空间
|
||||||
|
roleRef:
|
||||||
|
apiGroup: rbac.authorization.k8s.io
|
||||||
|
kind: ClusterRole # 引用的角色类型
|
||||||
|
name: filebeat # 引用的角色名称
|
||||||
|
---
|
||||||
|
# 将 Filebeat 的服务账户与角色绑定(RoleBinding)
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: RoleBinding
|
||||||
|
metadata:
|
||||||
|
name: filebeat # 绑定名称
|
||||||
|
namespace: kube-system # 作用命名空间
|
||||||
|
subjects:
|
||||||
|
- kind: ServiceAccount # 主体类型为服务账户
|
||||||
|
name: filebeat # 服务账户名称
|
||||||
|
namespace: kube-system # 服务账户所在命名空间
|
||||||
|
roleRef:
|
||||||
|
apiGroup: rbac.authorization.k8s.io
|
||||||
|
kind: Role # 引用的角色类型
|
||||||
|
name: filebeat # 引用的角色名称
|
||||||
93
k8s_yaml/ELK/filebast/02-filebeat-configmap.yaml
Normal file
93
k8s_yaml/ELK/filebast/02-filebeat-configmap.yaml
Normal file
@@ -0,0 +1,93 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: ConfigMap
|
||||||
|
metadata:
|
||||||
|
name: filebeat-config
|
||||||
|
namespace: kube-system
|
||||||
|
data:
|
||||||
|
filebeat.yml: |
|
||||||
|
setup.ilm.enabled: false
|
||||||
|
setup.template.enabled: false
|
||||||
|
|
||||||
|
filebeat.autodiscover:
|
||||||
|
providers:
|
||||||
|
- type: kubernetes
|
||||||
|
node: ${NODE_NAME}
|
||||||
|
hints.enabled: false
|
||||||
|
|
||||||
|
templates:
|
||||||
|
# ---------- Template 1: java语言的admin、agent、payment Pod, java21项目多行堆栈文本日志 ----------
|
||||||
|
- condition:
|
||||||
|
equals:
|
||||||
|
kubernetes.namespace: "sit" # 假设你的业务 pod 在 sit 命名空间
|
||||||
|
# or:
|
||||||
|
# - equals:
|
||||||
|
# kubernetes.labels.app: "flymoon-admin"
|
||||||
|
# - equals:
|
||||||
|
# kubernetes.labels.app: "flymoon-agent"
|
||||||
|
# - equals:
|
||||||
|
# kubernetes.labels.app: "flymoon-payment"
|
||||||
|
config:
|
||||||
|
- type: filestream
|
||||||
|
id: "k8s-log-${data.kubernetes.container.id}"
|
||||||
|
prospector.scanner.symlinks: true
|
||||||
|
parsers:
|
||||||
|
- container: ~
|
||||||
|
paths:
|
||||||
|
- /var/log/containers/*-${data.kubernetes.container.id}.log
|
||||||
|
# multiline:
|
||||||
|
# pattern: '^\d{4}-\d{2}-\d{2}-\d{2}:\d{2}:\d{2}\.\d{3}'
|
||||||
|
# negate: true
|
||||||
|
# match: after
|
||||||
|
# ignore_older: 24h
|
||||||
|
# scan_frequency: 10s
|
||||||
|
# clean_inactive: 25h
|
||||||
|
# close_inactive: 5m
|
||||||
|
# close_renamed: true
|
||||||
|
# start_position: beginning
|
||||||
|
fields:
|
||||||
|
application: ${data.kubernetes.labels.app}
|
||||||
|
log_type: ${data.kubernetes.labels.log_type}
|
||||||
|
environment: ${data.kubernetes.labels.environment}
|
||||||
|
instance: ${data.kubernetes.host}
|
||||||
|
|
||||||
|
processors:
|
||||||
|
- add_kubernetes_metadata:
|
||||||
|
host: ${NODE_NAME}
|
||||||
|
- add_fields:
|
||||||
|
fields:
|
||||||
|
log_source: k8s
|
||||||
|
target: 'mylog'
|
||||||
|
- dissect:
|
||||||
|
tokenizer: "%{timestamp} [%{thread}] %{level} %{class} - [%{method},%{line}] - %{message}"
|
||||||
|
field: "message"
|
||||||
|
target_prefix: "mylog"
|
||||||
|
ignore_missing: true
|
||||||
|
overwrite_keys: true
|
||||||
|
|
||||||
|
# ---------- java语言的email服务的Pod, java1.8项目自由文本格式日志, java21项目格式不太一样, 但也有堆栈信息----------
|
||||||
|
|
||||||
|
|
||||||
|
# ---------- go语言的中转服务的Pod, go项目json格式日志 ----------
|
||||||
|
|
||||||
|
|
||||||
|
# ---------- python语言的lessie-agent的Pod, python项目只有文本格式日志, 需排除掉一些不采集的日志 ----------
|
||||||
|
|
||||||
|
|
||||||
|
# ---------- python语言的apex的Pod, python项目json格式日志 ----------
|
||||||
|
|
||||||
|
|
||||||
|
# ---------- 前端存储静态资源的nginx pod, nginx 格式日志 ----------
|
||||||
|
|
||||||
|
|
||||||
|
# ---- 输出到 Elasticsearch ----
|
||||||
|
output.elasticsearch:
|
||||||
|
hosts: ["http://10.0.0.38:9200"]
|
||||||
|
username: "admin"
|
||||||
|
password: "G7ZSKFM4AQwHQpwA"
|
||||||
|
|
||||||
|
# 动态索引命名：k8s-环境-应用-日期
|
||||||
|
index: "k8s-%{[kubernetes.labels.environment]}-%{[kubernetes.labels.app]}-%{+yyyy.MM.dd}"
|
||||||
|
|
||||||
|
|
||||||
|
logging.level: debug
|
||||||
|
logging.selectors: ["*"]
|
||||||
53
k8s_yaml/ELK/filebast/021-filebeat-configmap.yaml
Normal file
53
k8s_yaml/ELK/filebast/021-filebeat-configmap.yaml
Normal file
@@ -0,0 +1,53 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: ConfigMap
|
||||||
|
metadata:
|
||||||
|
name: filebeat-config
|
||||||
|
namespace: kube-system
|
||||||
|
data:
|
||||||
|
filebeat.yml: |
|
||||||
|
setup.ilm.enabled: false
|
||||||
|
setup.template.enabled: false
|
||||||
|
|
||||||
|
filebeat.autodiscover:
|
||||||
|
providers:
|
||||||
|
- type: kubernetes
|
||||||
|
node: ${NODE_NAME}
|
||||||
|
hints.enabled: true
|
||||||
|
hints.default_config:
|
||||||
|
type: filestream
|
||||||
|
id: container-${data.kubernetes.container.id}
|
||||||
|
prospector.scanner.symlinks: true
|
||||||
|
parsers:
|
||||||
|
- container: ~
|
||||||
|
paths:
|
||||||
|
- /var/log/containers/*-${data.kubernetes.container.id}.log
|
||||||
|
|
||||||
|
# templates:
|
||||||
|
# - condition:
|
||||||
|
# exists: ['kubernetes.pod.name']
|
||||||
|
# config:
|
||||||
|
# - type: container
|
||||||
|
# id: "debug"
|
||||||
|
# paths:
|
||||||
|
# - /var/log/containers/*.log
|
||||||
|
# # follow_symlinks: true
|
||||||
|
# # parsers:
|
||||||
|
# # - container: ~
|
||||||
|
|
||||||
|
|
||||||
|
# ---- 输出到 Elasticsearch ----
|
||||||
|
output.elasticsearch:
|
||||||
|
hosts: ["http://10.0.0.38:9200"]
|
||||||
|
username: "admin"
|
||||||
|
password: "G7ZSKFM4AQwHQpwA"
|
||||||
|
|
||||||
|
# 动态索引命名：k8s-环境-应用-日期
|
||||||
|
index: "k8s-%{[kubernetes.labels.environment]}-%{[kubernetes.labels.app]}-%{+yyyy.MM.dd}"
|
||||||
|
|
||||||
|
logging.level: debug
|
||||||
|
logging.selectors: ["*"]
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
60
k8s_yaml/ELK/filebast/03-filebeat-daemonset.yaml
Normal file
60
k8s_yaml/ELK/filebast/03-filebeat-daemonset.yaml
Normal file
@@ -0,0 +1,60 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: DaemonSet
|
||||||
|
metadata:
|
||||||
|
name: filebeat
|
||||||
|
namespace: kube-system
|
||||||
|
labels:
|
||||||
|
k8s-app: filebeat
|
||||||
|
spec:
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
k8s-app: filebeat
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
k8s-app: filebeat
|
||||||
|
spec:
|
||||||
|
serviceAccountName: filebeat
|
||||||
|
terminationGracePeriodSeconds: 30
|
||||||
|
containers:
|
||||||
|
- name: filebeat
|
||||||
|
image: docker.elastic.co/beats/filebeat:9.2.2
|
||||||
|
args:
|
||||||
|
- "-e"
|
||||||
|
env:
|
||||||
|
- name: NODE_NAME
|
||||||
|
valueFrom:
|
||||||
|
fieldRef:
|
||||||
|
fieldPath: spec.nodeName
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
memory: 300Mi
|
||||||
|
requests:
|
||||||
|
cpu: 100m
|
||||||
|
memory: 200Mi
|
||||||
|
volumeMounts:
|
||||||
|
- name: config
|
||||||
|
mountPath: /usr/share/filebeat/filebeat.yml
|
||||||
|
subPath: filebeat.yml
|
||||||
|
- name: data
|
||||||
|
mountPath: /var/lib/filebeat-data
|
||||||
|
- name: containers
|
||||||
|
mountPath: /var/log/containers
|
||||||
|
readOnly: true
|
||||||
|
- name: pods
|
||||||
|
mountPath: /var/log/pods
|
||||||
|
readOnly: true
|
||||||
|
volumes:
|
||||||
|
- name: config
|
||||||
|
configMap:
|
||||||
|
name: filebeat-config
|
||||||
|
- name: data
|
||||||
|
hostPath:
|
||||||
|
path: /var/lib/filebeat-data
|
||||||
|
type: DirectoryOrCreate
|
||||||
|
- name: containers
|
||||||
|
hostPath:
|
||||||
|
path: /var/log/containers
|
||||||
|
- name: pods
|
||||||
|
hostPath:
|
||||||
|
path: /var/log/pods
|
||||||
Reference in New Issue
Block a user