diff --git a/k8s_yaml/ELK/filebast/01-filebeat-serviceaccount.yaml b/k8s_yaml/ELK/filebast/01-filebeat-serviceaccount.yaml new file mode 100644 index 0000000..0eee6d5 --- /dev/null +++ b/k8s_yaml/ELK/filebast/01-filebeat-serviceaccount.yaml @@ -0,0 +1,73 @@ +# 定义 Filebeat 的服务账户(ServiceAccount) +apiVersion: v1 +kind: ServiceAccount +metadata: + name: filebeat # 服务账户名称 + namespace: kube-system # 所在命名空间 + labels: + k8s-app: filebeat # 标签,标识这是 Filebeat 应用 +--- +# 定义 Filebeat 的集群角色(ClusterRole),授予集群范围的权限 +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: filebeat # 集群角色名称 + labels: + k8s-app: filebeat # 标签 +rules: + # 授予对 namespaces, pods, nodes 资源的 get, list, watch 权限 + - apiGroups: [""] + resources: ["namespaces", "pods", "nodes"] + verbs: ["get", "list", "watch"] + # 授予对 ReplicaSets 的 get, list, watch 权限 + - apiGroups: ["apps"] + resources: ["replicasets"] + verbs: ["get", "list", "watch"] + # 授予对 Jobs 的 get, list, watch 权限 + - apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["get", "list", "watch"] +--- +# 定义 Filebeat 的角色(Role),授予命名空间范围的权限 +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: filebeat # 角色名称 + namespace: kube-system # 作用命名空间 + labels: + k8s-app: filebeat # 标签 +rules: + # 授予对 leases 资源的 get, create, update 权限 + # Leases 用于协调和领导者选举 + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "create", "update"] +--- +# 将 Filebeat 的服务账户与集群角色绑定(ClusterRoleBinding) +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: filebeat # 绑定名称 +subjects: + - kind: ServiceAccount # 主体类型为服务账户 + name: filebeat # 服务账户名称 + namespace: kube-system # 服务账户所在命名空间 +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole # 引用的角色类型 + name: filebeat # 引用的角色名称 +--- +# 将 Filebeat 的服务账户与角色绑定(RoleBinding) +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: filebeat # 绑定名称 + namespace: kube-system # 作用命名空间 +subjects: + - kind: ServiceAccount # 主体类型为服务账户 + name: filebeat # 服务账户名称 + namespace: kube-system # 服务账户所在命名空间 +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role # 引用的角色类型 + name: filebeat # 引用的角色名称 \ No newline at end of file diff --git a/k8s_yaml/ELK/filebast/02-filebeat-configmap.yaml b/k8s_yaml/ELK/filebast/02-filebeat-configmap.yaml new file mode 100644 index 0000000..771e2f9 --- /dev/null +++ b/k8s_yaml/ELK/filebast/02-filebeat-configmap.yaml @@ -0,0 +1,93 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: filebeat-config + namespace: kube-system +data: + filebeat.yml: | + setup.ilm.enabled: false + setup.template.enabled: false + + filebeat.autodiscover: + providers: + - type: kubernetes + node: ${NODE_NAME} + hints.enabled: false + + templates: + # ---------- Template 1: java语言的admin、agent、payment Pod, java21项目多行堆栈文本日志 ---------- + - condition: + equals: + kubernetes.namespace: "sit" # 假设你的业务 pod 在 sit 命名空间 + # or: + # - equals: + # kubernetes.labels.app: "flymoon-admin" + # - equals: + # kubernetes.labels.app: "flymoon-agent" + # - equals: + # kubernetes.labels.app: "flymoon-payment" + config: + - type: filestream + id: "k8s-log-${data.kubernetes.container.id}" + prospector.scanner.symlinks: true + parsers: + - container: ~ + paths: + - /var/log/containers/*-${data.kubernetes.container.id}.log + # multiline: + # pattern: '^\d{4}-\d{2}-\d{2}-\d{2}:\d{2}:\d{2}\.\d{3}' + # negate: true + # match: after + # ignore_older: 24h + # scan_frequency: 10s + # clean_inactive: 25h + # close_inactive: 5m + # close_renamed: true + # start_position: beginning + fields: + application: ${data.kubernetes.labels.app} + log_type: ${data.kubernetes.labels.log_type} + environment: ${data.kubernetes.labels.environment} + instance: ${data.kubernetes.host} + + processors: + - add_kubernetes_metadata: + host: ${NODE_NAME} + - add_fields: + fields: + log_source: k8s + target: 'mylog' + - dissect: + tokenizer: "%{timestamp} [%{thread}] %{level} %{class} - [%{method},%{line}] - %{message}" + field: "message" + target_prefix: "mylog" + ignore_missing: true + overwrite_keys: true + + # ---------- java语言的email服务的Pod, java1.8项目自由文本格式日志, java21项目格式不太一样, 但也有堆栈信息---------- + + + # ---------- go语言的中转服务的Pod, go项目json格式日志 ---------- + + + # ---------- python语言的lessie-agent的Pod, python项目只有文本格式日志, 需排除掉一些不采集的日志 ---------- + + + # ---------- python语言的apex的Pod, python项目json格式日志 ---------- + + + # ---------- 前端存储静态资源的nginx pod, nginx 格式日志 ---------- + + + # ---- 输出到 Elasticsearch ---- + output.elasticsearch: + hosts: ["http://10.0.0.38:9200"] + username: "admin" + password: "G7ZSKFM4AQwHQpwA" + + # 动态索引命名:k8s-环境-应用-日期 + index: "k8s-%{[kubernetes.labels.environment]}-%{[kubernetes.labels.app]}-%{+yyyy.MM.dd}" + + + logging.level: debug + logging.selectors: ["*"] diff --git a/k8s_yaml/ELK/filebast/021-filebeat-configmap.yaml b/k8s_yaml/ELK/filebast/021-filebeat-configmap.yaml new file mode 100644 index 0000000..930338b --- /dev/null +++ b/k8s_yaml/ELK/filebast/021-filebeat-configmap.yaml @@ -0,0 +1,53 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: filebeat-config + namespace: kube-system +data: + filebeat.yml: | + setup.ilm.enabled: false + setup.template.enabled: false + + filebeat.autodiscover: + providers: + - type: kubernetes + node: ${NODE_NAME} + hints.enabled: true + hints.default_config: + type: filestream + id: container-${data.kubernetes.container.id} + prospector.scanner.symlinks: true + parsers: + - container: ~ + paths: + - /var/log/containers/*-${data.kubernetes.container.id}.log + + # templates: + # - condition: + # exists: ['kubernetes.pod.name'] + # config: + # - type: container + # id: "debug" + # paths: + # - /var/log/containers/*.log + # # follow_symlinks: true + # # parsers: + # # - container: ~ + + + # ---- 输出到 Elasticsearch ---- + output.elasticsearch: + hosts: ["http://10.0.0.38:9200"] + username: "admin" + password: "G7ZSKFM4AQwHQpwA" + + # 动态索引命名:k8s-环境-应用-日期 + index: "k8s-%{[kubernetes.labels.environment]}-%{[kubernetes.labels.app]}-%{+yyyy.MM.dd}" + + logging.level: debug + logging.selectors: ["*"] + + + + + diff --git a/k8s_yaml/ELK/filebast/03-filebeat-daemonset.yaml b/k8s_yaml/ELK/filebast/03-filebeat-daemonset.yaml new file mode 100644 index 0000000..492e2e6 --- /dev/null +++ b/k8s_yaml/ELK/filebast/03-filebeat-daemonset.yaml @@ -0,0 +1,60 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: filebeat + namespace: kube-system + labels: + k8s-app: filebeat +spec: + selector: + matchLabels: + k8s-app: filebeat + template: + metadata: + labels: + k8s-app: filebeat + spec: + serviceAccountName: filebeat + terminationGracePeriodSeconds: 30 + containers: + - name: filebeat + image: docker.elastic.co/beats/filebeat:9.2.2 + args: + - "-e" + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + limits: + memory: 300Mi + requests: + cpu: 100m + memory: 200Mi + volumeMounts: + - name: config + mountPath: /usr/share/filebeat/filebeat.yml + subPath: filebeat.yml + - name: data + mountPath: /var/lib/filebeat-data + - name: containers + mountPath: /var/log/containers + readOnly: true + - name: pods + mountPath: /var/log/pods + readOnly: true + volumes: + - name: config + configMap: + name: filebeat-config + - name: data + hostPath: + path: /var/lib/filebeat-data + type: DirectoryOrCreate + - name: containers + hostPath: + path: /var/log/containers + - name: pods + hostPath: + path: /var/log/pods