增加配置文件

k8s_yaml/ELK/filebast/01-filebeat-serviceaccount.yaml

@@ -0,0 +1,73 @@
+# 定义 Filebeat 的服务账户(ServiceAccount)
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: filebeat                     # 服务账户名称
+  namespace: kube-system             # 所在命名空间
+  labels:
+    k8s-app: filebeat                # 标签，标识这是 Filebeat 应用
+---
+# 定义 Filebeat 的集群角色(ClusterRole)，授予集群范围的权限
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: filebeat                     # 集群角色名称
+  labels:
+    k8s-app: filebeat                # 标签
+rules:
+  # 授予对 namespaces, pods, nodes 资源的 get, list, watch 权限
+  - apiGroups: [""]
+    resources: ["namespaces", "pods", "nodes"]
+    verbs: ["get", "list", "watch"]
+  # 授予对 ReplicaSets 的 get, list, watch 权限
+  - apiGroups: ["apps"]
+    resources: ["replicasets"]
+    verbs: ["get", "list", "watch"]
+  # 授予对 Jobs 的 get, list, watch 权限
+  - apiGroups: ["batch"]
+    resources: ["jobs"]
+    verbs: ["get", "list", "watch"]
+---
+# 定义 Filebeat 的角色(Role)，授予命名空间范围的权限
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: filebeat                     # 角色名称
+  namespace: kube-system             # 作用命名空间
+  labels:
+    k8s-app: filebeat                # 标签
+rules:
+  # 授予对 leases 资源的 get, create, update 权限
+  # Leases 用于协调和领导者选举
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["get", "create", "update"]
+---
+# 将 Filebeat 的服务账户与集群角色绑定(ClusterRoleBinding)
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: filebeat                     # 绑定名称
+subjects:
+  - kind: ServiceAccount             # 主体类型为服务账户
+    name: filebeat                   # 服务账户名称
+    namespace: kube-system           # 服务账户所在命名空间
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole                  # 引用的角色类型
+  name: filebeat                     # 引用的角色名称
+---
+# 将 Filebeat 的服务账户与角色绑定(RoleBinding)
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: filebeat                     # 绑定名称
+  namespace: kube-system             # 作用命名空间
+subjects:
+  - kind: ServiceAccount             # 主体类型为服务账户
+    name: filebeat                   # 服务账户名称
+    namespace: kube-system           # 服务账户所在命名空间
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role                         # 引用的角色类型
+  name: filebeat                     # 引用的角色名称

k8s_yaml/ELK/filebast/02-filebeat-configmap.yaml

@@ -0,0 +1,93 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: filebeat-config
+  namespace: kube-system
+data:
+  filebeat.yml: |
+    setup.ilm.enabled: false
+    setup.template.enabled: false
+
+    filebeat.autodiscover:
+      providers:
+        - type: kubernetes
+          node: ${NODE_NAME}
+          hints.enabled: false
+
+          templates:
+            # ---------- Template 1: java语言的admin、agent、payment Pod, java21项目多行堆栈文本日志 ----------
+            - condition:
+                equals:
+                  kubernetes.namespace: "sit" # 假设你的业务 pod 在 sit 命名空间
+                # or:
+                #   - equals:
+                #       kubernetes.labels.app: "flymoon-admin"
+                #   - equals:
+                #       kubernetes.labels.app: "flymoon-agent"
+                #   - equals:
+                #       kubernetes.labels.app: "flymoon-payment"
+              config:
+                - type: filestream
+                  id: "k8s-log-${data.kubernetes.container.id}"
+                  prospector.scanner.symlinks: true
+                  parsers:
+                    - container: ~
+                  paths:
+                    - /var/log/containers/*-${data.kubernetes.container.id}.log
+                  # multiline:
+                  #   pattern: '^\d{4}-\d{2}-\d{2}-\d{2}:\d{2}:\d{2}\.\d{3}'
+                  #   negate: true
+                  #   match: after
+                  # ignore_older: 24h
+                  # scan_frequency: 10s
+                  # clean_inactive: 25h
+                  # close_inactive: 5m
+                  # close_renamed: true
+                  # start_position: beginning
+                  fields:
+                    application: ${data.kubernetes.labels.app}
+                    log_type: ${data.kubernetes.labels.log_type}
+                    environment: ${data.kubernetes.labels.environment}
+                    instance: ${data.kubernetes.host}
+
+                  processors:
+                    - add_kubernetes_metadata:
+                          host: ${NODE_NAME}
+                    - add_fields:
+                        fields:
+                          log_source: k8s
+                        target: 'mylog'
+                    - dissect:
+                        tokenizer: "%{timestamp} [%{thread}] %{level} %{class} - [%{method},%{line}] - %{message}"
+                        field: "message"
+                        target_prefix: "mylog"
+                        ignore_missing: true
+                        overwrite_keys: true
+
+            # ---------- java语言的email服务的Pod, java1.8项目自由文本格式日志, java21项目格式不太一样, 但也有堆栈信息----------
+
+
+            # ---------- go语言的中转服务的Pod, go项目json格式日志 ----------
+
+
+            # ---------- python语言的lessie-agent的Pod, python项目只有文本格式日志, 需排除掉一些不采集的日志 ----------
+
+
+            # ---------- python语言的apex的Pod, python项目json格式日志 ----------
+
+
+            # ---------- 前端存储静态资源的nginx pod, nginx 格式日志 ----------
+
+
+    # ---- 输出到 Elasticsearch ----
+    output.elasticsearch:
+      hosts: ["http://10.0.0.38:9200"]
+      username: "admin"
+      password: "G7ZSKFM4AQwHQpwA"
+
+      # 动态索引命名：k8s-环境-应用-日期
+      index: "k8s-%{[kubernetes.labels.environment]}-%{[kubernetes.labels.app]}-%{+yyyy.MM.dd}"
+
+
+    logging.level: debug
+    logging.selectors: ["*"]

k8s_yaml/ELK/filebast/021-filebeat-configmap.yaml

@@ -0,0 +1,53 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: filebeat-config
+  namespace: kube-system
+data:
+  filebeat.yml: |
+    setup.ilm.enabled: false
+    setup.template.enabled: false
+
+    filebeat.autodiscover:
+      providers:
+        - type: kubernetes
+          node: ${NODE_NAME}
+          hints.enabled: true
+          hints.default_config:
+            type: filestream
+            id: container-${data.kubernetes.container.id}
+            prospector.scanner.symlinks: true
+            parsers:
+              - container: ~
+            paths:
+              - /var/log/containers/*-${data.kubernetes.container.id}.log
+
+          # templates:
+          #   - condition:
+          #       exists: ['kubernetes.pod.name'] 
+          #     config:
+          #       - type: container
+          #         id: "debug"
+          #         paths:
+          #           - /var/log/containers/*.log
+          #         # follow_symlinks: true
+          #         # parsers:
+          #         #   - container: ~
+
+
+    # ---- 输出到 Elasticsearch ----
+    output.elasticsearch:
+      hosts: ["http://10.0.0.38:9200"]
+      username: "admin"
+      password: "G7ZSKFM4AQwHQpwA"
+
+      # 动态索引命名：k8s-环境-应用-日期
+      index: "k8s-%{[kubernetes.labels.environment]}-%{[kubernetes.labels.app]}-%{+yyyy.MM.dd}"
+
+    logging.level: debug
+    logging.selectors: ["*"]
+
+
+
+
+

k8s_yaml/ELK/filebast/03-filebeat-daemonset.yaml

@@ -0,0 +1,60 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: filebeat
+  namespace: kube-system
+  labels:
+    k8s-app: filebeat
+spec:
+  selector:
+    matchLabels:
+      k8s-app: filebeat
+  template:
+    metadata:
+      labels:
+        k8s-app: filebeat
+    spec:
+      serviceAccountName: filebeat
+      terminationGracePeriodSeconds: 30
+      containers:
+      - name: filebeat
+        image: docker.elastic.co/beats/filebeat:9.2.2
+        args:
+          - "-e"
+        env:
+          - name: NODE_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: spec.nodeName
+        resources:
+          limits:
+            memory: 300Mi
+          requests:
+            cpu: 100m
+            memory: 200Mi
+        volumeMounts:
+          - name: config
+            mountPath: /usr/share/filebeat/filebeat.yml
+            subPath: filebeat.yml
+          - name: data
+            mountPath: /var/lib/filebeat-data
+          - name: containers
+            mountPath: /var/log/containers
+            readOnly: true
+          - name: pods
+            mountPath: /var/log/pods
+            readOnly: true
+      volumes:
+        - name: config
+          configMap:
+            name: filebeat-config
+        - name: data
+          hostPath:
+            path: /var/lib/filebeat-data
+            type: DirectoryOrCreate
+        - name: containers
+          hostPath:
+            path: /var/log/containers
+        - name: pods
+          hostPath:
+            path: /var/log/pods