jenkins-pipeline/k8s_yaml/ELK/filebast/02-filebeat-configmap.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: filebeat-config
  namespace: kube-system
data:
  filebeat.yml: |
    setup.ilm.enabled: false
    setup.template.enabled: false

    filebeat.autodiscover:
      providers:
        - type: kubernetes
          templates:
            # ---------- ↓ go语言的中转服务的Pod, go项目json格式日志 ↓ ----------
            - condition:
                and:
                  - equals:
                      kubernetes.namespace: sit
                  - equals:
                      kubernetes.labels.app: "lessie-go-api"
              config:
                - type: filestream
                  id: "container-${data.kubernetes.container.id}"
                  prospector.scanner.symlinks: true
                  close.on_state_change.removed: false
                  parsers:
                    - container: ~
                  paths:
                    - /var/log/containers/*-${data.kubernetes.container.id}.log

                  processors:
                    - add_kubernetes_metadata:
                        host: ${NODE_NAME}
                    - decode_json_fields:
                        fields: ["message"]
                        target: "mylog"
                        overwrite_keys: true
                        add_error_key: true
                    - drop_fields:
                        fields: 
                          - "kubernetes.node.labels"
                          - "kubernetes.namespace_labels.kubernetes_io/metadata_name"
                        ignore_missing: true

            # ---------- ↑ go语言的中转服务的Pod, go项目json格式日志 ↑ ----------
            
            # ---------- ↓ java语言的中转服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↓ ----------
            - condition:
                and:
                  - equals:
                      kubernetes.namespace: sit
                  - or:
                    - equals:
                        kubernetes.labels.app: "flymoon-admin"
                    - equals:
                        kubernetes.labels.app: "flymoon-agent"
                    - equals:
                        kubernetes.labels.app: "flymoon-payment"
              config:
                - type: filestream
                  id: "container-${data.kubernetes.container.id}"
                  prospector.scanner.symlinks: true
                  close.on_state_change.removed: false
                  parsers:
                    - container: ~
                    - multiline:
                        type: pattern
                        pattern: '^\d{4}-\d{2}-\d{2}-\d{2}:\d{2}:\d{2}\.\d{3}'
                        negate: true
                        match: after
                  paths:
                    - /var/log/containers/*-${data.kubernetes.container.id}.log

                  processors:
                    - add_kubernetes_metadata:
                        host: ${NODE_NAME}
                    - dissect:
                        tokenizer: '%{timestamp}  %{level}  %{pid}  ---  [%{thread}]  %{class}  :  [%{app_name->}]  %{message}'
                        field: "message"
                        target_prefix: "mylog"
                        ignore_missing: true
                        overwrite_keys: true
                    - drop_fields:
                        fields: ["kubernetes.node.labels", "kubernetes.annotations"]
                        ignore_missing: true

            # ---------- ↑ java语言的中转服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↑ ----------

            # ---------- ↓ python语言的中转服务的Pod, lessie agent 项目自由文本格式日志 ↓ ----------



            # ---------- ↑ python语言的中转服务的Pod, lessie agent 项目自由文本格式日志 ↑ ----------


    # ---- 输出到 Elasticsearch ----
    output.elasticsearch:
      hosts: ["http://10.0.0.38:9200"]
      username: "admin"
      password: "G7ZSKFM4AQwHQpwA"
      index: "k8s-%{[kubernetes.labels.environment]}-%{[kubernetes.labels.app]}-%{+yyyy.MM.dd}"
      # index: "k8s-%{[kubernetes.labels.app]}-%{+yyyy.MM.dd}"

    logging.level: debug
    logging.selectors:  ["*"]