dxin/jenkins-pipeline
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

增加测试配置文件

Browse Source
This commit is contained in:
dxin
2025-12-14 23:43:32 +08:00
parent 036fe5335b
commit 290abb3fa5
2 changed files with 163 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files

240
k8s_yaml/ELK/filebast/02-filebeat-configmap.yaml
View File

@@ -12,54 +12,54 @@ data:
providers: providers:
- type: kubernetes - type: kubernetes
node: ${NODE_NAME} node: ${NODE_NAME}
hints.enabled: true hints.enabled: false
templates: templates:
# ---------- Template 1: java语言的admin、agent、payment Pod, java21项目多行堆栈文本日志 ---------- # ---------- Template 1: java语言的admin、agent、payment Pod, java21项目多行堆栈文本日志 ----------
- condition: # - condition:
# 匹配 sit 命名空间下的 3个 flymoon 应用 # # 匹配 sit 命名空间下的 3个 flymoon 应用
and: # and:
- equals: # - equals:
kubernetes.namespace: "sit" # kubernetes.namespace: "sit"
- regexp: # - regexp:
kubernetes.labels.app: "(flymoon-admin|flymoon-agent|flymoon-payment)" # kubernetes.labels.app: "(flymoon-admin|flymoon-agent|flymoon-payment)"
config: # config:
- type: filestream # - type: filestream
id: "k8s-java-log-${data.kubernetes.container.id}" # id: "k8s-java-log-${data.kubernetes.container.id}"
prospector.scanner.symlinks: true # prospector.scanner.symlinks: true
parsers: # parsers:
- container: ~ # - container: ~
paths: # paths:
- /var/log/containers/*-${data.kubernetes.container.id}.log # - /var/log/containers/*-${data.kubernetes.container.id}.log
multiline: # multiline:
pattern: '^\d{4}-\d{2}-\d{2}-\d{2}:\d{2}:\d{2}\.\d{3}' # pattern: '^\d{4}-\d{2}-\d{2}-\d{2}:\d{2}:\d{2}\.\d{3}'
negate: true # negate: true
match: after # match: after
ignore_older: 24h # ignore_older: 24h
scan_frequency: 10s # scan_frequency: 10s
clean_inactive: 25h # clean_inactive: 25h
close_inactive: 5m # close_inactive: 5m
close_renamed: true # close_renamed: true
start_position: beginning # start_position: beginning
fields: # fields:
application: ${data.kubernetes.labels.app} # application: ${data.kubernetes.labels.app}
log_type: ${data.kubernetes.labels.log_type} # log_type: ${data.kubernetes.labels.log_type}
environment: ${data.kubernetes.labels.environment} # environment: ${data.kubernetes.labels.environment}
instance: ${data.kubernetes.host} # instance: ${data.kubernetes.host}
processors: # processors:
- add_kubernetes_metadata: # - add_kubernetes_metadata:
host: ${NODE_NAME} # host: ${NODE_NAME}
- add_fields: # - add_fields:
fields: # fields:
log_source: k8s # log_source: k8s
target: 'mylog' # target: 'mylog'
- dissect: # - dissect:
tokenizer: "%{timestamp} [%{thread}] %{level} %{class} - [%{method},%{line}] - %{message}" # tokenizer: "%{timestamp} [%{thread}] %{level} %{class} - [%{method},%{line}] - %{message}"
field: "message" # field: "message"
target_prefix: "mylog" # target_prefix: "mylog"
ignore_missing: true # ignore_missing: true
overwrite_keys: true # overwrite_keys: true
# ---------- java语言的email服务的Pod, java1.8项目自由文本格式日志, java21项目格式不太一样, 但也有堆栈信息---------- # ---------- java语言的email服务的Pod, java1.8项目自由文本格式日志, java21项目格式不太一样, 但也有堆栈信息----------
@@ -105,10 +105,6 @@ data:
# 核心处理器：解析 JSON 格式日志 # 核心处理器：解析 JSON 格式日志
- decode_json_fields: - decode_json_fields:
# 仅在 log_type 字段等于 go.log 时执行解析
when:
equals:
log_type: go.log
fields: ["message"] fields: ["message"]
target: "" target: ""
overwrite_keys: true overwrite_keys: true
@@ -116,88 +112,86 @@ data:
# ---------- python语言的lessie-agent的Pod, python项目只有文本格式日志, 需排除掉一些不采集的日志 ---------- # ---------- python语言的lessie-agent的Pod, python项目只有文本格式日志, 需排除掉一些不采集的日志 ----------
- condition: # - condition:
# 匹配 sit 命名空间下的 lessie-agent 应用 # # 匹配 sit 命名空间下的 lessie-agent 应用
and: # and:
- equals: # - equals:
kubernetes.namespace: "sit" # kubernetes.namespace: "sit"
- equals: # - equals:
kubernetes.labels.app: "lessie-agent" # kubernetes.labels.app: "lessie-agent"
config: # config:
- type: filestream # - type: filestream
id: "k8s-python-log-${data.kubernetes.container.id}" # id: "k8s-python-log-${data.kubernetes.container.id}"
prospector.scanner.symlinks: true # prospector.scanner.symlinks: true
parsers: # parsers:
- container: ~ # - container: ~
paths: # paths:
- /var/log/containers/*-${data.kubernetes.container.id}.log # - /var/log/containers/*-${data.kubernetes.container.id}.log
# 核心采集配置：只包含以时间戳开头的行 # # 核心采集配置：只包含以时间戳开头的行
include_lines: ['^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}'] # include_lines: ['^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}']
ignore_older: 24h # ignore_older: 24h
scan_frequency: 10s # scan_frequency: 10s
clean_inactive: 25h # clean_inactive: 25h
close_inactive: 5m # close_inactive: 5m
close_renamed: true # close_renamed: true
start_position: beginning # start_position: beginning
fields: # fields:
application: ${data.kubernetes.labels.app} # lessie-agent # application: ${data.kubernetes.labels.app} # lessie-agent
log_type: "lessie_search.log" # 保持与处理器 when 条件一致 # log_type: "lessie_search.log" # 保持与处理器 when 条件一致
environment: ${data.kubernetes.labels.environment} # environment: ${data.kubernetes.labels.environment}
instance: ${data.kubernetes.host} # instance: ${data.kubernetes.host}
processors: # processors:
- add_kubernetes_metadata: # - add_kubernetes_metadata:
host: ${NODE_NAME} # host: ${NODE_NAME}
- add_fields: # - add_fields:
fields: # fields:
log_source: k8s # log_source: k8s
target: 'mylog' # target: 'mylog'
# --- 处理器部分：移植您非 K8s 环境的逻辑 --- # # 1. 基础 Dissect 解析
# - dissect:
# when:
# equals:
# log_type: lessie_search.log
# tokenizer: '%{timestamp} - %{level} - %{module} - %{function} - %{message}'
# field: "message"
# target_prefix: "mylog"
# ignore_missing: true
# overwrite_keys: true
# 1. 基础 Dissect 解析 # # 2. 针对带有 [level: | event: | msg: | context:] 的日志，再做一次 dissect
- dissect: # - dissect:
when: # when:
equals: # regexp:
log_type: lessie_search.log # mylog.message: '^\[level:.*\]'
tokenizer: '%{timestamp} - %{level} - %{module} - %{function} - %{message}' # tokenizer: '[level: %{event_level} | event: %{event} | msg: %{msg} | context: %{context}]'
field: "message" # field: "mylog.message"
target_prefix: "mylog" # target_prefix: "mylog"
ignore_missing: true # ignore_missing: true
overwrite_keys: true # overwrite_keys: true
# 2. 针对带有 [level: | event: | msg: | context:] 的日志，再做一次 dissect # # 3. 把 context 再拆成独立字段 (JavaScript 脚本处理器)
- dissect: # - script:
when: # lang: javascript
regexp: # id: parse_context
mylog.message: '^\[level:.*\]' # source: >
tokenizer: '[level: %{event_level} | event: %{event} | msg: %{msg} | context: %{context}]' # function process(event) {
field: "mylog.message" # var ctx = event.Get("mylog.context");
target_prefix: "mylog" # if (ctx) {
ignore_missing: true # var parts = ctx.split(",");
overwrite_keys: true # parts.forEach(function(p) {
# var kv = p.split(":");
# 3. 把 context 再拆成独立字段 (JavaScript 脚本处理器) # if (kv.length == 2) {
- script: # // 确保 kv[0] 是有效的字段名
lang: javascript # event.Put("mylog." + kv[0].trim(), kv[1].trim());
id: parse_context # }
source: > # });
function process(event) { # }
var ctx = event.Get("mylog.context"); # }
if (ctx) {
var parts = ctx.split(",");
parts.forEach(function(p) {
var kv = p.split(":");
if (kv.length == 2) {
// 确保 kv[0] 是有效的字段名
event.Put("mylog." + kv[0].trim(), kv[1].trim());
}
});
}
}
# ---------- python语言的apex的Pod, python项目json格式日志 ---------- # ---------- python语言的apex的Pod, python项目json格式日志 ----------

46
k8s_yaml/ELK/filebast/022-filebeat-configmap.yaml Normal file
View File

@@ -0,0 +1,46 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: filebeat-config
namespace: kube-system
data:
filebeat.yml: |
setup.ilm.enabled: false
setup.template.enabled: false
filebeat.autodiscover:
providers:
# 配置 Provider
- type: kubernetes
node: ${NODE_NAME}
hints.enabled: false
templates:
# ---------- go语言的中转服务的Pod, go项目json格式日志 ----------
- condition:
equals:
kubernetes.namespace: kube-system
config:
- type: filestream
id: "k8s-go-json-log-${data.kubernetes.container.id}"
prospector.scanner.symlinks: true
parsers:
- container: ~
paths:
- /var/log/containers/*-${data.kubernetes.container.id}.log
fields:
application: ${data.kubernetes.labels.app}
log_type: "goho.log"
environment: ${data.kubernetes.labels.environment}
instance: ${data.kubernetes.host}
# ---- 输出到 Elasticsearch ----
output.elasticsearch:
hosts: ["http://10.0.0.38:9200"]
username: "admin"
password: "G7ZSKFM4AQwHQpwA"
index: "k8s-%{[kubernetes.labels.environment]}-%{[kubernetes.labels.app]}-%{+yyyy.MM.dd}"
logging.level: debug
logging.selectors: ["*"]