dxin/Work-configuration-file
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

改改改

Browse Source
This commit is contained in:
dxinn
2026-01-09 17:50:32 +08:00
parent cf14d8a6db
commit 0384834345
37 changed files with 1944 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

73
OpenTelemetry/ES/filebast/01-filebeat-serviceaccount.yaml Normal file
View File

@@ -0,0 +1,73 @@
# 定义 Filebeat 的服务账户(ServiceAccount)
apiVersion: v1
kind: ServiceAccount
metadata:
name: filebeat # 服务账户名称
namespace: kube-system # 所在命名空间
labels:
k8s-app: filebeat # 标签，标识这是 Filebeat 应用
---
# 定义 Filebeat 的集群角色(ClusterRole)，授予集群范围的权限
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: filebeat # 集群角色名称
labels:
k8s-app: filebeat # 标签
rules:
# 授予对 namespaces, pods, nodes 资源的 get, list, watch 权限
- apiGroups: [""]
resources: ["namespaces", "pods", "nodes"]
verbs: ["get", "list", "watch"]
# 授予对 ReplicaSets 的 get, list, watch 权限
- apiGroups: ["apps"]
resources: ["replicasets"]
verbs: ["get", "list", "watch"]
# 授予对 Jobs 的 get, list, watch 权限
- apiGroups: ["batch"]
resources: ["jobs"]
verbs: ["get", "list", "watch"]
---
# 定义 Filebeat 的角色(Role)，授予命名空间范围的权限
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: filebeat # 角色名称
namespace: kube-system # 作用命名空间
labels:
k8s-app: filebeat # 标签
rules:
# 授予对 leases 资源的 get, create, update 权限
# Leases 用于协调和领导者选举
- apiGroups: ["coordination.k8s.io"]
resources: ["leases"]
verbs: ["get", "create", "update"]
---
# 将 Filebeat 的服务账户与集群角色绑定(ClusterRoleBinding)
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: filebeat # 绑定名称
subjects:
- kind: ServiceAccount # 主体类型为服务账户
name: filebeat # 服务账户名称
namespace: kube-system # 服务账户所在命名空间
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole # 引用的角色类型
name: filebeat # 引用的角色名称
---
# 将 Filebeat 的服务账户与角色绑定(RoleBinding)
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: filebeat # 绑定名称
namespace: kube-system # 作用命名空间
subjects:
- kind: ServiceAccount # 主体类型为服务账户
name: filebeat # 服务账户名称
namespace: kube-system # 服务账户所在命名空间
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role # 引用的角色类型
name: filebeat # 引用的角色名称

233
OpenTelemetry/ES/filebast/02-filebeat-configmap.yaml Normal file
View File

@@ -0,0 +1,233 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: filebeat-config
namespace: kube-system
data:
filebeat.yml: |
setup.ilm.enabled: false
setup.template.enabled: false
filebeat.autodiscover:
providers:
- type: kubernetes
templates:
# ---------- ↓ json格式日志 ↓ ----------
- condition:
and:
- regexp:
kubernetes.namespace: "^(sit|apex-evaluation)$"
- regexp:
kubernetes.labels.app: "^(lessie-go-api|apex)$"
config:
- type: filestream
id: "container-${data.kubernetes.container.id}"
prospector.scanner.symlinks: true
close.on_state_change.removed: false
parsers:
- container: ~
paths:
- /var/log/containers/*-${data.kubernetes.container.id}.log
processors:
- add_kubernetes_metadata:
host: ${NODE_NAME}
- decode_json_fields:
fields: ["message"]
target: "mylog"
overwrite_keys: true
add_error_key: true
- drop_fields:
fields:
- "kubernetes.node.labels"
- "kubernetes.namespace_labels.kubernetes_io/metadata_name"
ignore_missing: true
# ---------- ↑ json格式日志 ↑ ----------
# ---------- ↓ java语言的服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↓ ----------
- condition:
and:
- equals:
kubernetes.namespace: sit
- or:
- equals:
kubernetes.labels.app: "flymoon-admin"
- equals:
kubernetes.labels.app: "flymoon-agent"
- equals:
kubernetes.labels.app: "flymoon-payment"
config:
- type: filestream
id: "container-${data.kubernetes.container.id}"
prospector.scanner.symlinks: true
close.on_state_change.removed: false
parsers:
- container: ~
- multiline:
type: pattern
pattern: '^\d{4}-\d{2}-\d{2}-\d{2}:\d{2}:\d{2}\.\d{3}'
negate: true
match: after
paths:
- /var/log/containers/*-${data.kubernetes.container.id}.log
processors:
- add_kubernetes_metadata:
host: ${NODE_NAME}
- dissect:
tokenizer: '%{timestamp} %{level} %{pid} --- [%{thread}] %{class} : [%{app_name->}] %{message}'
field: "message"
target_prefix: "mylog"
ignore_missing: true
overwrite_keys: true
- drop_fields:
fields: ["kubernetes.node.labels", "kubernetes.annotations"]
ignore_missing: true
# ---------- ↑ java语言的服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↑ ----------
# ---------- ↓ java语言的服务的Pod, email 项目自由文本格式日志 ↓ ----------
- condition:
and:
- equals:
kubernetes.namespace: sit
- equals:
kubernetes.labels.app: "flymoon-email"
config:
- type: filestream
id: "container-${data.kubernetes.container.id}"
prospector.scanner.symlinks: true
close.on_state_change.removed: false
parsers:
- container: ~
- multiline:
type: pattern
pattern: '^\d{4}-\d{2}-\d{2}'
negate: true
match: after
paths:
- /var/log/containers/*-${data.kubernetes.container.id}.log
processors:
- add_kubernetes_metadata:
host: ${NODE_NAME}
- dissect:
tokenizer: '%{timestamp} %{level} %{pid} --- [%{thread}] %{class} : %{message}'
field: "message"
target_prefix: "mylog"
ignore_missing: true
overwrite_keys: true
- drop_fields:
fields: ["kubernetes.node.labels", "kubernetes.annotations"]
ignore_missing: true
# ---------- ↑ java语言的服务的Pod, email 项目自由文本格式日志 ↑ ----------
# ---------- ↓ python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↓ ----------
- condition:
and:
- equals:
kubernetes.namespace: sit
- equals:
kubernetes.labels.app: "lessie-agents"
config:
- type: filestream
id: "container-${data.kubernetes.container.id}"
prospector.scanner.symlinks: true
close.on_state_change.removed: false
parsers:
- container: ~
paths:
- /var/log/containers/*-${data.kubernetes.container.id}.log
processors:
- add_kubernetes_metadata:
host: ${NODE_NAME}
# 第一层：仅解析符合时间戳开头的日志行(for业务告警的日志格式)
- dissect:
when:
regexp:
message: '^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}.*'
tokenizer: '%{timestamp} - %{level} - %{module} - %{function} - %{msg_body}'
field: "message"
target_prefix: "mylog"
ignore_missing: true
overwrite_keys: true
# 第二层：针对带有 [level: | event: | msg: | context:] 的日志，再做一次 dissect
- dissect:
when:
contains:
mylog.msg_body: "[level:"
tokenizer: '[level: %{event_level} | event: %{event} | msg: %{msg} | context: %{ctx_raw}]'
field: "mylog.msg_body"
target_prefix: "mylog"
ignore_missing: true
overwrite_keys: true
# 第三层：把 ctx_raw 再拆成独立字段
- script:
lang: javascript
id: parse_context
source: >
function process(event) {
var ctx = event.Get("mylog.ctx_raw");
if (!ctx) return;
var parts = ctx.trim().split(",");
for (var i = 0; i < parts.length; i++) {
var pair = parts[i].split(":");
if (pair.length === 2) {
event.Put("mylog." + pair[0].trim(), pair[1].trim());
}
}
}
# 第四层: 去除大量不需要的k8s元数据字段
- drop_fields:
fields:
- "kubernetes.node.labels"
- "kubernetes.annotations"
ignore_missing: true
# ---------- ↑ python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↑ ----------
# ---------- ↓ apex 动态创建的 python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↓ ----------
- condition:
and:
- equals:
kubernetes.namespace: apex-evaluation
- equals:
kubernetes.labels.apex: "lessie-agents"
config:
- type: filestream
id: "container-${data.kubernetes.container.id}"
prospector.scanner.symlinks: true
close.on_state_change.removed: false
parsers:
- container: ~
paths:
- /var/log/containers/*-${data.kubernetes.container.id}.log
processors:
- drop_fields:
fields:
- "kubernetes.node.labels"
- "kubernetes.annotations"
ignore_missing: true
# ---------- ↑ apex 动态创建的 python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↑ ----------
# ---- 输出到 Elasticsearch ----
output.elasticsearch:
hosts: ["http://10.0.0.38:9200"]
username: "admin"
password: "G7ZSKFM4AQwHQpwA"
indices:
- index: "k8s-%{[kubernetes.labels.environment]}-%{[kubernetes.labels.app]}-%{+yyyy.MM}"
when:
regexp:
kubernetes.labels.app: "(lessie-go-api|flymoon-admin|flymoon-agent|flymoon-payment|flymoon-email|lessie-agents|apex)"
- index: "apex-python-%{+yyyy.MM}"
when:
equals:
kubernetes.labels.apex: "lessie-agents"
logging.level: info
logging.selectors: ["*"]

233
OpenTelemetry/ES/filebast/022-filebeat-configmap.yaml Normal file
View File

@@ -0,0 +1,233 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: filebeat-config
namespace: kube-system
data:
filebeat.yml: |
setup.ilm.enabled: false
setup.template.enabled: false
filebeat.autodiscover:
providers:
- type: kubernetes
templates:
# ---------- ↓ json格式日志 ↓ ----------
- condition:
and:
- regexp:
kubernetes.namespace: "^(sit|apex-evaluation)$"
- regexp:
kubernetes.labels.app: "^(lessie-go-api|apex)$"
config:
- type: filestream
id: "container-${data.kubernetes.container.id}"
prospector.scanner.symlinks: true
close.on_state_change.removed: false
parsers:
- container: ~
paths:
- /var/log/containers/*-${data.kubernetes.container.id}.log
processors:
- add_kubernetes_metadata:
host: ${NODE_NAME}
- decode_json_fields:
fields: ["message"]
target: "mylog"
overwrite_keys: true
add_error_key: true
- drop_fields:
fields:
- "kubernetes.node.labels"
- "kubernetes.namespace_labels.kubernetes_io/metadata_name"
ignore_missing: true
# ---------- ↑ json格式日志 ↑ ----------
# ---------- ↓ java语言的服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↓ ----------
- condition:
and:
- equals:
kubernetes.namespace: sit
- or:
- equals:
kubernetes.labels.app: "flymoon-admin"
- equals:
kubernetes.labels.app: "flymoon-agent"
- equals:
kubernetes.labels.app: "flymoon-payment"
config:
- type: filestream
id: "container-${data.kubernetes.container.id}"
prospector.scanner.symlinks: true
close.on_state_change.removed: false
parsers:
- container: ~
- multiline:
type: pattern
pattern: '^\d{4}-\d{2}-\d{2}-\d{2}:\d{2}:\d{2}\.\d{3}'
negate: true
match: after
paths:
- /var/log/containers/*-${data.kubernetes.container.id}.log
processors:
- add_kubernetes_metadata:
host: ${NODE_NAME}
- dissect:
tokenizer: '%{timestamp} %{level} %{pid} --- [%{thread}] %{class} : [%{app_name->}] %{message}'
field: "message"
target_prefix: "mylog"
ignore_missing: true
overwrite_keys: true
- drop_fields:
fields: ["kubernetes.node.labels", "kubernetes.annotations"]
ignore_missing: true
# ---------- ↑ java语言的服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↑ ----------
# ---------- ↓ java语言的服务的Pod, email 项目自由文本格式日志 ↓ ----------
- condition:
and:
- equals:
kubernetes.namespace: sit
- equals:
kubernetes.labels.app: "flymoon-email"
config:
- type: filestream
id: "container-${data.kubernetes.container.id}"
prospector.scanner.symlinks: true
close.on_state_change.removed: false
parsers:
- container: ~
- multiline:
type: pattern
pattern: '^\d{4}-\d{2}-\d{2}'
negate: true
match: after
paths:
- /var/log/containers/*-${data.kubernetes.container.id}.log
processors:
- add_kubernetes_metadata:
host: ${NODE_NAME}
- dissect:
tokenizer: '%{timestamp} %{level} %{pid} --- [%{thread}] %{class} : %{message}'
field: "message"
target_prefix: "mylog"
ignore_missing: true
overwrite_keys: true
- drop_fields:
fields: ["kubernetes.node.labels", "kubernetes.annotations"]
ignore_missing: true
# ---------- ↑ java语言的服务的Pod, email 项目自由文本格式日志 ↑ ----------
# ---------- ↓ python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↓ ----------
- condition:
and:
- equals:
kubernetes.namespace: sit
- equals:
kubernetes.labels.app: "lessie-agents"
config:
- type: filestream
id: "container-${data.kubernetes.container.id}"
prospector.scanner.symlinks: true
close.on_state_change.removed: false
parsers:
- container: ~
paths:
- /var/log/containers/*-${data.kubernetes.container.id}.log
processors:
- add_kubernetes_metadata:
host: ${NODE_NAME}
# 第一层：仅解析符合时间戳开头的日志行(for业务告警的日志格式)
- dissect:
when:
regexp:
message: '^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}.*'
tokenizer: '%{timestamp} - %{level} - %{module} - %{function} - %{msg_body}'
field: "message"
target_prefix: "mylog"
ignore_missing: true
overwrite_keys: true
# 第二层：针对带有 [level: | event: | msg: | context:] 的日志，再做一次 dissect
- dissect:
when:
contains:
mylog.msg_body: "[level:"
tokenizer: '[level: %{event_level} | event: %{event} | msg: %{msg} | context: %{ctx_raw}]'
field: "mylog.msg_body"
target_prefix: "mylog"
ignore_missing: true
overwrite_keys: true
# 第三层：把 ctx_raw 再拆成独立字段
- script:
lang: javascript
id: parse_context
source: >
function process(event) {
var ctx = event.Get("mylog.ctx_raw");
if (!ctx) return;
var parts = ctx.trim().split(",");
for (var i = 0; i < parts.length; i++) {
var pair = parts[i].split(":");
if (pair.length === 2) {
event.Put("mylog." + pair[0].trim(), pair[1].trim());
}
}
}
# 第四层: 去除大量不需要的k8s元数据字段
- drop_fields:
fields:
- "kubernetes.node.labels"
- "kubernetes.annotations"
ignore_missing: true
# ---------- ↑ python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↑ ----------
# ---------- ↓ apex 动态创建的 python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↓ ----------
- condition:
and:
- equals:
kubernetes.namespace: apex-evaluation
- equals:
kubernetes.labels.apex: "lessie-agents"
config:
- type: filestream
id: "container-${data.kubernetes.container.id}"
prospector.scanner.symlinks: true
close.on_state_change.removed: false
parsers:
- container: ~
paths:
- /var/log/containers/*-${data.kubernetes.container.id}.log
processors:
- drop_fields:
fields:
- "kubernetes.node.labels"
- "kubernetes.annotations"
ignore_missing: true
# ---------- ↑ apex 动态创建的 python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↑ ----------
# ---- 输出到 Elasticsearch ----
output.elasticsearch:
hosts: ["http://10.0.0.38:9200"]
username: "admin"
password: "G7ZSKFM4AQwHQpwA"
indices:
- index: "k8s-%{[kubernetes.labels.environment]}-%{[kubernetes.labels.app]}-%{+yyyy.MM}"
when:
regexp:
kubernetes.labels.app: "(lessie-go-api|flymoon-admin|flymoon-agent|flymoon-payment|flymoon-email|lessie-agents|apex)"
- index: "apex-python-%{+yyyy.MM}"
when:
equals:
kubernetes.labels.apex: "lessie-agents"
logging.level: info
logging.selectors: ["*"]

65
OpenTelemetry/ES/filebast/03-filebeat-daemonset.yaml Normal file
View File

@@ -0,0 +1,65 @@
# 滚动更新
# kubectl rollout restart daemonset filebeat -n kube-system
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: filebeat
namespace: kube-system
labels:
k8s-app: filebeat
spec:
selector:
matchLabels:
k8s-app: filebeat
template:
metadata:
labels:
k8s-app: filebeat
spec:
serviceAccountName: filebeat
terminationGracePeriodSeconds: 30
containers:
- name: filebeat
image: docker.elastic.co/beats/filebeat:9.2.2
args:
- "-e"
env:
- name: TZ
value: Asia/Shanghai
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
resources:
limits:
memory: 300Mi
requests:
cpu: 100m
memory: 200Mi
volumeMounts:
- name: config
mountPath: /usr/share/filebeat/filebeat.yml
subPath: filebeat.yml
- name: data
mountPath: /var/lib/filebeat-data
- name: containers
mountPath: /var/log/containers
readOnly: true
- name: pods
mountPath: /var/log/pods
readOnly: true
volumes:
- name: config
configMap:
name: filebeat-config
- name: data
hostPath:
path: /var/lib/filebeat-data
type: DirectoryOrCreate
- name: containers
hostPath:
path: /var/log/containers
- name: pods
hostPath:
path: /var/log/pods

226
OpenTelemetry/ES/filebast/filebeat.yaml Normal file
View File

@@ -0,0 +1,226 @@
setup.ilm.enabled: false
setup.template.enabled: false
filebeat.autodiscover:
providers:
- type: kubernetes
templates:
# ---------- ↓ json格式日志 ↓ ----------
- condition:
and:
- regexp:
kubernetes.namespace: "^(sit|apex-evaluation)$"
- regexp:
kubernetes.labels.app: "^(lessie-go-api|apex)$"
config:
- type: filestream
id: "container-${data.kubernetes.container.id}"
prospector.scanner.symlinks: true
close.on_state_change.removed: false
parsers:
- container: ~
paths:
- /var/log/containers/*-${data.kubernetes.container.id}.log
processors:
- add_kubernetes_metadata:
host: ${NODE_NAME}
- decode_json_fields:
fields: ["message"]
target: "mylog"
overwrite_keys: true
add_error_key: true
- drop_fields:
fields:
- "kubernetes.node.labels"
- "kubernetes.namespace_labels.kubernetes_io/metadata_name"
ignore_missing: true
# ---------- ↑ json格式日志 ↑ ----------
# ---------- ↓ java语言的服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↓ ----------
- condition:
and:
- equals:
kubernetes.namespace: sit
- or:
- equals:
kubernetes.labels.app: "flymoon-admin"
- equals:
kubernetes.labels.app: "flymoon-agent"
- equals:
kubernetes.labels.app: "flymoon-payment"
config:
- type: filestream
id: "container-${data.kubernetes.container.id}"
prospector.scanner.symlinks: true
close.on_state_change.removed: false
parsers:
- container: ~
- multiline:
type: pattern
pattern: '^\d{4}-\d{2}-\d{2}-\d{2}:\d{2}:\d{2}\.\d{3}'
negate: true
match: after
paths:
- /var/log/containers/*-${data.kubernetes.container.id}.log
processors:
- add_kubernetes_metadata:
host: ${NODE_NAME}
- dissect:
tokenizer: '%{timestamp} %{level} %{pid} --- [%{thread}] %{class} : [%{app_name->}] %{message}'
field: "message"
target_prefix: "mylog"
ignore_missing: true
overwrite_keys: true
- drop_fields:
fields: ["kubernetes.node.labels", "kubernetes.annotations"]
ignore_missing: true
# ---------- ↑ java语言的服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↑ ----------
# ---------- ↓ java语言的服务的Pod, email 项目自由文本格式日志 ↓ ----------
- condition:
and:
- equals:
kubernetes.namespace: sit
- equals:
kubernetes.labels.app: "flymoon-email"
config:
- type: filestream
id: "container-${data.kubernetes.container.id}"
prospector.scanner.symlinks: true
close.on_state_change.removed: false
parsers:
- container: ~
- multiline:
type: pattern
pattern: '^\d{4}-\d{2}-\d{2}'
negate: true
match: after
paths:
- /var/log/containers/*-${data.kubernetes.container.id}.log
processors:
- add_kubernetes_metadata:
host: ${NODE_NAME}
- dissect:
tokenizer: '%{timestamp} %{level} %{pid} --- [%{thread}] %{class} : %{message}'
field: "message"
target_prefix: "mylog"
ignore_missing: true
overwrite_keys: true
- drop_fields:
fields: ["kubernetes.node.labels", "kubernetes.annotations"]
ignore_missing: true
# ---------- ↑ java语言的服务的Pod, email 项目自由文本格式日志 ↑ ----------
# ---------- ↓ python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↓ ----------
- condition:
and:
- equals:
kubernetes.namespace: sit
- equals:
kubernetes.labels.app: "lessie-agents"
config:
- type: filestream
id: "container-${data.kubernetes.container.id}"
prospector.scanner.symlinks: true
close.on_state_change.removed: false
parsers:
- container: ~
paths:
- /var/log/containers/*-${data.kubernetes.container.id}.log
processors:
- add_kubernetes_metadata:
host: ${NODE_NAME}
# 第一层：仅解析符合时间戳开头的日志行(for业务告警的日志格式)
- dissect:
when:
regexp:
message: '^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}.*'
tokenizer: '%{timestamp} - %{level} - %{module} - %{function} - %{msg_body}'
field: "message"
target_prefix: "mylog"
ignore_missing: true
overwrite_keys: true
# 第二层：针对带有 [level: | event: | msg: | context:] 的日志，再做一次 dissect
- dissect:
when:
contains:
mylog.msg_body: "[level:"
tokenizer: '[level: %{event_level} | event: %{event} | msg: %{msg} | context: %{ctx_raw}]'
field: "mylog.msg_body"
target_prefix: "mylog"
ignore_missing: true
overwrite_keys: true
# 第三层：把 ctx_raw 再拆成独立字段
- script:
lang: javascript
id: parse_context
source: >
function process(event) {
var ctx = event.Get("mylog.ctx_raw");
if (!ctx) return;
var parts = ctx.trim().split(",");
for (var i = 0; i < parts.length; i++) {
var pair = parts[i].split(":");
if (pair.length === 2) {
event.Put("mylog." + pair[0].trim(), pair[1].trim());
}
}
}
# 第四层: 去除大量不需要的k8s元数据字段
- drop_fields:
fields:
- "kubernetes.node.labels"
- "kubernetes.annotations"
ignore_missing: true
# ---------- ↑ python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↑ ----------
# ---------- ↓ apex 动态创建的 python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↓ ----------
- condition:
and:
- equals:
kubernetes.namespace: apex-evaluation
- equals:
kubernetes.labels.apex: "lessie-agents"
config:
- type: filestream
id: "container-${data.kubernetes.container.id}"
prospector.scanner.symlinks: true
close.on_state_change.removed: false
parsers:
- container: ~
paths:
- /var/log/containers/*-${data.kubernetes.container.id}.log
processors:
- drop_fields:
fields:
- "kubernetes.node.labels"
- "kubernetes.annotations"
ignore_missing: true
# ---------- ↑ apex 动态创建的 python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↑ ----------
# ---- 输出到 Elasticsearch ----
output.elasticsearch:
hosts: ["http://10.0.0.38:9200"]
username: "admin"
password: "G7ZSKFM4AQwHQpwA"
indices:
- index: "k8s-%{[kubernetes.labels.environment]}-%{[kubernetes.labels.app]}-%{+yyyy.MM.dd}"
when:
regexp:
kubernetes.labels.app: "(lessie-go-api|flymoon-admin|flymoon-agent|flymoon-payment|flymoon-email|lessie-agents|apex)"
- index: "apex-python-%{+yyyy.MM.dd}"
when:
equals:
kubernetes.labels.apex: "lessie-agents"
logging.level: info
logging.selectors: ["*"]

143
OpenTelemetry/ES/单节点/安装es.conf Normal file
View File

@@ -0,0 +1,143 @@
# 前置 & 准备工作
sudo dnf update -y
sudo dnf install -y nano wget curl unzip
# 安全组防火墙开放9200端口、5601端口
# 安装 Elasticsearch 9.2.2
# 导入官方 GPG key
sudo rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch
# 新建 yum repo 文件
sudo tee /etc/yum.repos.d/elasticsearch.repo <<-'EOF'
[elasticsearch]
name=Elasticsearch repository for 9.x packages
baseurl=https://artifacts.elastic.co/packages/9.x/yum
gpgcheck=1
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=1
autorefresh=1
type=rpm-md
EOF
# 安装 Elasticsearch：
sudo dnf install elasticsearch --enablerepo=elasticsearch
# 先不管直接启动、报错再查看日志，有可能是权限问题
sudo systemctl daemon-reload
sudo systemctl enable elasticsearch
sudo systemctl start elasticsearch
sudo systemctl status elasticsearch
sudo journalctl -u elasticsearch -f
# 手动创建日志目录 + 设置权限
sudo mkdir -p /usr/share/elasticsearch/logs
sudo chown -R elasticsearch:elasticsearch /usr/share/elasticsearch/logs
sudo chmod 750 /usr/share/elasticsearch/logs
# 设置 elastic 超级用户密码 (推荐立即设定)：
sudo /usr/share/elasticsearch/bin/elasticsearch-reset-password -u elastic
# 查看自签名证书，有则正常
ll /etc/elasticsearch/certs/
# 查看 HTTP CA 证书指纹（用于其他客户端配置）
sudo openssl x509 -fingerprint -sha256 -in /etc/elasticsearch/certs/http_ca.crt -noout
# 设置环境变量（替换为你的实际密码）
export ELASTIC_PASSWORD='MyElastic123!'
# 测试 HTTPS 请求（必须用 --cacert，因启用了 TLS）
curl --cacert /etc/elasticsearch/certs/http_ca.crt \
-u elastic:$ELASTIC_PASSWORD \
https://localhost:9200
# 查看默认的配置文件
grep -v '^\s*#\|^\s*$' /etc/elasticsearch/elasticsearch.yml
# 按实际情况修改配置文件集群名、非本地访问等
cluster.name: my-test-es
path.data: /var/lib/elasticsearch
path.logs: /var/log/elasticsearch
network.host: 0.0.0.0
xpack.security.enabled: true
xpack.security.enrollment.enabled: true
xpack.security.http.ssl:
enabled: true
keystore.path: certs/http.p12
xpack.security.transport.ssl:
enabled: true
verification_mode: certificate
keystore.path: certs/transport.p12
truststore.path: certs/transport.p12
cluster.initial_master_nodes: ["weblessie-server-02"]
http.host: 0.0.0.0
# 更改es的jvm大小
vim /etc/elasticsearch/jvm.options
-Xms4g
-Xmx4g
# 重启
sudo systemctl restart elasticsearch
# 准备token，后续在Kibana中使用
sudo /usr/share/elasticsearch/bin/elasticsearch-create-enrollment-token -s kibana
# 准备安装 Kibana 9.2.2
# 新建 repo /etc/yum.repos.d/kibana.repo
sudo tee /etc/yum.repos.d/kibana.repo <<-'EOF'
[kibana]
name=Kibana repository for 9.x packages
baseurl=https://artifacts.elastic.co/packages/9.x/yum
gpgcheck=1
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=1
autorefresh=1
type=rpm-md
EOF
# 安装 Kibana：
sudo dnf install kibana --enablerepo=kibana
# 启动
sudo systemctl daemon-reload
sudo systemctl enable --now kibana
# 访问 Kibana，输入生成的token
http://ip:5601
# 获取 “verification code”
/usr/share/kibana/bin/kibana-verification-code
# 使用官方工具生成加密密钥（最规范）
sudo /usr/share/kibana/bin/kibana-encryption-keys generate --force
# 输出应类似：
# ✔ Encryption keys generated and written to /etc/kibana/kibana.yml:
# xpack.encryptedSavedObjects.encryptionKey
# xpack.reporting.encryptionKey
# xpack.security.encryptionKey
# 修改配置文件
grep -v '^\s*#\|^\s*$' /etc/kibana/kibana.yml
server.host: "0.0.0.0"
logging:
appenders:
file:
type: file
fileName: /var/log/kibana/kibana.log
layout:
type: json
root:
appenders:
- default
- file
pid.file: /run/kibana/kibana.pid
i18n.locale: "zh-CN"
elasticsearch.hosts: [https://10.0.0.38:9200]
elasticsearch.serviceAccountToken: AAEAAWVsYXN0aWMva2liYW5hL2Vucm9sbC1wcm9jZXNzLXRva2VuLTE3NjUzNDE4OTI3MjY6Um9KdUo2N1hSZVNPeGNzOXFDaUh2dw
elasticsearch.ssl.certificateAuthorities: [/var/lib/kibana/ca_1765341893683.crt]
xpack.fleet.outputs: [{id: fleet-default-output, name: default, is_default: true, is_default_monitoring: true, type: elasticsearch, hosts: [https://10.0.0.38:9200], ca_trusted_fingerprint: 80af64db043e12ebda11c10f70042af91306a705fdcb6285814a84b420c734a5}]
xpack.encryptedSavedObjects.encryptionKey: f10166c761265d5ca61e7fa2c1acac73
xpack.reporting.encryptionKey: 1772a5152522675d5a38470e905b2817
xpack.security.encryptionKey: d4b30e82e47f530a998e29cb0b8e5295

41
OpenTelemetry/ES/单节点/证书使用示例.conf Normal file
View File

@@ -0,0 +1,41 @@
# 获取ES 的证书指纹
sudo openssl x509 -fingerprint -sha256 -in /etc/elasticsearch/certs/http_ca.crt -noout
sha256 Fingerprint=80:AF:64:DB:04:3E:12:EB:DA:11:C1:0F:70:04:2A:F9:13:06:A7:05:FD:CB:62:85:81:4A:84:B4:20:C7:34:A5
# kibana web创建的用户
admin
G7ZSKFM4AQwHQpwA
# Filebeat
output.elasticsearch:
hosts: ["https://49.51.33.153:9200"]
username: "elastic"
password: "-0NiIBOJGn2CATuPWzNc"
# 用指纹验证（代替证书文件）
ssl.verification_mode: "certificate"
ssl.certificate_authorities: [] # 留空（不校验完整链）
ssl.supported_protocols: [TLSv1.2, TLSv1.3]
# 关键：指定 CA 指纹（必须全大写，无 0x，带冒号）
ssl.ca_trusted_fingerprint: "80AF64DB043E12EBDA11C10F70042AF91306A705FD2CB6285814A84B420C734A5"
# python
from elasticsearch import Elasticsearch
es = Elasticsearch(
hosts=["https://49.51.33.153:9200"],
basic_auth=("elastic", "-0NiIBOJGn2CATuPWzNc"),
# 指纹必须去掉冒号，全大写
ssl_assert_fingerprint="80AF64DB043E12EBDA11C10F70042AF91306A705FD2CB6285814A84B420C734A5",
verify_certs=True # 必须为 True
)
print(es.info())