From f71063450b31fd38e9951783e92dd8f354d35563 Mon Sep 17 00:00:00 2001 From: dxinn <1554389441@qq.com> Date: Fri, 9 Jan 2026 17:52:16 +0800 Subject: [PATCH] =?UTF-8?q?=E6=94=B9=E6=94=B9=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- SCM/部署镜像/apex/DM_prod_apex.groovy | 67 +++-- SCM/部署镜像/apex/DM_prod_apex_web.groovy | 67 +++-- SCM/部署镜像/apex/DM_test_apex.groovy | 67 +++-- SCM/部署镜像/apex/DM_test_apex_web.groovy | 67 +++-- .../ELK/filebast/02-filebeat-configmap.yaml | 100 ++++++-- .../ELK/filebast/022-filebeat-configmap.yaml | 100 ++++++-- .../ELK/filebast/023-filebeat-configmap.yaml | 233 ------------------ .../ELK/filebast/03-filebeat-daemonset.yaml | 3 + k8s_yaml/ELK/filebast/filebeat.yaml | 216 ++++++++++++++-- 9 files changed, 537 insertions(+), 383 deletions(-) delete mode 100644 k8s_yaml/ELK/filebast/023-filebeat-configmap.yaml diff --git a/SCM/部署镜像/apex/DM_prod_apex.groovy b/SCM/部署镜像/apex/DM_prod_apex.groovy index 3721d3b..880180b 100644 --- a/SCM/部署镜像/apex/DM_prod_apex.groovy +++ b/SCM/部署镜像/apex/DM_prod_apex.groovy @@ -243,40 +243,55 @@ jenkins执行人: ${env.ACTUAL_USER} when { expression { return params.ROLLBACK_VERSION == false } } steps { script { - withCredentials([usernamePassword(credentialsId: 'fly_gitlab_auth', usernameVariable: 'GIT_USER', passwordVariable: 'GIT_PASS')]) { + withCredentials([usernamePassword( + credentialsId: 'fly_gitlab_auth', + usernameVariable: 'GIT_USER', + passwordVariable: 'GIT_PASS' + )]) { sh """ - cd ${WORKSPACE} - git config user.name "jenkins" - git config user.email "jenkins@local" + set -e + + cd ${WORKSPACE} + git config user.name "jenkins" + git config user.email "jenkins@local" + + git checkout ${params.BRANCH_NAME} + + echo "machine 172.24.16.20 login ${GIT_USER} password ${GIT_PASS}" > ~/.netrc + chmod 600 ~/.netrc + + MAX_RETRY=3 + i=1 + + while [ \$i -le \$MAX_RETRY ]; do + echo "🔁 尝试第 \$i 次提交..." + + git fetch http://172.24.16.20/opt/opt-config.git ${params.BRANCH_NAME} - # 检查工作树是否有变化 if ! git diff --exit-code ${Deployment_yaml} > /dev/null 2>&1; then - echo "检测到更改，正在提交..." git add ${Deployment_yaml} - git commit -m "更新镜像 \${CHANGE_MSG}" + git commit -m "更新镜像 ${CHANGE_MSG}" || true else - echo "${Deployment_yaml} 没有变化, 无需commit" + echo "文件无变化，跳过 commit" fi - # 检查是否需要推送（是否有新的提交） - LOCAL=\$(git rev-parse @) - REMOTE=\$(git rev-parse @{u} 2>/dev/null || true) - BASE=\$(git merge-base @ @{u} 2>/dev/null || true) - - if [ "\$LOCAL" = "\$REMOTE" ]; then - echo "已与远程系统同步更新" - elif [ "\$LOCAL" = "\$BASE" ]; then - echo "需要从远程获取数据" - git pull http://172.24.16.20/opt/opt-config.git ${params.BRANCH_NAME} - elif [ "\$REMOTE" = "\$BASE" ]; then - echo "将更改推送到远程服务器..." - # 生成临时 .netrc 文件 - echo "machine 172.24.16.20 login \$GIT_USER password \$GIT_PASS" > ~/.netrc - chmod 600 ~/.netrc - git push http://172.24.16.20/opt/opt-config.git ${params.BRANCH_NAME} - else - echo "与远程模式不同，跳过推送操作" + if git rebase FETCH_HEAD; then + if git push http://172.24.16.20/opt/opt-config.git ${params.BRANCH_NAME}; then + echo "✅ 推送成功" + exit 0 + fi fi + + echo "⚠️ 推送失败，回滚并重试..." + git rebase --abort || true + git reset --hard FETCH_HEAD + + i=\$((i+1)) + sleep 2 + done + + echo "❌ 多次尝试失败，可能存在并发冲突" + exit 1 """ } } diff --git a/SCM/部署镜像/apex/DM_prod_apex_web.groovy b/SCM/部署镜像/apex/DM_prod_apex_web.groovy index 0e1cf0c..90d9bcf 100644 --- a/SCM/部署镜像/apex/DM_prod_apex_web.groovy +++ b/SCM/部署镜像/apex/DM_prod_apex_web.groovy @@ -243,40 +243,55 @@ jenkins执行人: ${env.ACTUAL_USER} when { expression { return params.ROLLBACK_VERSION == false } } steps { script { - withCredentials([usernamePassword(credentialsId: 'fly_gitlab_auth', usernameVariable: 'GIT_USER', passwordVariable: 'GIT_PASS')]) { + withCredentials([usernamePassword( + credentialsId: 'fly_gitlab_auth', + usernameVariable: 'GIT_USER', + passwordVariable: 'GIT_PASS' + )]) { sh """ - cd ${WORKSPACE} - git config user.name "jenkins" - git config user.email "jenkins@local" + set -e + + cd ${WORKSPACE} + git config user.name "jenkins" + git config user.email "jenkins@local" + + git checkout ${params.BRANCH_NAME} + + echo "machine 172.24.16.20 login ${GIT_USER} password ${GIT_PASS}" > ~/.netrc + chmod 600 ~/.netrc + + MAX_RETRY=3 + i=1 + + while [ \$i -le \$MAX_RETRY ]; do + echo "🔁 尝试第 \$i 次提交..." + + git fetch http://172.24.16.20/opt/opt-config.git ${params.BRANCH_NAME} - # 检查工作树是否有变化 if ! git diff --exit-code ${Deployment_yaml} > /dev/null 2>&1; then - echo "检测到更改，正在提交..." git add ${Deployment_yaml} - git commit -m "更新镜像 \${CHANGE_MSG}" + git commit -m "更新镜像 \${CHANGE_MSG}" || true else - echo "${Deployment_yaml} 没有变化, 无需commit" + echo "文件无变化，跳过 commit" fi - # 检查是否需要推送（是否有新的提交） - LOCAL=\$(git rev-parse @) - REMOTE=\$(git rev-parse @{u} 2>/dev/null || true) - BASE=\$(git merge-base @ @{u} 2>/dev/null || true) - - if [ "\$LOCAL" = "\$REMOTE" ]; then - echo "已与远程系统同步更新" - elif [ "\$LOCAL" = "\$BASE" ]; then - echo "需要从远程获取数据" - git pull http://172.24.16.20/opt/opt-config.git ${params.BRANCH_NAME} - elif [ "\$REMOTE" = "\$BASE" ]; then - echo "将更改推送到远程服务器..." - # 生成临时 .netrc 文件 - echo "machine 172.24.16.20 login \$GIT_USER password \$GIT_PASS" > ~/.netrc - chmod 600 ~/.netrc - git push http://172.24.16.20/opt/opt-config.git ${params.BRANCH_NAME} - else - echo "与远程模式不同，跳过推送操作" + if git rebase FETCH_HEAD; then + if git push http://172.24.16.20/opt/opt-config.git ${params.BRANCH_NAME}; then + echo "✅ 推送成功" + exit 0 + fi fi + + echo "⚠️ 推送失败，回滚并重试..." + git rebase --abort || true + git reset --hard FETCH_HEAD + + i=\$((i+1)) + sleep 2 + done + + echo "❌ 多次尝试失败，可能存在并发冲突" + exit 1 """ } } diff --git a/SCM/部署镜像/apex/DM_test_apex.groovy b/SCM/部署镜像/apex/DM_test_apex.groovy index 8df693d..34bd937 100644 --- a/SCM/部署镜像/apex/DM_test_apex.groovy +++ b/SCM/部署镜像/apex/DM_test_apex.groovy @@ -243,40 +243,55 @@ jenkins执行人: ${env.ACTUAL_USER} when { expression { return params.ROLLBACK_VERSION == false } } steps { script { - withCredentials([usernamePassword(credentialsId: 'fly_gitlab_auth', usernameVariable: 'GIT_USER', passwordVariable: 'GIT_PASS')]) { + withCredentials([usernamePassword( + credentialsId: 'fly_gitlab_auth', + usernameVariable: 'GIT_USER', + passwordVariable: 'GIT_PASS' + )]) { sh """ - cd ${WORKSPACE} - git config user.name "jenkins" - git config user.email "jenkins@local" + set -e + + cd ${WORKSPACE} + git config user.name "jenkins" + git config user.email "jenkins@local" + + git checkout ${params.BRANCH_NAME} + + echo "machine 172.24.16.20 login ${GIT_USER} password ${GIT_PASS}" > ~/.netrc + chmod 600 ~/.netrc + + MAX_RETRY=3 + i=1 + + while [ \$i -le \$MAX_RETRY ]; do + echo "🔁 尝试第 \$i 次提交..." + + git fetch http://172.24.16.20/opt/opt-config.git ${params.BRANCH_NAME} - # 检查工作树是否有变化 if ! git diff --exit-code ${Deployment_yaml} > /dev/null 2>&1; then - echo "检测到更改，正在提交..." git add ${Deployment_yaml} - git commit -m "更新镜像 \${CHANGE_MSG}" + git commit -m "更新镜像 ${CHANGE_MSG}" || true else - echo "${Deployment_yaml} 没有变化, 无需commit" + echo "文件无变化，跳过 commit" fi - # 检查是否需要推送（是否有新的提交） - LOCAL=\$(git rev-parse @) - REMOTE=\$(git rev-parse @{u} 2>/dev/null || true) - BASE=\$(git merge-base @ @{u} 2>/dev/null || true) - - if [ "\$LOCAL" = "\$REMOTE" ]; then - echo "已与远程系统同步更新" - elif [ "\$LOCAL" = "\$BASE" ]; then - echo "需要从远程获取数据" - git pull http://172.24.16.20/opt/opt-config.git ${params.BRANCH_NAME} - elif [ "\$REMOTE" = "\$BASE" ]; then - echo "将更改推送到远程服务器..." - # 生成临时 .netrc 文件 - echo "machine 172.24.16.20 login \$GIT_USER password \$GIT_PASS" > ~/.netrc - chmod 600 ~/.netrc - git push http://172.24.16.20/opt/opt-config.git ${params.BRANCH_NAME} - else - echo "与远程模式不同，跳过推送操作" + if git rebase FETCH_HEAD; then + if git push http://172.24.16.20/opt/opt-config.git ${params.BRANCH_NAME}; then + echo "✅ 推送成功" + exit 0 + fi fi + + echo "⚠️ 推送失败，回滚并重试..." + git rebase --abort || true + git reset --hard FETCH_HEAD + + i=\$((i+1)) + sleep 2 + done + + echo "❌ 多次尝试失败，可能存在并发冲突" + exit 1 """ } } diff --git a/SCM/部署镜像/apex/DM_test_apex_web.groovy b/SCM/部署镜像/apex/DM_test_apex_web.groovy index 0e603aa..e48b2f6 100644 --- a/SCM/部署镜像/apex/DM_test_apex_web.groovy +++ b/SCM/部署镜像/apex/DM_test_apex_web.groovy @@ -243,40 +243,55 @@ jenkins执行人: ${env.ACTUAL_USER} when { expression { return params.ROLLBACK_VERSION == false } } steps { script { - withCredentials([usernamePassword(credentialsId: 'fly_gitlab_auth', usernameVariable: 'GIT_USER', passwordVariable: 'GIT_PASS')]) { + withCredentials([usernamePassword( + credentialsId: 'fly_gitlab_auth', + usernameVariable: 'GIT_USER', + passwordVariable: 'GIT_PASS' + )]) { sh """ - cd ${WORKSPACE} - git config user.name "jenkins" - git config user.email "jenkins@local" + set -e + + cd ${WORKSPACE} + git config user.name "jenkins" + git config user.email "jenkins@local" + + git checkout ${params.BRANCH_NAME} + + echo "machine 172.24.16.20 login ${GIT_USER} password ${GIT_PASS}" > ~/.netrc + chmod 600 ~/.netrc + + MAX_RETRY=3 + i=1 + + while [ \$i -le \$MAX_RETRY ]; do + echo "🔁 尝试第 \$i 次提交..." + + git fetch http://172.24.16.20/opt/opt-config.git ${params.BRANCH_NAME} - # 检查工作树是否有变化 if ! git diff --exit-code ${Deployment_yaml} > /dev/null 2>&1; then - echo "检测到更改，正在提交..." git add ${Deployment_yaml} - git commit -m "更新镜像 \${CHANGE_MSG}" + git commit -m "更新镜像 ${CHANGE_MSG}" || true else - echo "${Deployment_yaml} 没有变化, 无需commit" + echo "文件无变化，跳过 commit" fi - # 检查是否需要推送（是否有新的提交） - LOCAL=\$(git rev-parse @) - REMOTE=\$(git rev-parse @{u} 2>/dev/null || true) - BASE=\$(git merge-base @ @{u} 2>/dev/null || true) - - if [ "\$LOCAL" = "\$REMOTE" ]; then - echo "已与远程系统同步更新" - elif [ "\$LOCAL" = "\$BASE" ]; then - echo "需要从远程获取数据" - git pull http://172.24.16.20/opt/opt-config.git ${params.BRANCH_NAME} - elif [ "\$REMOTE" = "\$BASE" ]; then - echo "将更改推送到远程服务器..." - # 生成临时 .netrc 文件 - echo "machine 172.24.16.20 login \$GIT_USER password \$GIT_PASS" > ~/.netrc - chmod 600 ~/.netrc - git push http://172.24.16.20/opt/opt-config.git ${params.BRANCH_NAME} - else - echo "与远程模式不同，跳过推送操作" + if git rebase FETCH_HEAD; then + if git push http://172.24.16.20/opt/opt-config.git ${params.BRANCH_NAME}; then + echo "✅ 推送成功" + exit 0 + fi fi + + echo "⚠️ 推送失败，回滚并重试..." + git rebase --abort || true + git reset --hard FETCH_HEAD + + i=\$((i+1)) + sleep 2 + done + + echo "❌ 多次尝试失败，可能存在并发冲突" + exit 1 """ } } diff --git a/k8s_yaml/ELK/filebast/02-filebeat-configmap.yaml b/k8s_yaml/ELK/filebast/02-filebeat-configmap.yaml index 0ce9be0..6ea96f4 100644 --- a/k8s_yaml/ELK/filebast/02-filebeat-configmap.yaml +++ b/k8s_yaml/ELK/filebast/02-filebeat-configmap.yaml @@ -12,13 +12,13 @@ data: providers: - type: kubernetes templates: - # ---------- ↓ go语言的中转服务的Pod, go项目json格式日志 ↓ ---------- + # ---------- ↓ json格式日志 ↓ ---------- - condition: and: - - equals: - kubernetes.namespace: sit - - equals: - kubernetes.labels.app: "lessie-go-api" + - regexp: + kubernetes.namespace: "^(sit|apex-evaluation)$" + - regexp: + kubernetes.labels.app: "^(lessie-go-api|apex)$" config: - type: filestream id: "container-${data.kubernetes.container.id}" @@ -28,7 +28,6 @@ data: - container: ~ paths: - /var/log/containers/*-${data.kubernetes.container.id}.log - processors: - add_kubernetes_metadata: host: ${NODE_NAME} @@ -42,10 +41,10 @@ data: - "kubernetes.node.labels" - "kubernetes.namespace_labels.kubernetes_io/metadata_name" ignore_missing: true - - # ---------- ↑ go语言的中转服务的Pod, go项目json格式日志 ↑ ---------- + # ---------- ↑ json格式日志 ↑ ---------- - # ---------- ↓ java语言的中转服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↓ ---------- + + # ---------- ↓ java语言的服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↓ ---------- - condition: and: - equals: @@ -71,7 +70,6 @@ data: match: after paths: - /var/log/containers/*-${data.kubernetes.container.id}.log - processors: - add_kubernetes_metadata: host: ${NODE_NAME} @@ -85,7 +83,44 @@ data: fields: ["kubernetes.node.labels", "kubernetes.annotations"] ignore_missing: true - # ---------- ↑ java语言的中转服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↑ ---------- + # ---------- ↑ java语言的服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↑ ---------- + + + # ---------- ↓ java语言的服务的Pod, email 项目自由文本格式日志 ↓ ---------- + - condition: + and: + - equals: + kubernetes.namespace: sit + - equals: + kubernetes.labels.app: "flymoon-email" + config: + - type: filestream + id: "container-${data.kubernetes.container.id}" + prospector.scanner.symlinks: true + close.on_state_change.removed: false + parsers: + - container: ~ + - multiline: + type: pattern + pattern: '^\d{4}-\d{2}-\d{2}' + negate: true + match: after + paths: + - /var/log/containers/*-${data.kubernetes.container.id}.log + processors: + - add_kubernetes_metadata: + host: ${NODE_NAME} + - dissect: + tokenizer: '%{timestamp} %{level} %{pid} --- [%{thread}] %{class} : %{message}' + field: "message" + target_prefix: "mylog" + ignore_missing: true + overwrite_keys: true + - drop_fields: + fields: ["kubernetes.node.labels", "kubernetes.annotations"] + ignore_missing: true + # ---------- ↑ java语言的服务的Pod, email 项目自由文本格式日志 ↑ ---------- + # ---------- ↓ python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↓ ---------- - condition: @@ -103,7 +138,6 @@ data: - container: ~ paths: - /var/log/containers/*-${data.kubernetes.container.id}.log - processors: - add_kubernetes_metadata: host: ${NODE_NAME} @@ -149,17 +183,51 @@ data: - "kubernetes.node.labels" - "kubernetes.annotations" ignore_missing: true - # ---------- ↑ python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↑ ---------- + # ---------- ↓ apex 动态创建的 python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↓ ---------- + - condition: + and: + - equals: + kubernetes.namespace: apex-evaluation + - equals: + kubernetes.labels.apex: "lessie-agents" + config: + - type: filestream + id: "container-${data.kubernetes.container.id}" + prospector.scanner.symlinks: true + close.on_state_change.removed: false + parsers: + - container: ~ + paths: + - /var/log/containers/*-${data.kubernetes.container.id}.log + processors: + - drop_fields: + fields: + - "kubernetes.node.labels" + - "kubernetes.annotations" + ignore_missing: true + # ---------- ↑ apex 动态创建的 python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↑ ---------- + + + # ---- 输出到 Elasticsearch ---- output.elasticsearch: hosts: ["http://10.0.0.38:9200"] username: "admin" password: "G7ZSKFM4AQwHQpwA" - index: "k8s-%{[kubernetes.labels.environment]}-%{[kubernetes.labels.app]}-%{+yyyy.MM.dd}" - # index: "k8s-%{[kubernetes.labels.app]}-%{+yyyy.MM.dd}" - logging.level: debug + indices: + - index: "k8s-%{[kubernetes.labels.environment]}-%{[kubernetes.labels.app]}-%{+yyyy.MM}" + when: + regexp: + kubernetes.labels.app: "(lessie-go-api|flymoon-admin|flymoon-agent|flymoon-payment|flymoon-email|lessie-agents|apex)" + + - index: "apex-python-%{+yyyy.MM}" + when: + equals: + kubernetes.labels.apex: "lessie-agents" + + logging.level: info logging.selectors: ["*"] \ No newline at end of file diff --git a/k8s_yaml/ELK/filebast/022-filebeat-configmap.yaml b/k8s_yaml/ELK/filebast/022-filebeat-configmap.yaml index 0ce9be0..6ea96f4 100644 --- a/k8s_yaml/ELK/filebast/022-filebeat-configmap.yaml +++ b/k8s_yaml/ELK/filebast/022-filebeat-configmap.yaml @@ -12,13 +12,13 @@ data: providers: - type: kubernetes templates: - # ---------- ↓ go语言的中转服务的Pod, go项目json格式日志 ↓ ---------- + # ---------- ↓ json格式日志 ↓ ---------- - condition: and: - - equals: - kubernetes.namespace: sit - - equals: - kubernetes.labels.app: "lessie-go-api" + - regexp: + kubernetes.namespace: "^(sit|apex-evaluation)$" + - regexp: + kubernetes.labels.app: "^(lessie-go-api|apex)$" config: - type: filestream id: "container-${data.kubernetes.container.id}" @@ -28,7 +28,6 @@ data: - container: ~ paths: - /var/log/containers/*-${data.kubernetes.container.id}.log - processors: - add_kubernetes_metadata: host: ${NODE_NAME} @@ -42,10 +41,10 @@ data: - "kubernetes.node.labels" - "kubernetes.namespace_labels.kubernetes_io/metadata_name" ignore_missing: true - - # ---------- ↑ go语言的中转服务的Pod, go项目json格式日志 ↑ ---------- + # ---------- ↑ json格式日志 ↑ ---------- - # ---------- ↓ java语言的中转服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↓ ---------- + + # ---------- ↓ java语言的服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↓ ---------- - condition: and: - equals: @@ -71,7 +70,6 @@ data: match: after paths: - /var/log/containers/*-${data.kubernetes.container.id}.log - processors: - add_kubernetes_metadata: host: ${NODE_NAME} @@ -85,7 +83,44 @@ data: fields: ["kubernetes.node.labels", "kubernetes.annotations"] ignore_missing: true - # ---------- ↑ java语言的中转服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↑ ---------- + # ---------- ↑ java语言的服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↑ ---------- + + + # ---------- ↓ java语言的服务的Pod, email 项目自由文本格式日志 ↓ ---------- + - condition: + and: + - equals: + kubernetes.namespace: sit + - equals: + kubernetes.labels.app: "flymoon-email" + config: + - type: filestream + id: "container-${data.kubernetes.container.id}" + prospector.scanner.symlinks: true + close.on_state_change.removed: false + parsers: + - container: ~ + - multiline: + type: pattern + pattern: '^\d{4}-\d{2}-\d{2}' + negate: true + match: after + paths: + - /var/log/containers/*-${data.kubernetes.container.id}.log + processors: + - add_kubernetes_metadata: + host: ${NODE_NAME} + - dissect: + tokenizer: '%{timestamp} %{level} %{pid} --- [%{thread}] %{class} : %{message}' + field: "message" + target_prefix: "mylog" + ignore_missing: true + overwrite_keys: true + - drop_fields: + fields: ["kubernetes.node.labels", "kubernetes.annotations"] + ignore_missing: true + # ---------- ↑ java语言的服务的Pod, email 项目自由文本格式日志 ↑ ---------- + # ---------- ↓ python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↓ ---------- - condition: @@ -103,7 +138,6 @@ data: - container: ~ paths: - /var/log/containers/*-${data.kubernetes.container.id}.log - processors: - add_kubernetes_metadata: host: ${NODE_NAME} @@ -149,17 +183,51 @@ data: - "kubernetes.node.labels" - "kubernetes.annotations" ignore_missing: true - # ---------- ↑ python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↑ ---------- + # ---------- ↓ apex 动态创建的 python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↓ ---------- + - condition: + and: + - equals: + kubernetes.namespace: apex-evaluation + - equals: + kubernetes.labels.apex: "lessie-agents" + config: + - type: filestream + id: "container-${data.kubernetes.container.id}" + prospector.scanner.symlinks: true + close.on_state_change.removed: false + parsers: + - container: ~ + paths: + - /var/log/containers/*-${data.kubernetes.container.id}.log + processors: + - drop_fields: + fields: + - "kubernetes.node.labels" + - "kubernetes.annotations" + ignore_missing: true + # ---------- ↑ apex 动态创建的 python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↑ ---------- + + + # ---- 输出到 Elasticsearch ---- output.elasticsearch: hosts: ["http://10.0.0.38:9200"] username: "admin" password: "G7ZSKFM4AQwHQpwA" - index: "k8s-%{[kubernetes.labels.environment]}-%{[kubernetes.labels.app]}-%{+yyyy.MM.dd}" - # index: "k8s-%{[kubernetes.labels.app]}-%{+yyyy.MM.dd}" - logging.level: debug + indices: + - index: "k8s-%{[kubernetes.labels.environment]}-%{[kubernetes.labels.app]}-%{+yyyy.MM}" + when: + regexp: + kubernetes.labels.app: "(lessie-go-api|flymoon-admin|flymoon-agent|flymoon-payment|flymoon-email|lessie-agents|apex)" + + - index: "apex-python-%{+yyyy.MM}" + when: + equals: + kubernetes.labels.apex: "lessie-agents" + + logging.level: info logging.selectors: ["*"] \ No newline at end of file diff --git a/k8s_yaml/ELK/filebast/023-filebeat-configmap.yaml b/k8s_yaml/ELK/filebast/023-filebeat-configmap.yaml deleted file mode 100644 index e6eeb60..0000000 --- a/k8s_yaml/ELK/filebast/023-filebeat-configmap.yaml +++ /dev/null @@ -1,233 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: filebeat-config - namespace: kube-system -data: - filebeat.yml: | - setup.ilm.enabled: false - setup.template.enabled: false - - filebeat.autodiscover: - providers: - - type: kubernetes - templates: - # ---------- ↓ json格式日志 ↓ ---------- - - condition: - and: - - regexp: - kubernetes.namespace: "^(sit|apex-evaluation)$" - - regexp: - kubernetes.labels.app: "^(lessie-go-api|apex)$" - config: - - type: filestream - id: "container-${data.kubernetes.container.id}" - prospector.scanner.symlinks: true - close.on_state_change.removed: false - parsers: - - container: ~ - paths: - - /var/log/containers/*-${data.kubernetes.container.id}.log - processors: - - add_kubernetes_metadata: - host: ${NODE_NAME} - - decode_json_fields: - fields: ["message"] - target: "mylog" - overwrite_keys: true - add_error_key: true - - drop_fields: - fields: - - "kubernetes.node.labels" - - "kubernetes.namespace_labels.kubernetes_io/metadata_name" - ignore_missing: true - # ---------- ↑ json格式日志 ↑ ---------- - - - # ---------- ↓ java语言的服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↓ ---------- - - condition: - and: - - equals: - kubernetes.namespace: sit - - or: - - equals: - kubernetes.labels.app: "flymoon-admin" - - equals: - kubernetes.labels.app: "flymoon-agent" - - equals: - kubernetes.labels.app: "flymoon-payment" - config: - - type: filestream - id: "container-${data.kubernetes.container.id}" - prospector.scanner.symlinks: true - close.on_state_change.removed: false - parsers: - - container: ~ - - multiline: - type: pattern - pattern: '^\d{4}-\d{2}-\d{2}-\d{2}:\d{2}:\d{2}\.\d{3}' - negate: true - match: after - paths: - - /var/log/containers/*-${data.kubernetes.container.id}.log - processors: - - add_kubernetes_metadata: - host: ${NODE_NAME} - - dissect: - tokenizer: '%{timestamp} %{level} %{pid} --- [%{thread}] %{class} : [%{app_name->}] %{message}' - field: "message" - target_prefix: "mylog" - ignore_missing: true - overwrite_keys: true - - drop_fields: - fields: ["kubernetes.node.labels", "kubernetes.annotations"] - ignore_missing: true - - # ---------- ↑ java语言的服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↑ ---------- - - - # ---------- ↓ java语言的服务的Pod, email 项目自由文本格式日志 ↓ ---------- - - condition: - and: - - equals: - kubernetes.namespace: sit - - equals: - kubernetes.labels.app: "flymoon-email" - config: - - type: filestream - id: "container-${data.kubernetes.container.id}" - prospector.scanner.symlinks: true - close.on_state_change.removed: false - parsers: - - container: ~ - - multiline: - type: pattern - pattern: '^\d{4}-\d{2}-\d{2}' - negate: true - match: after - paths: - - /var/log/containers/*-${data.kubernetes.container.id}.log - processors: - - add_kubernetes_metadata: - host: ${NODE_NAME} - - dissect: - tokenizer: '%{timestamp} %{level} %{pid} --- [%{thread}] %{class} : %{message}' - field: "message" - target_prefix: "mylog" - ignore_missing: true - overwrite_keys: true - - drop_fields: - fields: ["kubernetes.node.labels", "kubernetes.annotations"] - ignore_missing: true - # ---------- ↑ java语言的服务的Pod, email 项目自由文本格式日志 ↑ ---------- - - - # ---------- ↓ python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↓ ---------- - - condition: - and: - - equals: - kubernetes.namespace: sit - - equals: - kubernetes.labels.app: "lessie-agents" - config: - - type: filestream - id: "container-${data.kubernetes.container.id}" - prospector.scanner.symlinks: true - close.on_state_change.removed: false - parsers: - - container: ~ - paths: - - /var/log/containers/*-${data.kubernetes.container.id}.log - processors: - - add_kubernetes_metadata: - host: ${NODE_NAME} - # 第一层：仅解析符合时间戳开头的日志行(for业务告警的日志格式) - - dissect: - when: - regexp: - message: '^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}.*' - tokenizer: '%{timestamp} - %{level} - %{module} - %{function} - %{msg_body}' - field: "message" - target_prefix: "mylog" - ignore_missing: true - overwrite_keys: true - # 第二层：针对带有 [level: | event: | msg: | context:] 的日志，再做一次 dissect - - dissect: - when: - contains: - mylog.msg_body: "[level:" - tokenizer: '[level: %{event_level} | event: %{event} | msg: %{msg} | context: %{ctx_raw}]' - field: "mylog.msg_body" - target_prefix: "mylog" - ignore_missing: true - overwrite_keys: true - # 第三层：把 ctx_raw 再拆成独立字段 - - script: - lang: javascript - id: parse_context - source: > - function process(event) { - var ctx = event.Get("mylog.ctx_raw"); - if (!ctx) return; - var parts = ctx.trim().split(","); - for (var i = 0; i < parts.length; i++) { - var pair = parts[i].split(":"); - if (pair.length === 2) { - event.Put("mylog." + pair[0].trim(), pair[1].trim()); - } - } - } - # 第四层: 去除大量不需要的k8s元数据字段 - - drop_fields: - fields: - - "kubernetes.node.labels" - - "kubernetes.annotations" - ignore_missing: true - # ---------- ↑ python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↑ ---------- - - - # ---------- ↓ apex 动态创建的 python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↓ ---------- - - condition: - and: - - equals: - kubernetes.namespace: apex-evaluation - - equals: - kubernetes.labels.apex: "lessie-agents" - config: - - type: filestream - id: "container-${data.kubernetes.container.id}" - prospector.scanner.symlinks: true - close.on_state_change.removed: false - parsers: - - container: ~ - paths: - - /var/log/containers/*-${data.kubernetes.container.id}.log - processors: - - drop_fields: - fields: - - "kubernetes.node.labels" - - "kubernetes.annotations" - ignore_missing: true - # ---------- ↑ apex 动态创建的 python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↑ ---------- - - - - # ---- 输出到 Elasticsearch ---- - output.elasticsearch: - hosts: ["http://10.0.0.38:9200"] - username: "admin" - password: "G7ZSKFM4AQwHQpwA" - - indices: - - index: "k8s-%{[kubernetes.labels.environment]}-%{[kubernetes.labels.app]}-%{+yyyy.MM.dd}" - when: - regexp: - kubernetes.labels.app: "(lessie-go-api|flymoon-admin|flymoon-agent|flymoon-payment|flymoon-email|lessie-agents|apex)" - - - index: "apex-python-%{[kubernetes.pod.name]}" - when: - equals: - kubernetes.labels.apex: "lessie-agents" - - logging.level: info - logging.selectors: ["*"] \ No newline at end of file diff --git a/k8s_yaml/ELK/filebast/03-filebeat-daemonset.yaml b/k8s_yaml/ELK/filebast/03-filebeat-daemonset.yaml index ca1adc3..c385c21 100644 --- a/k8s_yaml/ELK/filebast/03-filebeat-daemonset.yaml +++ b/k8s_yaml/ELK/filebast/03-filebeat-daemonset.yaml @@ -1,3 +1,6 @@ +# 滚动更新 +# kubectl rollout restart daemonset filebeat -n kube-system + apiVersion: apps/v1 kind: DaemonSet metadata: diff --git a/k8s_yaml/ELK/filebast/filebeat.yaml b/k8s_yaml/ELK/filebast/filebeat.yaml index 4cfcf3d..77260ee 100644 --- a/k8s_yaml/ELK/filebast/filebeat.yaml +++ b/k8s_yaml/ELK/filebast/filebeat.yaml @@ -4,27 +4,205 @@ setup.template.enabled: false filebeat.autodiscover: providers: - type: kubernetes - node: ${NODE_NAME} - # hints.enabled: false templates: - # ---------- go语言的中转服务的Pod, go项目json格式日志 ---------- + # ---------- ↓ json格式日志 ↓ ---------- - condition: - equals: - kubernetes.labels.app: lessie-go-api + and: + - regexp: + kubernetes.namespace: "^(sit|apex-evaluation)$" + - regexp: + kubernetes.labels.app: "^(lessie-go-api|apex)$" config: - type: filestream - id: "k8s-go-json-log-${data.kubernetes.container.id}" + id: "container-${data.kubernetes.container.id}" prospector.scanner.symlinks: true close.on_state_change.removed: false parsers: - container: ~ paths: - /var/log/containers/*-${data.kubernetes.container.id}.log - fields: - application: ${data.kubernetes.labels.app} - log_type: "goho.log" - environment: ${data.kubernetes.labels.environment} - instance: ${data.kubernetes.host} + processors: + - add_kubernetes_metadata: + host: ${NODE_NAME} + - decode_json_fields: + fields: ["message"] + target: "mylog" + overwrite_keys: true + add_error_key: true + - drop_fields: + fields: + - "kubernetes.node.labels" + - "kubernetes.namespace_labels.kubernetes_io/metadata_name" + ignore_missing: true + # ---------- ↑ json格式日志 ↑ ---------- + + + # ---------- ↓ java语言的服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↓ ---------- + - condition: + and: + - equals: + kubernetes.namespace: sit + - or: + - equals: + kubernetes.labels.app: "flymoon-admin" + - equals: + kubernetes.labels.app: "flymoon-agent" + - equals: + kubernetes.labels.app: "flymoon-payment" + config: + - type: filestream + id: "container-${data.kubernetes.container.id}" + prospector.scanner.symlinks: true + close.on_state_change.removed: false + parsers: + - container: ~ + - multiline: + type: pattern + pattern: '^\d{4}-\d{2}-\d{2}-\d{2}:\d{2}:\d{2}\.\d{3}' + negate: true + match: after + paths: + - /var/log/containers/*-${data.kubernetes.container.id}.log + processors: + - add_kubernetes_metadata: + host: ${NODE_NAME} + - dissect: + tokenizer: '%{timestamp} %{level} %{pid} --- [%{thread}] %{class} : [%{app_name->}] %{message}' + field: "message" + target_prefix: "mylog" + ignore_missing: true + overwrite_keys: true + - drop_fields: + fields: ["kubernetes.node.labels", "kubernetes.annotations"] + ignore_missing: true + + # ---------- ↑ java语言的服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↑ ---------- + + + # ---------- ↓ java语言的服务的Pod, email 项目自由文本格式日志 ↓ ---------- + - condition: + and: + - equals: + kubernetes.namespace: sit + - equals: + kubernetes.labels.app: "flymoon-email" + config: + - type: filestream + id: "container-${data.kubernetes.container.id}" + prospector.scanner.symlinks: true + close.on_state_change.removed: false + parsers: + - container: ~ + - multiline: + type: pattern + pattern: '^\d{4}-\d{2}-\d{2}' + negate: true + match: after + paths: + - /var/log/containers/*-${data.kubernetes.container.id}.log + processors: + - add_kubernetes_metadata: + host: ${NODE_NAME} + - dissect: + tokenizer: '%{timestamp} %{level} %{pid} --- [%{thread}] %{class} : %{message}' + field: "message" + target_prefix: "mylog" + ignore_missing: true + overwrite_keys: true + - drop_fields: + fields: ["kubernetes.node.labels", "kubernetes.annotations"] + ignore_missing: true + # ---------- ↑ java语言的服务的Pod, email 项目自由文本格式日志 ↑ ---------- + + + # ---------- ↓ python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↓ ---------- + - condition: + and: + - equals: + kubernetes.namespace: sit + - equals: + kubernetes.labels.app: "lessie-agents" + config: + - type: filestream + id: "container-${data.kubernetes.container.id}" + prospector.scanner.symlinks: true + close.on_state_change.removed: false + parsers: + - container: ~ + paths: + - /var/log/containers/*-${data.kubernetes.container.id}.log + processors: + - add_kubernetes_metadata: + host: ${NODE_NAME} + # 第一层：仅解析符合时间戳开头的日志行(for业务告警的日志格式) + - dissect: + when: + regexp: + message: '^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}.*' + tokenizer: '%{timestamp} - %{level} - %{module} - %{function} - %{msg_body}' + field: "message" + target_prefix: "mylog" + ignore_missing: true + overwrite_keys: true + # 第二层：针对带有 [level: | event: | msg: | context:] 的日志，再做一次 dissect + - dissect: + when: + contains: + mylog.msg_body: "[level:" + tokenizer: '[level: %{event_level} | event: %{event} | msg: %{msg} | context: %{ctx_raw}]' + field: "mylog.msg_body" + target_prefix: "mylog" + ignore_missing: true + overwrite_keys: true + # 第三层：把 ctx_raw 再拆成独立字段 + - script: + lang: javascript + id: parse_context + source: > + function process(event) { + var ctx = event.Get("mylog.ctx_raw"); + if (!ctx) return; + var parts = ctx.trim().split(","); + for (var i = 0; i < parts.length; i++) { + var pair = parts[i].split(":"); + if (pair.length === 2) { + event.Put("mylog." + pair[0].trim(), pair[1].trim()); + } + } + } + # 第四层: 去除大量不需要的k8s元数据字段 + - drop_fields: + fields: + - "kubernetes.node.labels" + - "kubernetes.annotations" + ignore_missing: true + # ---------- ↑ python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↑ ---------- + + + # ---------- ↓ apex 动态创建的 python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↓ ---------- + - condition: + and: + - equals: + kubernetes.namespace: apex-evaluation + - equals: + kubernetes.labels.apex: "lessie-agents" + config: + - type: filestream + id: "container-${data.kubernetes.container.id}" + prospector.scanner.symlinks: true + close.on_state_change.removed: false + parsers: + - container: ~ + paths: + - /var/log/containers/*-${data.kubernetes.container.id}.log + processors: + - drop_fields: + fields: + - "kubernetes.node.labels" + - "kubernetes.annotations" + ignore_missing: true + # ---------- ↑ apex 动态创建的 python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↑ ---------- + # ---- 输出到 Elasticsearch ---- @@ -32,7 +210,17 @@ output.elasticsearch: hosts: ["http://10.0.0.38:9200"] username: "admin" password: "G7ZSKFM4AQwHQpwA" - index: "k8s-%{[kubernetes.labels.environment]}-%{[kubernetes.labels.app]}-%{+yyyy.MM.dd}" -logging.level: debug -logging.selectors: ["autodiscover", "input"] \ No newline at end of file + indices: + - index: "k8s-%{[kubernetes.labels.environment]}-%{[kubernetes.labels.app]}-%{+yyyy.MM.dd}" + when: + regexp: + kubernetes.labels.app: "(lessie-go-api|flymoon-admin|flymoon-agent|flymoon-payment|flymoon-email|lessie-agents|apex)" + + - index: "apex-python-%{+yyyy.MM.dd}" + when: + equals: + kubernetes.labels.apex: "lessie-agents" + +logging.level: info +logging.selectors: ["*"] \ No newline at end of file