改改改
This commit is contained in:
@@ -243,40 +243,55 @@ jenkins执行人: ${env.ACTUAL_USER}
|
|||||||
when { expression { return params.ROLLBACK_VERSION == false } }
|
when { expression { return params.ROLLBACK_VERSION == false } }
|
||||||
steps {
|
steps {
|
||||||
script {
|
script {
|
||||||
withCredentials([usernamePassword(credentialsId: 'fly_gitlab_auth', usernameVariable: 'GIT_USER', passwordVariable: 'GIT_PASS')]) {
|
withCredentials([usernamePassword(
|
||||||
|
credentialsId: 'fly_gitlab_auth',
|
||||||
|
usernameVariable: 'GIT_USER',
|
||||||
|
passwordVariable: 'GIT_PASS'
|
||||||
|
)]) {
|
||||||
sh """
|
sh """
|
||||||
|
set -e
|
||||||
|
|
||||||
cd ${WORKSPACE}
|
cd ${WORKSPACE}
|
||||||
git config user.name "jenkins"
|
git config user.name "jenkins"
|
||||||
git config user.email "jenkins@local"
|
git config user.email "jenkins@local"
|
||||||
|
|
||||||
# 检查工作树是否有变化
|
git checkout ${params.BRANCH_NAME}
|
||||||
if ! git diff --exit-code ${Deployment_yaml} > /dev/null 2>&1; then
|
|
||||||
echo "检测到更改，正在提交..."
|
|
||||||
git add ${Deployment_yaml}
|
|
||||||
git commit -m "更新镜像 \${CHANGE_MSG}"
|
|
||||||
else
|
|
||||||
echo "${Deployment_yaml} 没有变化, 无需commit"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# 检查是否需要推送（是否有新的提交）
|
echo "machine 172.24.16.20 login ${GIT_USER} password ${GIT_PASS}" > ~/.netrc
|
||||||
LOCAL=\$(git rev-parse @)
|
|
||||||
REMOTE=\$(git rev-parse @{u} 2>/dev/null || true)
|
|
||||||
BASE=\$(git merge-base @ @{u} 2>/dev/null || true)
|
|
||||||
|
|
||||||
if [ "\$LOCAL" = "\$REMOTE" ]; then
|
|
||||||
echo "已与远程系统同步更新"
|
|
||||||
elif [ "\$LOCAL" = "\$BASE" ]; then
|
|
||||||
echo "需要从远程获取数据"
|
|
||||||
git pull http://172.24.16.20/opt/opt-config.git ${params.BRANCH_NAME}
|
|
||||||
elif [ "\$REMOTE" = "\$BASE" ]; then
|
|
||||||
echo "将更改推送到远程服务器..."
|
|
||||||
# 生成临时 .netrc 文件
|
|
||||||
echo "machine 172.24.16.20 login \$GIT_USER password \$GIT_PASS" > ~/.netrc
|
|
||||||
chmod 600 ~/.netrc
|
chmod 600 ~/.netrc
|
||||||
git push http://172.24.16.20/opt/opt-config.git ${params.BRANCH_NAME}
|
|
||||||
|
MAX_RETRY=3
|
||||||
|
i=1
|
||||||
|
|
||||||
|
while [ \$i -le \$MAX_RETRY ]; do
|
||||||
|
echo "🔁 尝试第 \$i 次提交..."
|
||||||
|
|
||||||
|
git fetch http://172.24.16.20/opt/opt-config.git ${params.BRANCH_NAME}
|
||||||
|
|
||||||
|
if ! git diff --exit-code ${Deployment_yaml} > /dev/null 2>&1; then
|
||||||
|
git add ${Deployment_yaml}
|
||||||
|
git commit -m "更新镜像 ${CHANGE_MSG}" || true
|
||||||
else
|
else
|
||||||
echo "与远程模式不同，跳过推送操作"
|
echo "文件无变化，跳过 commit"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
if git rebase FETCH_HEAD; then
|
||||||
|
if git push http://172.24.16.20/opt/opt-config.git ${params.BRANCH_NAME}; then
|
||||||
|
echo "✅ 推送成功"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "⚠️ 推送失败，回滚并重试..."
|
||||||
|
git rebase --abort || true
|
||||||
|
git reset --hard FETCH_HEAD
|
||||||
|
|
||||||
|
i=\$((i+1))
|
||||||
|
sleep 2
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "❌ 多次尝试失败，可能存在并发冲突"
|
||||||
|
exit 1
|
||||||
"""
|
"""
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -243,40 +243,55 @@ jenkins执行人: ${env.ACTUAL_USER}
|
|||||||
when { expression { return params.ROLLBACK_VERSION == false } }
|
when { expression { return params.ROLLBACK_VERSION == false } }
|
||||||
steps {
|
steps {
|
||||||
script {
|
script {
|
||||||
withCredentials([usernamePassword(credentialsId: 'fly_gitlab_auth', usernameVariable: 'GIT_USER', passwordVariable: 'GIT_PASS')]) {
|
withCredentials([usernamePassword(
|
||||||
|
credentialsId: 'fly_gitlab_auth',
|
||||||
|
usernameVariable: 'GIT_USER',
|
||||||
|
passwordVariable: 'GIT_PASS'
|
||||||
|
)]) {
|
||||||
sh """
|
sh """
|
||||||
|
set -e
|
||||||
|
|
||||||
cd ${WORKSPACE}
|
cd ${WORKSPACE}
|
||||||
git config user.name "jenkins"
|
git config user.name "jenkins"
|
||||||
git config user.email "jenkins@local"
|
git config user.email "jenkins@local"
|
||||||
|
|
||||||
# 检查工作树是否有变化
|
git checkout ${params.BRANCH_NAME}
|
||||||
if ! git diff --exit-code ${Deployment_yaml} > /dev/null 2>&1; then
|
|
||||||
echo "检测到更改，正在提交..."
|
|
||||||
git add ${Deployment_yaml}
|
|
||||||
git commit -m "更新镜像 \${CHANGE_MSG}"
|
|
||||||
else
|
|
||||||
echo "${Deployment_yaml} 没有变化, 无需commit"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# 检查是否需要推送（是否有新的提交）
|
echo "machine 172.24.16.20 login ${GIT_USER} password ${GIT_PASS}" > ~/.netrc
|
||||||
LOCAL=\$(git rev-parse @)
|
|
||||||
REMOTE=\$(git rev-parse @{u} 2>/dev/null || true)
|
|
||||||
BASE=\$(git merge-base @ @{u} 2>/dev/null || true)
|
|
||||||
|
|
||||||
if [ "\$LOCAL" = "\$REMOTE" ]; then
|
|
||||||
echo "已与远程系统同步更新"
|
|
||||||
elif [ "\$LOCAL" = "\$BASE" ]; then
|
|
||||||
echo "需要从远程获取数据"
|
|
||||||
git pull http://172.24.16.20/opt/opt-config.git ${params.BRANCH_NAME}
|
|
||||||
elif [ "\$REMOTE" = "\$BASE" ]; then
|
|
||||||
echo "将更改推送到远程服务器..."
|
|
||||||
# 生成临时 .netrc 文件
|
|
||||||
echo "machine 172.24.16.20 login \$GIT_USER password \$GIT_PASS" > ~/.netrc
|
|
||||||
chmod 600 ~/.netrc
|
chmod 600 ~/.netrc
|
||||||
git push http://172.24.16.20/opt/opt-config.git ${params.BRANCH_NAME}
|
|
||||||
|
MAX_RETRY=3
|
||||||
|
i=1
|
||||||
|
|
||||||
|
while [ \$i -le \$MAX_RETRY ]; do
|
||||||
|
echo "🔁 尝试第 \$i 次提交..."
|
||||||
|
|
||||||
|
git fetch http://172.24.16.20/opt/opt-config.git ${params.BRANCH_NAME}
|
||||||
|
|
||||||
|
if ! git diff --exit-code ${Deployment_yaml} > /dev/null 2>&1; then
|
||||||
|
git add ${Deployment_yaml}
|
||||||
|
git commit -m "更新镜像 \${CHANGE_MSG}" || true
|
||||||
else
|
else
|
||||||
echo "与远程模式不同，跳过推送操作"
|
echo "文件无变化，跳过 commit"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
if git rebase FETCH_HEAD; then
|
||||||
|
if git push http://172.24.16.20/opt/opt-config.git ${params.BRANCH_NAME}; then
|
||||||
|
echo "✅ 推送成功"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "⚠️ 推送失败，回滚并重试..."
|
||||||
|
git rebase --abort || true
|
||||||
|
git reset --hard FETCH_HEAD
|
||||||
|
|
||||||
|
i=\$((i+1))
|
||||||
|
sleep 2
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "❌ 多次尝试失败，可能存在并发冲突"
|
||||||
|
exit 1
|
||||||
"""
|
"""
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -243,40 +243,55 @@ jenkins执行人: ${env.ACTUAL_USER}
|
|||||||
when { expression { return params.ROLLBACK_VERSION == false } }
|
when { expression { return params.ROLLBACK_VERSION == false } }
|
||||||
steps {
|
steps {
|
||||||
script {
|
script {
|
||||||
withCredentials([usernamePassword(credentialsId: 'fly_gitlab_auth', usernameVariable: 'GIT_USER', passwordVariable: 'GIT_PASS')]) {
|
withCredentials([usernamePassword(
|
||||||
|
credentialsId: 'fly_gitlab_auth',
|
||||||
|
usernameVariable: 'GIT_USER',
|
||||||
|
passwordVariable: 'GIT_PASS'
|
||||||
|
)]) {
|
||||||
sh """
|
sh """
|
||||||
|
set -e
|
||||||
|
|
||||||
cd ${WORKSPACE}
|
cd ${WORKSPACE}
|
||||||
git config user.name "jenkins"
|
git config user.name "jenkins"
|
||||||
git config user.email "jenkins@local"
|
git config user.email "jenkins@local"
|
||||||
|
|
||||||
# 检查工作树是否有变化
|
git checkout ${params.BRANCH_NAME}
|
||||||
if ! git diff --exit-code ${Deployment_yaml} > /dev/null 2>&1; then
|
|
||||||
echo "检测到更改，正在提交..."
|
|
||||||
git add ${Deployment_yaml}
|
|
||||||
git commit -m "更新镜像 \${CHANGE_MSG}"
|
|
||||||
else
|
|
||||||
echo "${Deployment_yaml} 没有变化, 无需commit"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# 检查是否需要推送（是否有新的提交）
|
echo "machine 172.24.16.20 login ${GIT_USER} password ${GIT_PASS}" > ~/.netrc
|
||||||
LOCAL=\$(git rev-parse @)
|
|
||||||
REMOTE=\$(git rev-parse @{u} 2>/dev/null || true)
|
|
||||||
BASE=\$(git merge-base @ @{u} 2>/dev/null || true)
|
|
||||||
|
|
||||||
if [ "\$LOCAL" = "\$REMOTE" ]; then
|
|
||||||
echo "已与远程系统同步更新"
|
|
||||||
elif [ "\$LOCAL" = "\$BASE" ]; then
|
|
||||||
echo "需要从远程获取数据"
|
|
||||||
git pull http://172.24.16.20/opt/opt-config.git ${params.BRANCH_NAME}
|
|
||||||
elif [ "\$REMOTE" = "\$BASE" ]; then
|
|
||||||
echo "将更改推送到远程服务器..."
|
|
||||||
# 生成临时 .netrc 文件
|
|
||||||
echo "machine 172.24.16.20 login \$GIT_USER password \$GIT_PASS" > ~/.netrc
|
|
||||||
chmod 600 ~/.netrc
|
chmod 600 ~/.netrc
|
||||||
git push http://172.24.16.20/opt/opt-config.git ${params.BRANCH_NAME}
|
|
||||||
|
MAX_RETRY=3
|
||||||
|
i=1
|
||||||
|
|
||||||
|
while [ \$i -le \$MAX_RETRY ]; do
|
||||||
|
echo "🔁 尝试第 \$i 次提交..."
|
||||||
|
|
||||||
|
git fetch http://172.24.16.20/opt/opt-config.git ${params.BRANCH_NAME}
|
||||||
|
|
||||||
|
if ! git diff --exit-code ${Deployment_yaml} > /dev/null 2>&1; then
|
||||||
|
git add ${Deployment_yaml}
|
||||||
|
git commit -m "更新镜像 ${CHANGE_MSG}" || true
|
||||||
else
|
else
|
||||||
echo "与远程模式不同，跳过推送操作"
|
echo "文件无变化，跳过 commit"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
if git rebase FETCH_HEAD; then
|
||||||
|
if git push http://172.24.16.20/opt/opt-config.git ${params.BRANCH_NAME}; then
|
||||||
|
echo "✅ 推送成功"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "⚠️ 推送失败，回滚并重试..."
|
||||||
|
git rebase --abort || true
|
||||||
|
git reset --hard FETCH_HEAD
|
||||||
|
|
||||||
|
i=\$((i+1))
|
||||||
|
sleep 2
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "❌ 多次尝试失败，可能存在并发冲突"
|
||||||
|
exit 1
|
||||||
"""
|
"""
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -243,40 +243,55 @@ jenkins执行人: ${env.ACTUAL_USER}
|
|||||||
when { expression { return params.ROLLBACK_VERSION == false } }
|
when { expression { return params.ROLLBACK_VERSION == false } }
|
||||||
steps {
|
steps {
|
||||||
script {
|
script {
|
||||||
withCredentials([usernamePassword(credentialsId: 'fly_gitlab_auth', usernameVariable: 'GIT_USER', passwordVariable: 'GIT_PASS')]) {
|
withCredentials([usernamePassword(
|
||||||
|
credentialsId: 'fly_gitlab_auth',
|
||||||
|
usernameVariable: 'GIT_USER',
|
||||||
|
passwordVariable: 'GIT_PASS'
|
||||||
|
)]) {
|
||||||
sh """
|
sh """
|
||||||
|
set -e
|
||||||
|
|
||||||
cd ${WORKSPACE}
|
cd ${WORKSPACE}
|
||||||
git config user.name "jenkins"
|
git config user.name "jenkins"
|
||||||
git config user.email "jenkins@local"
|
git config user.email "jenkins@local"
|
||||||
|
|
||||||
# 检查工作树是否有变化
|
git checkout ${params.BRANCH_NAME}
|
||||||
if ! git diff --exit-code ${Deployment_yaml} > /dev/null 2>&1; then
|
|
||||||
echo "检测到更改，正在提交..."
|
|
||||||
git add ${Deployment_yaml}
|
|
||||||
git commit -m "更新镜像 \${CHANGE_MSG}"
|
|
||||||
else
|
|
||||||
echo "${Deployment_yaml} 没有变化, 无需commit"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# 检查是否需要推送（是否有新的提交）
|
echo "machine 172.24.16.20 login ${GIT_USER} password ${GIT_PASS}" > ~/.netrc
|
||||||
LOCAL=\$(git rev-parse @)
|
|
||||||
REMOTE=\$(git rev-parse @{u} 2>/dev/null || true)
|
|
||||||
BASE=\$(git merge-base @ @{u} 2>/dev/null || true)
|
|
||||||
|
|
||||||
if [ "\$LOCAL" = "\$REMOTE" ]; then
|
|
||||||
echo "已与远程系统同步更新"
|
|
||||||
elif [ "\$LOCAL" = "\$BASE" ]; then
|
|
||||||
echo "需要从远程获取数据"
|
|
||||||
git pull http://172.24.16.20/opt/opt-config.git ${params.BRANCH_NAME}
|
|
||||||
elif [ "\$REMOTE" = "\$BASE" ]; then
|
|
||||||
echo "将更改推送到远程服务器..."
|
|
||||||
# 生成临时 .netrc 文件
|
|
||||||
echo "machine 172.24.16.20 login \$GIT_USER password \$GIT_PASS" > ~/.netrc
|
|
||||||
chmod 600 ~/.netrc
|
chmod 600 ~/.netrc
|
||||||
git push http://172.24.16.20/opt/opt-config.git ${params.BRANCH_NAME}
|
|
||||||
|
MAX_RETRY=3
|
||||||
|
i=1
|
||||||
|
|
||||||
|
while [ \$i -le \$MAX_RETRY ]; do
|
||||||
|
echo "🔁 尝试第 \$i 次提交..."
|
||||||
|
|
||||||
|
git fetch http://172.24.16.20/opt/opt-config.git ${params.BRANCH_NAME}
|
||||||
|
|
||||||
|
if ! git diff --exit-code ${Deployment_yaml} > /dev/null 2>&1; then
|
||||||
|
git add ${Deployment_yaml}
|
||||||
|
git commit -m "更新镜像 ${CHANGE_MSG}" || true
|
||||||
else
|
else
|
||||||
echo "与远程模式不同，跳过推送操作"
|
echo "文件无变化，跳过 commit"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
if git rebase FETCH_HEAD; then
|
||||||
|
if git push http://172.24.16.20/opt/opt-config.git ${params.BRANCH_NAME}; then
|
||||||
|
echo "✅ 推送成功"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "⚠️ 推送失败，回滚并重试..."
|
||||||
|
git rebase --abort || true
|
||||||
|
git reset --hard FETCH_HEAD
|
||||||
|
|
||||||
|
i=\$((i+1))
|
||||||
|
sleep 2
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "❌ 多次尝试失败，可能存在并发冲突"
|
||||||
|
exit 1
|
||||||
"""
|
"""
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -12,13 +12,13 @@ data:
|
|||||||
providers:
|
providers:
|
||||||
- type: kubernetes
|
- type: kubernetes
|
||||||
templates:
|
templates:
|
||||||
# ---------- ↓ go语言的中转服务的Pod, go项目json格式日志 ↓ ----------
|
# ---------- ↓ json格式日志 ↓ ----------
|
||||||
- condition:
|
- condition:
|
||||||
and:
|
and:
|
||||||
- equals:
|
- regexp:
|
||||||
kubernetes.namespace: sit
|
kubernetes.namespace: "^(sit|apex-evaluation)$"
|
||||||
- equals:
|
- regexp:
|
||||||
kubernetes.labels.app: "lessie-go-api"
|
kubernetes.labels.app: "^(lessie-go-api|apex)$"
|
||||||
config:
|
config:
|
||||||
- type: filestream
|
- type: filestream
|
||||||
id: "container-${data.kubernetes.container.id}"
|
id: "container-${data.kubernetes.container.id}"
|
||||||
@@ -28,7 +28,6 @@ data:
|
|||||||
- container: ~
|
- container: ~
|
||||||
paths:
|
paths:
|
||||||
- /var/log/containers/*-${data.kubernetes.container.id}.log
|
- /var/log/containers/*-${data.kubernetes.container.id}.log
|
||||||
|
|
||||||
processors:
|
processors:
|
||||||
- add_kubernetes_metadata:
|
- add_kubernetes_metadata:
|
||||||
host: ${NODE_NAME}
|
host: ${NODE_NAME}
|
||||||
@@ -42,10 +41,10 @@ data:
|
|||||||
- "kubernetes.node.labels"
|
- "kubernetes.node.labels"
|
||||||
- "kubernetes.namespace_labels.kubernetes_io/metadata_name"
|
- "kubernetes.namespace_labels.kubernetes_io/metadata_name"
|
||||||
ignore_missing: true
|
ignore_missing: true
|
||||||
|
# ---------- ↑ json格式日志 ↑ ----------
|
||||||
|
|
||||||
# ---------- ↑ go语言的中转服务的Pod, go项目json格式日志 ↑ ----------
|
|
||||||
|
|
||||||
# ---------- ↓ java语言的中转服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↓ ----------
|
# ---------- ↓ java语言的服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↓ ----------
|
||||||
- condition:
|
- condition:
|
||||||
and:
|
and:
|
||||||
- equals:
|
- equals:
|
||||||
@@ -71,7 +70,6 @@ data:
|
|||||||
match: after
|
match: after
|
||||||
paths:
|
paths:
|
||||||
- /var/log/containers/*-${data.kubernetes.container.id}.log
|
- /var/log/containers/*-${data.kubernetes.container.id}.log
|
||||||
|
|
||||||
processors:
|
processors:
|
||||||
- add_kubernetes_metadata:
|
- add_kubernetes_metadata:
|
||||||
host: ${NODE_NAME}
|
host: ${NODE_NAME}
|
||||||
@@ -85,7 +83,44 @@ data:
|
|||||||
fields: ["kubernetes.node.labels", "kubernetes.annotations"]
|
fields: ["kubernetes.node.labels", "kubernetes.annotations"]
|
||||||
ignore_missing: true
|
ignore_missing: true
|
||||||
|
|
||||||
# ---------- ↑ java语言的中转服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↑ ----------
|
# ---------- ↑ java语言的服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↑ ----------
|
||||||
|
|
||||||
|
|
||||||
|
# ---------- ↓ java语言的服务的Pod, email 项目自由文本格式日志 ↓ ----------
|
||||||
|
- condition:
|
||||||
|
and:
|
||||||
|
- equals:
|
||||||
|
kubernetes.namespace: sit
|
||||||
|
- equals:
|
||||||
|
kubernetes.labels.app: "flymoon-email"
|
||||||
|
config:
|
||||||
|
- type: filestream
|
||||||
|
id: "container-${data.kubernetes.container.id}"
|
||||||
|
prospector.scanner.symlinks: true
|
||||||
|
close.on_state_change.removed: false
|
||||||
|
parsers:
|
||||||
|
- container: ~
|
||||||
|
- multiline:
|
||||||
|
type: pattern
|
||||||
|
pattern: '^\d{4}-\d{2}-\d{2}'
|
||||||
|
negate: true
|
||||||
|
match: after
|
||||||
|
paths:
|
||||||
|
- /var/log/containers/*-${data.kubernetes.container.id}.log
|
||||||
|
processors:
|
||||||
|
- add_kubernetes_metadata:
|
||||||
|
host: ${NODE_NAME}
|
||||||
|
- dissect:
|
||||||
|
tokenizer: '%{timestamp} %{level} %{pid} --- [%{thread}] %{class} : %{message}'
|
||||||
|
field: "message"
|
||||||
|
target_prefix: "mylog"
|
||||||
|
ignore_missing: true
|
||||||
|
overwrite_keys: true
|
||||||
|
- drop_fields:
|
||||||
|
fields: ["kubernetes.node.labels", "kubernetes.annotations"]
|
||||||
|
ignore_missing: true
|
||||||
|
# ---------- ↑ java语言的服务的Pod, email 项目自由文本格式日志 ↑ ----------
|
||||||
|
|
||||||
|
|
||||||
# ---------- ↓ python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↓ ----------
|
# ---------- ↓ python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↓ ----------
|
||||||
- condition:
|
- condition:
|
||||||
@@ -103,7 +138,6 @@ data:
|
|||||||
- container: ~
|
- container: ~
|
||||||
paths:
|
paths:
|
||||||
- /var/log/containers/*-${data.kubernetes.container.id}.log
|
- /var/log/containers/*-${data.kubernetes.container.id}.log
|
||||||
|
|
||||||
processors:
|
processors:
|
||||||
- add_kubernetes_metadata:
|
- add_kubernetes_metadata:
|
||||||
host: ${NODE_NAME}
|
host: ${NODE_NAME}
|
||||||
@@ -149,17 +183,51 @@ data:
|
|||||||
- "kubernetes.node.labels"
|
- "kubernetes.node.labels"
|
||||||
- "kubernetes.annotations"
|
- "kubernetes.annotations"
|
||||||
ignore_missing: true
|
ignore_missing: true
|
||||||
|
|
||||||
# ---------- ↑ python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↑ ----------
|
# ---------- ↑ python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↑ ----------
|
||||||
|
|
||||||
|
|
||||||
|
# ---------- ↓ apex 动态创建的 python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↓ ----------
|
||||||
|
- condition:
|
||||||
|
and:
|
||||||
|
- equals:
|
||||||
|
kubernetes.namespace: apex-evaluation
|
||||||
|
- equals:
|
||||||
|
kubernetes.labels.apex: "lessie-agents"
|
||||||
|
config:
|
||||||
|
- type: filestream
|
||||||
|
id: "container-${data.kubernetes.container.id}"
|
||||||
|
prospector.scanner.symlinks: true
|
||||||
|
close.on_state_change.removed: false
|
||||||
|
parsers:
|
||||||
|
- container: ~
|
||||||
|
paths:
|
||||||
|
- /var/log/containers/*-${data.kubernetes.container.id}.log
|
||||||
|
processors:
|
||||||
|
- drop_fields:
|
||||||
|
fields:
|
||||||
|
- "kubernetes.node.labels"
|
||||||
|
- "kubernetes.annotations"
|
||||||
|
ignore_missing: true
|
||||||
|
# ---------- ↑ apex 动态创建的 python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↑ ----------
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
# ---- 输出到 Elasticsearch ----
|
# ---- 输出到 Elasticsearch ----
|
||||||
output.elasticsearch:
|
output.elasticsearch:
|
||||||
hosts: ["http://10.0.0.38:9200"]
|
hosts: ["http://10.0.0.38:9200"]
|
||||||
username: "admin"
|
username: "admin"
|
||||||
password: "G7ZSKFM4AQwHQpwA"
|
password: "G7ZSKFM4AQwHQpwA"
|
||||||
index: "k8s-%{[kubernetes.labels.environment]}-%{[kubernetes.labels.app]}-%{+yyyy.MM.dd}"
|
|
||||||
# index: "k8s-%{[kubernetes.labels.app]}-%{+yyyy.MM.dd}"
|
|
||||||
|
|
||||||
logging.level: debug
|
indices:
|
||||||
|
- index: "k8s-%{[kubernetes.labels.environment]}-%{[kubernetes.labels.app]}-%{+yyyy.MM}"
|
||||||
|
when:
|
||||||
|
regexp:
|
||||||
|
kubernetes.labels.app: "(lessie-go-api|flymoon-admin|flymoon-agent|flymoon-payment|flymoon-email|lessie-agents|apex)"
|
||||||
|
|
||||||
|
- index: "apex-python-%{+yyyy.MM}"
|
||||||
|
when:
|
||||||
|
equals:
|
||||||
|
kubernetes.labels.apex: "lessie-agents"
|
||||||
|
|
||||||
|
logging.level: info
|
||||||
logging.selectors: ["*"]
|
logging.selectors: ["*"]
|
||||||
@@ -12,13 +12,13 @@ data:
|
|||||||
providers:
|
providers:
|
||||||
- type: kubernetes
|
- type: kubernetes
|
||||||
templates:
|
templates:
|
||||||
# ---------- ↓ go语言的中转服务的Pod, go项目json格式日志 ↓ ----------
|
# ---------- ↓ json格式日志 ↓ ----------
|
||||||
- condition:
|
- condition:
|
||||||
and:
|
and:
|
||||||
- equals:
|
- regexp:
|
||||||
kubernetes.namespace: sit
|
kubernetes.namespace: "^(sit|apex-evaluation)$"
|
||||||
- equals:
|
- regexp:
|
||||||
kubernetes.labels.app: "lessie-go-api"
|
kubernetes.labels.app: "^(lessie-go-api|apex)$"
|
||||||
config:
|
config:
|
||||||
- type: filestream
|
- type: filestream
|
||||||
id: "container-${data.kubernetes.container.id}"
|
id: "container-${data.kubernetes.container.id}"
|
||||||
@@ -28,7 +28,6 @@ data:
|
|||||||
- container: ~
|
- container: ~
|
||||||
paths:
|
paths:
|
||||||
- /var/log/containers/*-${data.kubernetes.container.id}.log
|
- /var/log/containers/*-${data.kubernetes.container.id}.log
|
||||||
|
|
||||||
processors:
|
processors:
|
||||||
- add_kubernetes_metadata:
|
- add_kubernetes_metadata:
|
||||||
host: ${NODE_NAME}
|
host: ${NODE_NAME}
|
||||||
@@ -42,10 +41,10 @@ data:
|
|||||||
- "kubernetes.node.labels"
|
- "kubernetes.node.labels"
|
||||||
- "kubernetes.namespace_labels.kubernetes_io/metadata_name"
|
- "kubernetes.namespace_labels.kubernetes_io/metadata_name"
|
||||||
ignore_missing: true
|
ignore_missing: true
|
||||||
|
# ---------- ↑ json格式日志 ↑ ----------
|
||||||
|
|
||||||
# ---------- ↑ go语言的中转服务的Pod, go项目json格式日志 ↑ ----------
|
|
||||||
|
|
||||||
# ---------- ↓ java语言的中转服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↓ ----------
|
# ---------- ↓ java语言的服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↓ ----------
|
||||||
- condition:
|
- condition:
|
||||||
and:
|
and:
|
||||||
- equals:
|
- equals:
|
||||||
@@ -71,7 +70,6 @@ data:
|
|||||||
match: after
|
match: after
|
||||||
paths:
|
paths:
|
||||||
- /var/log/containers/*-${data.kubernetes.container.id}.log
|
- /var/log/containers/*-${data.kubernetes.container.id}.log
|
||||||
|
|
||||||
processors:
|
processors:
|
||||||
- add_kubernetes_metadata:
|
- add_kubernetes_metadata:
|
||||||
host: ${NODE_NAME}
|
host: ${NODE_NAME}
|
||||||
@@ -85,7 +83,44 @@ data:
|
|||||||
fields: ["kubernetes.node.labels", "kubernetes.annotations"]
|
fields: ["kubernetes.node.labels", "kubernetes.annotations"]
|
||||||
ignore_missing: true
|
ignore_missing: true
|
||||||
|
|
||||||
# ---------- ↑ java语言的中转服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↑ ----------
|
# ---------- ↑ java语言的服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↑ ----------
|
||||||
|
|
||||||
|
|
||||||
|
# ---------- ↓ java语言的服务的Pod, email 项目自由文本格式日志 ↓ ----------
|
||||||
|
- condition:
|
||||||
|
and:
|
||||||
|
- equals:
|
||||||
|
kubernetes.namespace: sit
|
||||||
|
- equals:
|
||||||
|
kubernetes.labels.app: "flymoon-email"
|
||||||
|
config:
|
||||||
|
- type: filestream
|
||||||
|
id: "container-${data.kubernetes.container.id}"
|
||||||
|
prospector.scanner.symlinks: true
|
||||||
|
close.on_state_change.removed: false
|
||||||
|
parsers:
|
||||||
|
- container: ~
|
||||||
|
- multiline:
|
||||||
|
type: pattern
|
||||||
|
pattern: '^\d{4}-\d{2}-\d{2}'
|
||||||
|
negate: true
|
||||||
|
match: after
|
||||||
|
paths:
|
||||||
|
- /var/log/containers/*-${data.kubernetes.container.id}.log
|
||||||
|
processors:
|
||||||
|
- add_kubernetes_metadata:
|
||||||
|
host: ${NODE_NAME}
|
||||||
|
- dissect:
|
||||||
|
tokenizer: '%{timestamp} %{level} %{pid} --- [%{thread}] %{class} : %{message}'
|
||||||
|
field: "message"
|
||||||
|
target_prefix: "mylog"
|
||||||
|
ignore_missing: true
|
||||||
|
overwrite_keys: true
|
||||||
|
- drop_fields:
|
||||||
|
fields: ["kubernetes.node.labels", "kubernetes.annotations"]
|
||||||
|
ignore_missing: true
|
||||||
|
# ---------- ↑ java语言的服务的Pod, email 项目自由文本格式日志 ↑ ----------
|
||||||
|
|
||||||
|
|
||||||
# ---------- ↓ python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↓ ----------
|
# ---------- ↓ python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↓ ----------
|
||||||
- condition:
|
- condition:
|
||||||
@@ -103,7 +138,6 @@ data:
|
|||||||
- container: ~
|
- container: ~
|
||||||
paths:
|
paths:
|
||||||
- /var/log/containers/*-${data.kubernetes.container.id}.log
|
- /var/log/containers/*-${data.kubernetes.container.id}.log
|
||||||
|
|
||||||
processors:
|
processors:
|
||||||
- add_kubernetes_metadata:
|
- add_kubernetes_metadata:
|
||||||
host: ${NODE_NAME}
|
host: ${NODE_NAME}
|
||||||
@@ -149,17 +183,51 @@ data:
|
|||||||
- "kubernetes.node.labels"
|
- "kubernetes.node.labels"
|
||||||
- "kubernetes.annotations"
|
- "kubernetes.annotations"
|
||||||
ignore_missing: true
|
ignore_missing: true
|
||||||
|
|
||||||
# ---------- ↑ python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↑ ----------
|
# ---------- ↑ python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↑ ----------
|
||||||
|
|
||||||
|
|
||||||
|
# ---------- ↓ apex 动态创建的 python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↓ ----------
|
||||||
|
- condition:
|
||||||
|
and:
|
||||||
|
- equals:
|
||||||
|
kubernetes.namespace: apex-evaluation
|
||||||
|
- equals:
|
||||||
|
kubernetes.labels.apex: "lessie-agents"
|
||||||
|
config:
|
||||||
|
- type: filestream
|
||||||
|
id: "container-${data.kubernetes.container.id}"
|
||||||
|
prospector.scanner.symlinks: true
|
||||||
|
close.on_state_change.removed: false
|
||||||
|
parsers:
|
||||||
|
- container: ~
|
||||||
|
paths:
|
||||||
|
- /var/log/containers/*-${data.kubernetes.container.id}.log
|
||||||
|
processors:
|
||||||
|
- drop_fields:
|
||||||
|
fields:
|
||||||
|
- "kubernetes.node.labels"
|
||||||
|
- "kubernetes.annotations"
|
||||||
|
ignore_missing: true
|
||||||
|
# ---------- ↑ apex 动态创建的 python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↑ ----------
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
# ---- 输出到 Elasticsearch ----
|
# ---- 输出到 Elasticsearch ----
|
||||||
output.elasticsearch:
|
output.elasticsearch:
|
||||||
hosts: ["http://10.0.0.38:9200"]
|
hosts: ["http://10.0.0.38:9200"]
|
||||||
username: "admin"
|
username: "admin"
|
||||||
password: "G7ZSKFM4AQwHQpwA"
|
password: "G7ZSKFM4AQwHQpwA"
|
||||||
index: "k8s-%{[kubernetes.labels.environment]}-%{[kubernetes.labels.app]}-%{+yyyy.MM.dd}"
|
|
||||||
# index: "k8s-%{[kubernetes.labels.app]}-%{+yyyy.MM.dd}"
|
|
||||||
|
|
||||||
logging.level: debug
|
indices:
|
||||||
|
- index: "k8s-%{[kubernetes.labels.environment]}-%{[kubernetes.labels.app]}-%{+yyyy.MM}"
|
||||||
|
when:
|
||||||
|
regexp:
|
||||||
|
kubernetes.labels.app: "(lessie-go-api|flymoon-admin|flymoon-agent|flymoon-payment|flymoon-email|lessie-agents|apex)"
|
||||||
|
|
||||||
|
- index: "apex-python-%{+yyyy.MM}"
|
||||||
|
when:
|
||||||
|
equals:
|
||||||
|
kubernetes.labels.apex: "lessie-agents"
|
||||||
|
|
||||||
|
logging.level: info
|
||||||
logging.selectors: ["*"]
|
logging.selectors: ["*"]
|
||||||
@@ -1,233 +0,0 @@
|
|||||||
apiVersion: v1
|
|
||||||
kind: ConfigMap
|
|
||||||
metadata:
|
|
||||||
name: filebeat-config
|
|
||||||
namespace: kube-system
|
|
||||||
data:
|
|
||||||
filebeat.yml: |
|
|
||||||
setup.ilm.enabled: false
|
|
||||||
setup.template.enabled: false
|
|
||||||
|
|
||||||
filebeat.autodiscover:
|
|
||||||
providers:
|
|
||||||
- type: kubernetes
|
|
||||||
templates:
|
|
||||||
# ---------- ↓ json格式日志 ↓ ----------
|
|
||||||
- condition:
|
|
||||||
and:
|
|
||||||
- regexp:
|
|
||||||
kubernetes.namespace: "^(sit|apex-evaluation)$"
|
|
||||||
- regexp:
|
|
||||||
kubernetes.labels.app: "^(lessie-go-api|apex)$"
|
|
||||||
config:
|
|
||||||
- type: filestream
|
|
||||||
id: "container-${data.kubernetes.container.id}"
|
|
||||||
prospector.scanner.symlinks: true
|
|
||||||
close.on_state_change.removed: false
|
|
||||||
parsers:
|
|
||||||
- container: ~
|
|
||||||
paths:
|
|
||||||
- /var/log/containers/*-${data.kubernetes.container.id}.log
|
|
||||||
processors:
|
|
||||||
- add_kubernetes_metadata:
|
|
||||||
host: ${NODE_NAME}
|
|
||||||
- decode_json_fields:
|
|
||||||
fields: ["message"]
|
|
||||||
target: "mylog"
|
|
||||||
overwrite_keys: true
|
|
||||||
add_error_key: true
|
|
||||||
- drop_fields:
|
|
||||||
fields:
|
|
||||||
- "kubernetes.node.labels"
|
|
||||||
- "kubernetes.namespace_labels.kubernetes_io/metadata_name"
|
|
||||||
ignore_missing: true
|
|
||||||
# ---------- ↑ json格式日志 ↑ ----------
|
|
||||||
|
|
||||||
|
|
||||||
# ---------- ↓ java语言的服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↓ ----------
|
|
||||||
- condition:
|
|
||||||
and:
|
|
||||||
- equals:
|
|
||||||
kubernetes.namespace: sit
|
|
||||||
- or:
|
|
||||||
- equals:
|
|
||||||
kubernetes.labels.app: "flymoon-admin"
|
|
||||||
- equals:
|
|
||||||
kubernetes.labels.app: "flymoon-agent"
|
|
||||||
- equals:
|
|
||||||
kubernetes.labels.app: "flymoon-payment"
|
|
||||||
config:
|
|
||||||
- type: filestream
|
|
||||||
id: "container-${data.kubernetes.container.id}"
|
|
||||||
prospector.scanner.symlinks: true
|
|
||||||
close.on_state_change.removed: false
|
|
||||||
parsers:
|
|
||||||
- container: ~
|
|
||||||
- multiline:
|
|
||||||
type: pattern
|
|
||||||
pattern: '^\d{4}-\d{2}-\d{2}-\d{2}:\d{2}:\d{2}\.\d{3}'
|
|
||||||
negate: true
|
|
||||||
match: after
|
|
||||||
paths:
|
|
||||||
- /var/log/containers/*-${data.kubernetes.container.id}.log
|
|
||||||
processors:
|
|
||||||
- add_kubernetes_metadata:
|
|
||||||
host: ${NODE_NAME}
|
|
||||||
- dissect:
|
|
||||||
tokenizer: '%{timestamp} %{level} %{pid} --- [%{thread}] %{class} : [%{app_name->}] %{message}'
|
|
||||||
field: "message"
|
|
||||||
target_prefix: "mylog"
|
|
||||||
ignore_missing: true
|
|
||||||
overwrite_keys: true
|
|
||||||
- drop_fields:
|
|
||||||
fields: ["kubernetes.node.labels", "kubernetes.annotations"]
|
|
||||||
ignore_missing: true
|
|
||||||
|
|
||||||
# ---------- ↑ java语言的服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↑ ----------
|
|
||||||
|
|
||||||
|
|
||||||
# ---------- ↓ java语言的服务的Pod, email 项目自由文本格式日志 ↓ ----------
|
|
||||||
- condition:
|
|
||||||
and:
|
|
||||||
- equals:
|
|
||||||
kubernetes.namespace: sit
|
|
||||||
- equals:
|
|
||||||
kubernetes.labels.app: "flymoon-email"
|
|
||||||
config:
|
|
||||||
- type: filestream
|
|
||||||
id: "container-${data.kubernetes.container.id}"
|
|
||||||
prospector.scanner.symlinks: true
|
|
||||||
close.on_state_change.removed: false
|
|
||||||
parsers:
|
|
||||||
- container: ~
|
|
||||||
- multiline:
|
|
||||||
type: pattern
|
|
||||||
pattern: '^\d{4}-\d{2}-\d{2}'
|
|
||||||
negate: true
|
|
||||||
match: after
|
|
||||||
paths:
|
|
||||||
- /var/log/containers/*-${data.kubernetes.container.id}.log
|
|
||||||
processors:
|
|
||||||
- add_kubernetes_metadata:
|
|
||||||
host: ${NODE_NAME}
|
|
||||||
- dissect:
|
|
||||||
tokenizer: '%{timestamp} %{level} %{pid} --- [%{thread}] %{class} : %{message}'
|
|
||||||
field: "message"
|
|
||||||
target_prefix: "mylog"
|
|
||||||
ignore_missing: true
|
|
||||||
overwrite_keys: true
|
|
||||||
- drop_fields:
|
|
||||||
fields: ["kubernetes.node.labels", "kubernetes.annotations"]
|
|
||||||
ignore_missing: true
|
|
||||||
# ---------- ↑ java语言的服务的Pod, email 项目自由文本格式日志 ↑ ----------
|
|
||||||
|
|
||||||
|
|
||||||
# ---------- ↓ python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↓ ----------
|
|
||||||
- condition:
|
|
||||||
and:
|
|
||||||
- equals:
|
|
||||||
kubernetes.namespace: sit
|
|
||||||
- equals:
|
|
||||||
kubernetes.labels.app: "lessie-agents"
|
|
||||||
config:
|
|
||||||
- type: filestream
|
|
||||||
id: "container-${data.kubernetes.container.id}"
|
|
||||||
prospector.scanner.symlinks: true
|
|
||||||
close.on_state_change.removed: false
|
|
||||||
parsers:
|
|
||||||
- container: ~
|
|
||||||
paths:
|
|
||||||
- /var/log/containers/*-${data.kubernetes.container.id}.log
|
|
||||||
processors:
|
|
||||||
- add_kubernetes_metadata:
|
|
||||||
host: ${NODE_NAME}
|
|
||||||
# 第一层：仅解析符合时间戳开头的日志行(for业务告警的日志格式)
|
|
||||||
- dissect:
|
|
||||||
when:
|
|
||||||
regexp:
|
|
||||||
message: '^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}.*'
|
|
||||||
tokenizer: '%{timestamp} - %{level} - %{module} - %{function} - %{msg_body}'
|
|
||||||
field: "message"
|
|
||||||
target_prefix: "mylog"
|
|
||||||
ignore_missing: true
|
|
||||||
overwrite_keys: true
|
|
||||||
# 第二层：针对带有 [level: | event: | msg: | context:] 的日志，再做一次 dissect
|
|
||||||
- dissect:
|
|
||||||
when:
|
|
||||||
contains:
|
|
||||||
mylog.msg_body: "[level:"
|
|
||||||
tokenizer: '[level: %{event_level} | event: %{event} | msg: %{msg} | context: %{ctx_raw}]'
|
|
||||||
field: "mylog.msg_body"
|
|
||||||
target_prefix: "mylog"
|
|
||||||
ignore_missing: true
|
|
||||||
overwrite_keys: true
|
|
||||||
# 第三层：把 ctx_raw 再拆成独立字段
|
|
||||||
- script:
|
|
||||||
lang: javascript
|
|
||||||
id: parse_context
|
|
||||||
source: >
|
|
||||||
function process(event) {
|
|
||||||
var ctx = event.Get("mylog.ctx_raw");
|
|
||||||
if (!ctx) return;
|
|
||||||
var parts = ctx.trim().split(",");
|
|
||||||
for (var i = 0; i < parts.length; i++) {
|
|
||||||
var pair = parts[i].split(":");
|
|
||||||
if (pair.length === 2) {
|
|
||||||
event.Put("mylog." + pair[0].trim(), pair[1].trim());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
# 第四层: 去除大量不需要的k8s元数据字段
|
|
||||||
- drop_fields:
|
|
||||||
fields:
|
|
||||||
- "kubernetes.node.labels"
|
|
||||||
- "kubernetes.annotations"
|
|
||||||
ignore_missing: true
|
|
||||||
# ---------- ↑ python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↑ ----------
|
|
||||||
|
|
||||||
|
|
||||||
# ---------- ↓ apex 动态创建的 python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↓ ----------
|
|
||||||
- condition:
|
|
||||||
and:
|
|
||||||
- equals:
|
|
||||||
kubernetes.namespace: apex-evaluation
|
|
||||||
- equals:
|
|
||||||
kubernetes.labels.apex: "lessie-agents"
|
|
||||||
config:
|
|
||||||
- type: filestream
|
|
||||||
id: "container-${data.kubernetes.container.id}"
|
|
||||||
prospector.scanner.symlinks: true
|
|
||||||
close.on_state_change.removed: false
|
|
||||||
parsers:
|
|
||||||
- container: ~
|
|
||||||
paths:
|
|
||||||
- /var/log/containers/*-${data.kubernetes.container.id}.log
|
|
||||||
processors:
|
|
||||||
- drop_fields:
|
|
||||||
fields:
|
|
||||||
- "kubernetes.node.labels"
|
|
||||||
- "kubernetes.annotations"
|
|
||||||
ignore_missing: true
|
|
||||||
# ---------- ↑ apex 动态创建的 python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↑ ----------
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
# ---- 输出到 Elasticsearch ----
|
|
||||||
output.elasticsearch:
|
|
||||||
hosts: ["http://10.0.0.38:9200"]
|
|
||||||
username: "admin"
|
|
||||||
password: "G7ZSKFM4AQwHQpwA"
|
|
||||||
|
|
||||||
indices:
|
|
||||||
- index: "k8s-%{[kubernetes.labels.environment]}-%{[kubernetes.labels.app]}-%{+yyyy.MM.dd}"
|
|
||||||
when:
|
|
||||||
regexp:
|
|
||||||
kubernetes.labels.app: "(lessie-go-api|flymoon-admin|flymoon-agent|flymoon-payment|flymoon-email|lessie-agents|apex)"
|
|
||||||
|
|
||||||
- index: "apex-python-%{[kubernetes.pod.name]}"
|
|
||||||
when:
|
|
||||||
equals:
|
|
||||||
kubernetes.labels.apex: "lessie-agents"
|
|
||||||
|
|
||||||
logging.level: info
|
|
||||||
logging.selectors: ["*"]
|
|
||||||
@@ -1,3 +1,6 @@
|
|||||||
|
# 滚动更新
|
||||||
|
# kubectl rollout restart daemonset filebeat -n kube-system
|
||||||
|
|
||||||
apiVersion: apps/v1
|
apiVersion: apps/v1
|
||||||
kind: DaemonSet
|
kind: DaemonSet
|
||||||
metadata:
|
metadata:
|
||||||
|
|||||||
@@ -4,27 +4,205 @@ setup.template.enabled: false
|
|||||||
filebeat.autodiscover:
|
filebeat.autodiscover:
|
||||||
providers:
|
providers:
|
||||||
- type: kubernetes
|
- type: kubernetes
|
||||||
node: ${NODE_NAME}
|
|
||||||
# hints.enabled: false
|
|
||||||
templates:
|
templates:
|
||||||
# ---------- go语言的中转服务的Pod, go项目json格式日志 ----------
|
# ---------- ↓ json格式日志 ↓ ----------
|
||||||
- condition:
|
- condition:
|
||||||
equals:
|
and:
|
||||||
kubernetes.labels.app: lessie-go-api
|
- regexp:
|
||||||
|
kubernetes.namespace: "^(sit|apex-evaluation)$"
|
||||||
|
- regexp:
|
||||||
|
kubernetes.labels.app: "^(lessie-go-api|apex)$"
|
||||||
config:
|
config:
|
||||||
- type: filestream
|
- type: filestream
|
||||||
id: "k8s-go-json-log-${data.kubernetes.container.id}"
|
id: "container-${data.kubernetes.container.id}"
|
||||||
prospector.scanner.symlinks: true
|
prospector.scanner.symlinks: true
|
||||||
close.on_state_change.removed: false
|
close.on_state_change.removed: false
|
||||||
parsers:
|
parsers:
|
||||||
- container: ~
|
- container: ~
|
||||||
paths:
|
paths:
|
||||||
- /var/log/containers/*-${data.kubernetes.container.id}.log
|
- /var/log/containers/*-${data.kubernetes.container.id}.log
|
||||||
|
processors:
|
||||||
|
- add_kubernetes_metadata:
|
||||||
|
host: ${NODE_NAME}
|
||||||
|
- decode_json_fields:
|
||||||
|
fields: ["message"]
|
||||||
|
target: "mylog"
|
||||||
|
overwrite_keys: true
|
||||||
|
add_error_key: true
|
||||||
|
- drop_fields:
|
||||||
fields:
|
fields:
|
||||||
application: ${data.kubernetes.labels.app}
|
- "kubernetes.node.labels"
|
||||||
log_type: "goho.log"
|
- "kubernetes.namespace_labels.kubernetes_io/metadata_name"
|
||||||
environment: ${data.kubernetes.labels.environment}
|
ignore_missing: true
|
||||||
instance: ${data.kubernetes.host}
|
# ---------- ↑ json格式日志 ↑ ----------
|
||||||
|
|
||||||
|
|
||||||
|
# ---------- ↓ java语言的服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↓ ----------
|
||||||
|
- condition:
|
||||||
|
and:
|
||||||
|
- equals:
|
||||||
|
kubernetes.namespace: sit
|
||||||
|
- or:
|
||||||
|
- equals:
|
||||||
|
kubernetes.labels.app: "flymoon-admin"
|
||||||
|
- equals:
|
||||||
|
kubernetes.labels.app: "flymoon-agent"
|
||||||
|
- equals:
|
||||||
|
kubernetes.labels.app: "flymoon-payment"
|
||||||
|
config:
|
||||||
|
- type: filestream
|
||||||
|
id: "container-${data.kubernetes.container.id}"
|
||||||
|
prospector.scanner.symlinks: true
|
||||||
|
close.on_state_change.removed: false
|
||||||
|
parsers:
|
||||||
|
- container: ~
|
||||||
|
- multiline:
|
||||||
|
type: pattern
|
||||||
|
pattern: '^\d{4}-\d{2}-\d{2}-\d{2}:\d{2}:\d{2}\.\d{3}'
|
||||||
|
negate: true
|
||||||
|
match: after
|
||||||
|
paths:
|
||||||
|
- /var/log/containers/*-${data.kubernetes.container.id}.log
|
||||||
|
processors:
|
||||||
|
- add_kubernetes_metadata:
|
||||||
|
host: ${NODE_NAME}
|
||||||
|
- dissect:
|
||||||
|
tokenizer: '%{timestamp} %{level} %{pid} --- [%{thread}] %{class} : [%{app_name->}] %{message}'
|
||||||
|
field: "message"
|
||||||
|
target_prefix: "mylog"
|
||||||
|
ignore_missing: true
|
||||||
|
overwrite_keys: true
|
||||||
|
- drop_fields:
|
||||||
|
fields: ["kubernetes.node.labels", "kubernetes.annotations"]
|
||||||
|
ignore_missing: true
|
||||||
|
|
||||||
|
# ---------- ↑ java语言的服务的Pod, agnet\admin\payment 项目自由文本格式日志 ↑ ----------
|
||||||
|
|
||||||
|
|
||||||
|
# ---------- ↓ java语言的服务的Pod, email 项目自由文本格式日志 ↓ ----------
|
||||||
|
- condition:
|
||||||
|
and:
|
||||||
|
- equals:
|
||||||
|
kubernetes.namespace: sit
|
||||||
|
- equals:
|
||||||
|
kubernetes.labels.app: "flymoon-email"
|
||||||
|
config:
|
||||||
|
- type: filestream
|
||||||
|
id: "container-${data.kubernetes.container.id}"
|
||||||
|
prospector.scanner.symlinks: true
|
||||||
|
close.on_state_change.removed: false
|
||||||
|
parsers:
|
||||||
|
- container: ~
|
||||||
|
- multiline:
|
||||||
|
type: pattern
|
||||||
|
pattern: '^\d{4}-\d{2}-\d{2}'
|
||||||
|
negate: true
|
||||||
|
match: after
|
||||||
|
paths:
|
||||||
|
- /var/log/containers/*-${data.kubernetes.container.id}.log
|
||||||
|
processors:
|
||||||
|
- add_kubernetes_metadata:
|
||||||
|
host: ${NODE_NAME}
|
||||||
|
- dissect:
|
||||||
|
tokenizer: '%{timestamp} %{level} %{pid} --- [%{thread}] %{class} : %{message}'
|
||||||
|
field: "message"
|
||||||
|
target_prefix: "mylog"
|
||||||
|
ignore_missing: true
|
||||||
|
overwrite_keys: true
|
||||||
|
- drop_fields:
|
||||||
|
fields: ["kubernetes.node.labels", "kubernetes.annotations"]
|
||||||
|
ignore_missing: true
|
||||||
|
# ---------- ↑ java语言的服务的Pod, email 项目自由文本格式日志 ↑ ----------
|
||||||
|
|
||||||
|
|
||||||
|
# ---------- ↓ python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↓ ----------
|
||||||
|
- condition:
|
||||||
|
and:
|
||||||
|
- equals:
|
||||||
|
kubernetes.namespace: sit
|
||||||
|
- equals:
|
||||||
|
kubernetes.labels.app: "lessie-agents"
|
||||||
|
config:
|
||||||
|
- type: filestream
|
||||||
|
id: "container-${data.kubernetes.container.id}"
|
||||||
|
prospector.scanner.symlinks: true
|
||||||
|
close.on_state_change.removed: false
|
||||||
|
parsers:
|
||||||
|
- container: ~
|
||||||
|
paths:
|
||||||
|
- /var/log/containers/*-${data.kubernetes.container.id}.log
|
||||||
|
processors:
|
||||||
|
- add_kubernetes_metadata:
|
||||||
|
host: ${NODE_NAME}
|
||||||
|
# 第一层：仅解析符合时间戳开头的日志行(for业务告警的日志格式)
|
||||||
|
- dissect:
|
||||||
|
when:
|
||||||
|
regexp:
|
||||||
|
message: '^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}.*'
|
||||||
|
tokenizer: '%{timestamp} - %{level} - %{module} - %{function} - %{msg_body}'
|
||||||
|
field: "message"
|
||||||
|
target_prefix: "mylog"
|
||||||
|
ignore_missing: true
|
||||||
|
overwrite_keys: true
|
||||||
|
# 第二层：针对带有 [level: | event: | msg: | context:] 的日志，再做一次 dissect
|
||||||
|
- dissect:
|
||||||
|
when:
|
||||||
|
contains:
|
||||||
|
mylog.msg_body: "[level:"
|
||||||
|
tokenizer: '[level: %{event_level} | event: %{event} | msg: %{msg} | context: %{ctx_raw}]'
|
||||||
|
field: "mylog.msg_body"
|
||||||
|
target_prefix: "mylog"
|
||||||
|
ignore_missing: true
|
||||||
|
overwrite_keys: true
|
||||||
|
# 第三层：把 ctx_raw 再拆成独立字段
|
||||||
|
- script:
|
||||||
|
lang: javascript
|
||||||
|
id: parse_context
|
||||||
|
source: >
|
||||||
|
function process(event) {
|
||||||
|
var ctx = event.Get("mylog.ctx_raw");
|
||||||
|
if (!ctx) return;
|
||||||
|
var parts = ctx.trim().split(",");
|
||||||
|
for (var i = 0; i < parts.length; i++) {
|
||||||
|
var pair = parts[i].split(":");
|
||||||
|
if (pair.length === 2) {
|
||||||
|
event.Put("mylog." + pair[0].trim(), pair[1].trim());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
# 第四层: 去除大量不需要的k8s元数据字段
|
||||||
|
- drop_fields:
|
||||||
|
fields:
|
||||||
|
- "kubernetes.node.labels"
|
||||||
|
- "kubernetes.annotations"
|
||||||
|
ignore_missing: true
|
||||||
|
# ---------- ↑ python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↑ ----------
|
||||||
|
|
||||||
|
|
||||||
|
# ---------- ↓ apex 动态创建的 python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↓ ----------
|
||||||
|
- condition:
|
||||||
|
and:
|
||||||
|
- equals:
|
||||||
|
kubernetes.namespace: apex-evaluation
|
||||||
|
- equals:
|
||||||
|
kubernetes.labels.apex: "lessie-agents"
|
||||||
|
config:
|
||||||
|
- type: filestream
|
||||||
|
id: "container-${data.kubernetes.container.id}"
|
||||||
|
prospector.scanner.symlinks: true
|
||||||
|
close.on_state_change.removed: false
|
||||||
|
parsers:
|
||||||
|
- container: ~
|
||||||
|
paths:
|
||||||
|
- /var/log/containers/*-${data.kubernetes.container.id}.log
|
||||||
|
processors:
|
||||||
|
- drop_fields:
|
||||||
|
fields:
|
||||||
|
- "kubernetes.node.labels"
|
||||||
|
- "kubernetes.annotations"
|
||||||
|
ignore_missing: true
|
||||||
|
# ---------- ↑ apex 动态创建的 python语言的agents服务的Pod, lessie-agents 项目自由文本格式日志 ↑ ----------
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
# ---- 输出到 Elasticsearch ----
|
# ---- 输出到 Elasticsearch ----
|
||||||
@@ -32,7 +210,17 @@ output.elasticsearch:
|
|||||||
hosts: ["http://10.0.0.38:9200"]
|
hosts: ["http://10.0.0.38:9200"]
|
||||||
username: "admin"
|
username: "admin"
|
||||||
password: "G7ZSKFM4AQwHQpwA"
|
password: "G7ZSKFM4AQwHQpwA"
|
||||||
index: "k8s-%{[kubernetes.labels.environment]}-%{[kubernetes.labels.app]}-%{+yyyy.MM.dd}"
|
|
||||||
|
|
||||||
logging.level: debug
|
indices:
|
||||||
logging.selectors: ["autodiscover", "input"]
|
- index: "k8s-%{[kubernetes.labels.environment]}-%{[kubernetes.labels.app]}-%{+yyyy.MM.dd}"
|
||||||
|
when:
|
||||||
|
regexp:
|
||||||
|
kubernetes.labels.app: "(lessie-go-api|flymoon-admin|flymoon-agent|flymoon-payment|flymoon-email|lessie-agents|apex)"
|
||||||
|
|
||||||
|
- index: "apex-python-%{+yyyy.MM.dd}"
|
||||||
|
when:
|
||||||
|
equals:
|
||||||
|
kubernetes.labels.apex: "lessie-agents"
|
||||||
|
|
||||||
|
logging.level: info
|
||||||
|
logging.selectors: ["*"]
|
||||||
Reference in New Issue
Block a user