pipeline { agent any environment { REMOTE_HOST = "43.130.56.138" NGINX_HOST = "49.51.46.148" REMOTE_PROJECT_PATH = "/data/webapps/lessie_sourcing_agents" VENV_DIR = "/data/webapps/lessie_sourcing_agents/venv" REMOTE_PROJECT_PATH_02 = "/data/webapps/lessie_sourcing_agents_02" VENV_DIR_02 = "/data/webapps/lessie_sourcing_agents_02/venv" CONDA_PATH = "/root/miniconda3/bin/conda" PORT_A = "8000" PORT_B = "8002" CONNECTION_TIMEOUT = "500" // 等待连接关闭的超时时间(秒) NGINX_RELOAD_SCRIPT = "/data/sh/set_s1_backend_weight.sh" // 设置nginx后端组up/down WAIT_CONNECTIONS_SCRIPT = "/data/sh/wait_s1_for_connections.sh" // 等待连接数变为0 CHECK_STATUS_SCRIPT = "/data/sh/check_s1_backend_status.sh" // 检查后端组状态 } stages { stage('Checkout 代码') { steps { git branch: "${params.Code_branch}", credentialsId: 'fly_gitlab_auth', url: 'http://172.24.16.20/python/lessie-sourcing-agents.git' } } stage('A脱离后端组') { steps { echo "将实例A的权重设置为down,停止接收新连接" sh "ssh ${NGINX_HOST} '${NGINX_RELOAD_SCRIPT} ${PORT_A} down'" echo "等待实例A的现有连接关闭" sh "ssh ${REMOTE_HOST} '${WAIT_CONNECTIONS_SCRIPT} ${PORT_A} ${CONNECTION_TIMEOUT}'" } } stage('A下线&同步') { steps { echo("下线A后同步A") sh "ssh ${REMOTE_HOST} 'sh /data/sh/kill_lessie_sourcing_agents.sh'" sh """ ssh ${REMOTE_HOST} 'mkdir -p ${REMOTE_PROJECT_PATH}' rsync -avz --exclude 'venv' ${WORKSPACE}/ ${REMOTE_HOST}:${REMOTE_PROJECT_PATH}/ """ } } stage('A依赖&启动') { steps { echo("A依赖&启动") sh """ ssh ${REMOTE_HOST} ' cd ${REMOTE_PROJECT_PATH} source ~/.bashrc conda activate search source ${VENV_DIR}/bin/activate pip install --upgrade pip pip install -r requirements.txt TIMESTAMP=\$(date +"%Y%m%d_%H%M%S") LOGFILE="${REMOTE_PROJECT_PATH}/logs/lessie_sourcing_agents_\${TIMESTAMP}.log" nohup env APP_ENV=sit python ${REMOTE_PROJECT_PATH}/server.py > ${REMOTE_PROJECT_PATH}/logs/lessie_sourcing_agents_\${TIMESTAMP}.log 2>&1 & ln -sf "\$LOGFILE" ${REMOTE_PROJECT_PATH}/logs/lessie_sourcing_agents_latest.log ' """ } } stage('恢复A流量') { steps { echo "恢复实例A的流量" sh "sleep 5" sh "ssh ${NGINX_HOST} '${NGINX_RELOAD_SCRIPT} ${PORT_A} up'" echo "实例 A 部署完成" } } stage('探测A服务 ') { steps { echo("探测A服务") sh "ssh ${REMOTE_HOST} 'sh /data/sh/check_lessie_agents_8000.sh' " } } stage('截B流量') { steps { echo "将实例B的权重设置为down,停止接收新连接" sh "ssh ${NGINX_HOST} '${NGINX_RELOAD_SCRIPT} ${PORT_B} down'" echo "等待实例B的现有连接关闭" sh "ssh ${REMOTE_HOST} '${WAIT_CONNECTIONS_SCRIPT} ${PORT_B} ${CONNECTION_TIMEOUT}'" } } stage('B下线&同步') { steps { echo("B下线&同步") sh "ssh ${REMOTE_HOST} 'sh /data/sh/kill_lessie_sourcing_agents_8002.sh'" sh """ ssh ${REMOTE_HOST} 'mkdir -p ${REMOTE_PROJECT_PATH_02}' rsync -avz --exclude 'venv' ${WORKSPACE}/ ${REMOTE_HOST}:${REMOTE_PROJECT_PATH_02}/ """ } } stage('B依赖&启动') { steps { echo("B依赖&启动") sh """ ssh ${REMOTE_HOST} ' cd ${REMOTE_PROJECT_PATH_02} && source ~/.bashrc && conda activate search && source ${VENV_DIR_02}/bin/activate && pip install --upgrade pip && pip install -r requirements.txt TIMESTAMP_02=\$(date +"%Y%m%d_%H%M%S") LOGFILE="${REMOTE_PROJECT_PATH_02}/logs/lessie_sourcing_agents_\${TIMESTAMP_02}.log" nohup env APP_ENV=sit python ${REMOTE_PROJECT_PATH_02}/server8002.py --port 8002 > ${REMOTE_PROJECT_PATH_02}/logs/lessie_sourcing_agents_\${TIMESTAMP_02}.log 2>&1 & ln -sf "\$LOGFILE" ${REMOTE_PROJECT_PATH_02}/logs/lessie_sourcing_agents_latest.log ' """ } } stage('恢复B流量') { steps { echo "恢复实例B的流量" sh "sleep 5" sh "ssh ${NGINX_HOST} '${NGINX_RELOAD_SCRIPT} ${PORT_B} up'" echo "实例 B 部署完成" } } stage('探测B服务 ') { steps { echo("探测B服务") sh "ssh ${REMOTE_HOST} 'sh /data/sh/check_lessie_agents_8002.sh' " } } } post { success { echo '✅ 部署成功!' } failure { echo '❌ 部署失败,请检查日志!' } } } # ==========================原本======= pipeline { agent any environment { REMOTE_HOST = "43.130.56.138" NGINX_HOST = "49.51.46.148" REMOTE_PROJECT_PATH = "/data/webapps/lessie_sourcing_agents" VENV_DIR = "/data/webapps/lessie_sourcing_agents/venv" REMOTE_PROJECT_PATH_02 = "/data/webapps/lessie_sourcing_agents_02" VENV_DIR_02 = "/data/webapps/lessie_sourcing_agents_02/venv" CONDA_PATH = "/root/miniconda3/bin/conda" PORT_A = "8000" PORT_B = "8002" CONNECTION_TIMEOUT = "500" // 等待连接关闭的超时时间(秒) NGINX_RELOAD_SCRIPT = "/data/sh/set_s1_backend_weight.sh" // 设置nginx后端组up/down WAIT_CONNECTIONS_SCRIPT = "/data/sh/wait_s1_for_connections.sh" // 等待连接数变为0 CHECK_STATUS_SCRIPT = "/data/sh/check_s1_backend_status.sh" // 检查后端组状态 } stages { stage('Checkout 代码') { steps { git branch: "${params.Code_branch}", credentialsId: 'fly_gitlab_auth', url: 'http://172.24.16.20/python/lessie-sourcing-agents.git' } } stage('部署实例A') { steps { script { // 部署前检查:确保B节点处于up状态(避免单节点部署时服务不可用) checkOtherNodeStatus(env.PORT_B, "A") // 执行部署,若失败则进入post的failure块回滚 deployInstance("A", env.PORT_A, env.REMOTE_PROJECT_PATH, env.VENV_DIR) } } post { failure { echo "实例A部署失败,开始回滚..." rollbackInstance("A", env.PORT_A) // 回滚A节点状态 } } } stage('部署实例B') { steps { script { // 部署前检查:确保A节点处于up状态 checkOtherNodeStatus(env.PORT_A, "B") // 执行部署,若失败则进入post的failure块回滚 deployInstance("B", env.PORT_B, env.REMOTE_PROJECT_PATH_02, env.VENV_DIR_02) } } post { failure { echo "实例B部署失败,开始回滚..." rollbackInstance("B", env.PORT_B) // 回滚B节点状态 } } } } post { success { echo '✅ 部署成功!' } failure { echo '❌ 部署失败,请检查日志!' } } } // 部署前检查:确保其他节点处于up状态,避免服务中断 def checkOtherNodeStatus(otherPort, currentInstance) { echo "部署${currentInstance}前,检查另一节点(端口${otherPort})是否可用..." def status = sh( script: "ssh ${env.NGINX_HOST} '${env.CHECK_STATUS_SCRIPT} ${otherPort}'", returnStdout: true ).trim() if (status != "up") { error "另一节点(端口${otherPort})当前状态为${status},无法部署${currentInstance}(避免服务不可用)" } echo "另一节点(端口${otherPort})状态正常(up),可部署${currentInstance}" } // 定义部署单个实例的函数 def deployInstance(instanceName, port, projectPath, venvDir) { echo "开始部署实例 $instanceName (端口 $port)" // 步骤1: 将实例权重设置为down,停止接收新连接 echo "将实例 $instanceName 的权重设置为down,停止接收新连接" sh "ssh ${NGINX_HOST} '${NGINX_RELOAD_SCRIPT} ${port} down'" // 步骤2: 等待现有连接关闭 echo "等待实例 $instanceName 的现有连接关闭" sh "ssh ${REMOTE_HOST} '${WAIT_CONNECTIONS_SCRIPT} ${port} ${CONNECTION_TIMEOUT}'" // 步骤3: 停止实例 echo "停止实例 $instanceName" if (instanceName == "A") { sh "ssh ${REMOTE_HOST} 'sh /data/sh/kill_lessie_sourcing_agents.sh'" } else { sh "ssh ${REMOTE_HOST} 'sh /data/sh/kill_lessie_sourcing_agents_8002.sh'" } // 步骤4: 同步代码 echo "同步实例 $instanceName 的代码" sh """ ssh ${REMOTE_HOST} 'mkdir -p ${projectPath}' rsync -avz --exclude 'venv' ${WORKSPACE}/ ${REMOTE_HOST}:${projectPath}/ """ // 步骤5: 安装依赖 echo "安装实例 $instanceName 的依赖" sh """ ssh ${REMOTE_HOST} ' cd ${projectPath} && source ~/.bashrc && conda activate search && source ${venvDir}/bin/activate && pip install --upgrade pip && pip install -r requirements.txt ' """ // 步骤6: 启动实例 echo "启动实例 $instanceName" if (instanceName == "A") { sh """ ssh ${REMOTE_HOST} ' conda activate search source ${venvDir}/bin/activate TIMESTAMP=\$(date +"%Y%m%d_%H%M%S") LOGFILE="${projectPath}/logs/lessie_sourcing_agents_\${TIMESTAMP}.log" nohup env APP_ENV=sit python ${projectPath}/server.py > ${projectPath}/logs/lessie_sourcing_agents_\${TIMESTAMP}.log 2>&1 & ln -sf "\$LOGFILE" ${projectPath}/logs/lessie_sourcing_agents_latest.log ' """ } else { sh """ ssh ${REMOTE_HOST} ' conda activate search source ${venvDir}/bin/activate TIMESTAMP=\$(date +"%Y%m%d_%H%M%S") LOGFILE="${projectPath}/logs/lessie_sourcing_agents_\${TIMESTAMP}.log" nohup env APP_ENV=sit python ${projectPath}/server.py --port ${port} > ${projectPath}/logs/lessie_sourcing_agents_\${TIMESTAMP}.log 2>&1 & ln -sf "\$LOGFILE" ${projectPath}/logs/lessie_sourcing_agents_latest.log ' """ } // 步骤7: 等待服务启动并检查健康状态 echo "检查实例 $instanceName 的健康状态" sh "sleep 5" // 等待服务启动 sh "ssh ${REMOTE_HOST} 'sh /data/sh/check_lessie_agents_${port}.sh'" // 步骤8: 恢复流量 echo "恢复实例 $instanceName 的流量" sh "sleep 5" sh "ssh ${NGINX_HOST} '${NGINX_RELOAD_SCRIPT} ${port} up'" echo "实例 $instanceName 部署完成" } // 回滚函数:若部署失败,将节点恢复为up(确保服务可用) def rollbackInstance(instanceName, port) { echo "回滚实例${instanceName}(端口${port})状态..." // 无论原状态如何,失败后强制恢复为up(避免节点长期down) sh "ssh ${env.NGINX_HOST} '${env.NGINX_RELOAD_SCRIPT} ${port} up'" // 检查回滚后状态 def status = sh( script: "ssh ${env.NGINX_HOST} '${env.CHECK_STATUS_SCRIPT} ${port}'", returnStdout: true ).trim() if (status == "up") { echo "实例${instanceName}回滚成功,当前状态为up" } else { error "实例${instanceName}回滚失败,请手动检查nginx状态!" } } # -------------------------------------------------------------------------------------------------------------------------------------------- # -------------------------------------------------------------------------------------------------------------------------------------------- # -------------------------------------------------------------------------------------------------------------------------------------------- # -------------------------------------------------------------------------------------------------------------------------------------------- # -------------------------------------------------------------------------------------------------------------------------------------------- # -------------------------------------------------------------------------------------------------------------------------------------------- # -------------------------------------------------------------------------------------------------------------------------------------------- pipeline { agent any environment { REMOTE_HOST = "3.130.56.138" NGINX_HOST = "9.51.46.148" REMOTE_PROJECT_PATH = "/data/webapps/lessie_sourcing_agents" VENV_DIR = "/data/webapps/lessie_sourcing_agents/venv" REMOTE_PROJECT_PATH_02 = "/data/webapps/lessie_sourcing_agents_02" VENV_DIR_02 = "/data/webapps/lessie_sourcing_agents_02/venv" CONDA_PATH = "/root/miniconda3/bin/conda" PORT_A = "8000" PORT_B = "8002" CONNECTION_TIMEOUT = "500" // 等待连接关闭的超时时间(秒) NGINX_RELOAD_SCRIPT = "/data/sh/set_s1_backend_weight.sh" // 设置nginx后端组up/down WAIT_CONNECTIONS_SCRIPT = "/data/sh/wait_s1_for_connections.sh" // 等待连接数变为0 CHECK_STATUS_SCRIPT = "/data/sh/check_s1_backend_status.sh" // 检查后端组状态 } stages { stage('Checkout 代码') { steps { echo "🔍 开始拉取代码,分支:${params.Code_branch}" git branch: "${params.Code_branch}", credentialsId: 'fly_gitlab_auth', url: 'http://172.24.16.20/python/lessie-sourcing-agents.git' echo "✅ 代码拉取完成" } } stage('部署实例A') { stages { stage('A-前置检查:另一节点状态') { steps { script { checkOtherNodeStatus(env.PORT_B, "A") } } } stage('A-1. 设置实例A为down') { steps { echo "📌 开始将实例A(端口${PORT_A})设为down,停止接收新连接" sh label: "执行nginx down命令", script: "ssh ${NGINX_HOST} '${NGINX_RELOAD_SCRIPT} ${PORT_A} down'" echo "✅ 实例A已设为down" } } stage('A-2. 等待实例A连接关闭') { steps { echo "📌 等待实例A(端口${PORT_A})现有连接关闭,超时${CONNECTION_TIMEOUT}秒" sh label: "执行等待连接脚本", script: "ssh ${REMOTE_HOST} '${WAIT_CONNECTIONS_SCRIPT} ${PORT_A} ${CONNECTION_TIMEOUT}'" echo "✅ 实例A连接已关闭" } } stage('A-3. 停止实例A进程') { steps { echo "📌 停止实例A(端口${PORT_A})进程" sh label: "执行停止脚本", script: "ssh ${REMOTE_HOST} 'sh /data/sh/kill_lessie_sourcing_agents.sh'" echo "✅ 实例A进程已停止" } } stage('A-4. 同步实例A代码') { steps { echo "📌 同步代码到实例A目录:${REMOTE_PROJECT_PATH}" sh label: "执行rsync同步", script: """ ssh ${REMOTE_HOST} 'mkdir -p ${REMOTE_PROJECT_PATH}' rsync -avz --exclude 'venv' ${WORKSPACE}/ ${REMOTE_HOST}:${REMOTE_PROJECT_PATH}/ """ echo "✅ 实例A代码同步完成" } } stage('A-5. 安装实例A依赖') { steps { echo "📌 安装实例A依赖(目录:${REMOTE_PROJECT_PATH})" sh label: "执行pip安装", script: """ ssh ${REMOTE_HOST} ' cd ${REMOTE_PROJECT_PATH} && source ~/.bashrc && conda activate search && source ${VENV_DIR}/bin/activate && pip install --upgrade pip && pip install -r requirements.txt ' """ echo "✅ 实例A依赖安装完成" } } stage('A-6. 启动实例A') { steps { echo "📌 启动实例A(端口${PORT_A})" sh label: "执行启动命令", script: """ ssh ${REMOTE_HOST} ' conda activate search source ${VENV_DIR}/bin/activate TIMESTAMP=\$(date +"%Y%m%d_%H%M%S") LOGFILE="${REMOTE_PROJECT_PATH}/logs/lessie_sourcing_agents_\${TIMESTAMP}.log" nohup env APP_ENV=sit python ${REMOTE_PROJECT_PATH}/server.py > \$LOGFILE 2>&1 & ln -sf "\$LOGFILE" ${REMOTE_PROJECT_PATH}/logs/lessie_sourcing_agents_latest.log ' """ echo "✅ 实例A启动命令已执行" } } stage('A-7. 检查实例A健康状态') { steps { echo "📌 等待实例A(端口${PORT_A})启动并检查健康状态" sh label: "等待服务就绪", script: "sleep 5" sh label: "执行健康检查脚本", script: "ssh ${REMOTE_HOST} 'sh /data/sh/check_lessie_agents_${PORT_A}.sh'" echo "✅ 实例A健康检查通过" } } stage('A-8. 恢复实例A流量') { steps { echo "📌 恢复实例A(端口${PORT_A})流量(设为up)" sh label: "执行nginx up命令", script: "ssh ${NGINX_HOST} '${NGINX_RELOAD_SCRIPT} ${PORT_A} up'" echo "✅ 实例A流量已恢复" } } } post { failure { echo "❌ 实例A部署失败,开始回滚..." rollbackInstance("A", env.PORT_A) } } } stage('部署实例B') { stages { stage('B-前置检查:另一节点状态') { steps { script { checkOtherNodeStatus(env.PORT_A, "B") } } } stage('B-1. 设置实例B为down') { steps { echo "📌 开始将实例B(端口${PORT_B})设为down,停止接收新连接" sh label: "执行nginx down命令", script: "ssh ${NGINX_HOST} '${NGINX_RELOAD_SCRIPT} ${PORT_B} down'" echo "✅ 实例B已设为down" } } stage('B-2. 等待实例B连接关闭') { steps { echo "📌 等待实例B(端口${PORT_B})现有连接关闭,超时${CONNECTION_TIMEOUT}秒" sh label: "执行等待连接脚本", script: "ssh ${REMOTE_HOST} '${WAIT_CONNECTIONS_SCRIPT} ${PORT_B} ${CONNECTION_TIMEOUT}'" echo "✅ 实例B连接已关闭" } } stage('B-3. 停止实例B进程') { steps { echo "📌 停止实例B(端口${PORT_B})进程" sh label: "执行停止脚本", script: "ssh ${REMOTE_HOST} 'sh /data/sh/kill_lessie_sourcing_agents_8002.sh'" echo "✅ 实例B进程已停止" } } stage('B-4. 同步实例B代码') { steps { echo "📌 同步代码到实例B目录:${REMOTE_PROJECT_PATH_02}" sh label: "执行rsync同步", script: """ ssh ${REMOTE_HOST} 'mkdir -p ${REMOTE_PROJECT_PATH_02}' rsync -avz --exclude 'venv' ${WORKSPACE}/ ${REMOTE_HOST}:${REMOTE_PROJECT_PATH_02}/ """ echo "✅ 实例B代码同步完成" } } stage('B-5. 安装实例B依赖') { steps { echo "📌 安装实例B依赖(目录:${REMOTE_PROJECT_PATH_02})" sh label: "执行pip安装", script: """ ssh ${REMOTE_HOST} ' cd ${REMOTE_PROJECT_PATH_02} && source ~/.bashrc && conda activate search && source ${VENV_DIR_02}/bin/activate && pip install --upgrade pip && pip install -r requirements.txt ' """ echo "✅ 实例B依赖安装完成" } } stage('B-6. 启动实例B') { steps { echo "📌 启动实例B(端口${PORT_B})" sh label: "执行启动命令", script: """ ssh ${REMOTE_HOST} ' conda activate search source ${VENV_DIR_02}/bin/activate TIMESTAMP=\$(date +"%Y%m%d_%H%M%S") LOGFILE="${REMOTE_PROJECT_PATH_02}/logs/lessie_sourcing_agents_\${TIMESTAMP}.log" nohup env APP_ENV=sit python ${REMOTE_PROJECT_PATH_02}/server.py --port ${PORT_B} > \$LOGFILE 2>&1 & ln -sf "\$LOGFILE" ${REMOTE_PROJECT_PATH_02}/logs/lessie_sourcing_agents_latest.log ' """ echo "✅ 实例B启动命令已执行" } } stage('B-7. 检查实例B健康状态') { steps { echo "📌 等待实例B(端口${PORT_B})启动并检查健康状态" sh label: "等待服务就绪", script: "sleep 5" sh label: "执行健康检查脚本", script: "ssh ${REMOTE_HOST} 'sh /data/sh/check_lessie_agents_${PORT_B}.sh'" echo "✅ 实例B健康检查通过" } } stage('B-8. 恢复实例B流量') { steps { echo "📌 恢复实例B(端口${PORT_B})流量(设为up)" sh label: "执行nginx up命令", script: "ssh ${NGINX_HOST} '${NGINX_RELOAD_SCRIPT} ${PORT_B} up'" echo "✅ 实例B流量已恢复" } } } post { failure { echo "❌ 实例B部署失败,开始回滚..." rollbackInstance("B", env.PORT_B) } } } } post { success { echo '🎉 所有实例部署成功!' } failure { echo '❌ 部署失败,请查看上面的步骤日志定位问题!' } } } // 部署前检查:确保其他节点处于up状态,避免服务中断 def checkOtherNodeStatus(otherPort, currentInstance) { echo "🔍 部署${currentInstance}前,检查另一节点(端口${otherPort})是否可用..." def status = sh( label: "检查节点${otherPort}状态", script: "ssh ${env.NGINX_HOST} '${env.CHECK_STATUS_SCRIPT} ${otherPort}'", returnStdout: true ).trim() if (status != "up") { error "❌ 另一节点(端口${otherPort})当前状态为${status},无法部署${currentInstance}(避免服务不可用)" } echo "✅ 另一节点(端口${otherPort})状态正常(up),可部署${currentInstance}" } // 回滚函数:若部署失败,将节点恢复为up(确保服务可用) def rollbackInstance(instanceName, port) { echo "🔄 回滚实例${instanceName}(端口${port})状态..." sh label: "执行回滚up命令", script: "ssh ${env.NGINX_HOST} '${env.NGINX_RELOAD_SCRIPT} ${port} up'" // 检查回滚后状态 def status = sh( label: "检查回滚后状态", script: "ssh ${env.NGINX_HOST} '${env.CHECK_STATUS_SCRIPT} ${port}'", returnStdout: true ).trim() if (status == "up") { echo "✅ 实例${instanceName}回滚成功,当前状态为up" } else { error "❌ 实例${instanceName}回滚失败,请手动检查nginx状态!" } }