网站建设优化佛山,深圳南山区住房和建设局网站官网,和黄crm在线,兰州网站建设cheng业务环境介绍公司当前业务上线流程首先是通过nginx灰度#xff0c;dubbo-admin操作禁用#xff0c;然后发布上线主机#xff0c;发布成功后#xff0c;dubbo-admin启用#xff0c;nginx启用主机#xff1b;之前是通过手动操作#xff0c;很不方便#xff0c;本次优化为…业务环境介绍 公司当前业务上线流程首先是通过nginx灰度dubbo-admin操作禁用然后发布上线主机发布成功后dubbo-admin启用nginx启用主机之前是通过手动操作很不方便本次优化为pipeline方式实现自动发布需要saltstack api支持。pipeline发布流程图准备工作将saltstack端dubbo.py脚本部署好可通过salt直接调用将所有脚本分别放到对应的主机上面saltstack端、jenkins端jenkins提前准备好 gitlab、nginx ssh、saltapi 相关凭据mvn、jdk工具jenkins配置pipeline脚本// 此pipeline用于nginx dubbo结构的应用如没有dubbo删除dubbo相关步骤即可
// 提前定义
// gitlab、nginx ssh、saltapi 相关凭据
// mvnjdk等工具定义
// saltapi模板中servername地址// 以下所有变量根据实际情况修改
// 本项目的svn代码地址
def svnUrl https://svn****
// nginx ssh参数多个以逗号未分隔符
def nginxHosts 172.87.10.31,172.87.10.41
// zookeeper主机端口多个以逗号未分隔符
def zkHostPort 172.87.40.14:2181,172.87.40.24:2181
// jenkins发布机脚本绝对路径
def scriptAbsPath /app/jenkins_deploy/scripts
// saltstack 脚本基于salt base路径
def saltScriptPath jenkins_deploy/scripts
// FTP存储路径映射到saltstack的路径
def ftpSaltPath jenkins_deploy/jenkins_war_packages
// 项目包名称
def packageName app.war
// 目包相对路径相对于$workspace)
def source_file target/ ${packageName}
// 应用重启命令
def appRestartCommand /app/apache-tomcat-9.0.37/bin/restart.sh app
// 应用启动用户
def appStartUser app
// 构建命令
def buildCommand mvn clean package -P product war:war// saltapi模板
def Salt(salthost, saltfunc, saltargs) {result salt(authtype: pam, clientInterface: local( arguments: saltargs,function: saltfunc, target: salthost, targettype: list),credentialsId: saltapi,saveFile: true,servername: http://172.87.10.21:8000)return result
}pipeline {// 执行任务agent any// 定义工具tools {maven maven352jdk jdk1.8}stages {stage(Clone) {steps {checkout([$class: SubversionSCM, additionalCredentials: [], excludedCommitMessages: , excludedRegions: , excludedRevprop: , excludedUsers: , filterChangelog: false, ignoreDirPropChanges: false, includedRegions: , locations: [[credentialsId: jenkinsprd, depthOption: infinity, ignoreExternalsOption: true, local: ., remote: ${svnUrl}]], workspaceUpdater: [$class: UpdateUpdater]])}}// nginx 禁用upstream主机stage(Nginx disable host) {steps {Salt(${nginxHosts},cmd.script,salt://${saltScriptPath}/nginx_upstream.sh \disabled ${targetHosts} ${nginx_config_file}\)sh /usr/bin/python3 ${scriptAbsPath}/view_saltoutput.py ${WORKSPACE}/saltOutput.json ${nginxHosts} cmd.script}}// nginx 重新加载配置stage(Nginx reload 1) {input {message 是否重启nginxok 确定parameters {string(name: nginx, defaultValue: ${nginx_config_file}, description: 此文件有所改动请谨慎操作)}}steps {Salt(${nginxHosts},cmd.script,salt://${saltScriptPath}/nginx_reload.sh)sh /usr/bin/python3 ${scriptAbsPath}/view_saltoutput.py ${WORKSPACE}/saltOutput.json ${nginxHosts} cmd.script}}// dubbo 禁用生产者stage(Dubbo disable) {steps {script {// 获取主机注册的所有dubbo服务Salt(${targetHosts},dubbo.ls,${dubbo_port})dubbo_all_service sh(script: /usr/bin/python3 ${scriptAbsPath}/view_saltoutput.py ${WORKSPACE}/saltOutput.json ${targetHosts} dubbo.ls, returnStdout:true).trim()// dubbo禁用sh /usr/bin/python3 ${scriptAbsPath}/dubbo.py disable ${zkHostPort} ${targetHosts} ${dubbo_port} ${dubbo_all_service}}}}// mvn构建、代码发布到应用服务器stage(mvn build and deploy) {steps {// 构建命令sh ${buildCommand}// 需要修改war包所在的路径sh python3 ${scriptAbsPath}/ftp_upload.py ${WORKSPACE}/${source_file} ${source_file}// /tmp/路径不能修改这个路径要匹配restart.sh中配置的Salt(${targetHosts},state.sls,jenkins_deploy.scripts.sync_app_package pillar{package_name: ${packageName}})sh /usr/bin/python3 ${scriptAbsPath}/view_saltoutput.py ${WORKSPACE}/saltOutput.json ${targetHosts} cp.get_file// 项目包权限修改为项目启动用户Salt(${targetHosts},file.chown, /tmp/${packageName} ${appStartUser} ${appStartUser})}}// 重启应用服务stage(restart app service) {steps {// 应用重启命令Salt(${targetHosts},cmd.run,${appRestartCommand} env\{\LC_ALL\: \en_US.UTF-8\,\LANG\: \en_US.UTF-8\}\ runas${appStartUser})sh /usr/bin/python3 ${scriptAbsPath}/view_saltoutput.py ${WORKSPACE}/saltOutput.json ${targetHosts} cmd.run}}// 测试目标主机是否恢复stage(Test) {steps {println(Test ...)sleep 5}}// dubbo启用生产者stage(Dubbo enable) {input {message 是否启用dubbook 确定parameters {string(name: dubbo, defaultValue: ${targetHosts}-${dubbo_port}, description: 请确定应用正常启动谨慎操作)}}steps {script {// 获取主机注册的所有dubbo服务Salt(${targetHosts},dubbo.ls,${dubbo_port})dubbo_all_service sh(script: /usr/bin/python3 ${scriptAbsPath}/view_saltoutput.py ${WORKSPACE}/saltOutput.json ${targetHosts} dubbo.ls, returnStdout:true).trim()// dubbo启用sh /usr/bin/python3 ${scriptAbsPath}/dubbo.py enable ${zkHostPort} ${targetHosts} ${dubbo_port} ${dubbo_all_service}}}}// nginx 启用upstream主机stage(Nginx enable host) {steps {Salt(${nginxHosts},cmd.script,salt://${saltScriptPath}/nginx_upstream.sh \enabled ${targetHosts} ${nginx_config_file}\)sh /usr/bin/python3 ${scriptAbsPath}/view_saltoutput.py ${WORKSPACE}/saltOutput.json ${nginxHosts} cmd.script}}// nginx 重新加载配置stage(Nginx reload 2) {input {message 是否重启nginxok 确定parameters {string(name: nginx, defaultValue: ${nginx_config_file}, description: 此文件有所改动请谨慎操作)}}steps {Salt(${nginxHosts},cmd.script,salt://${saltScriptPath}/nginx_reload.sh)sh /usr/bin/python3 ${scriptAbsPath}/view_saltoutput.py ${WORKSPACE}/saltOutput.json ${nginxHosts} cmd.script}}}post {success {script {buildDescription 上次构建成功的主机${params.targetHosts}}}failure {script {// nginx 启用upstream主机Salt(${nginxHosts},cmd.script,salt://${saltScriptPath}/nginx_upstream.sh \enabled ${targetHosts} ${nginx_config_file}\)sh /usr/bin/python3 ${scriptAbsPath}/view_saltoutput.py ${WORKSPACE}/saltOutput.json ${nginxHosts} cmd.scriptbuildDescription 构建失败请查看错误}}aborted {script {// nginx 启用upstream主机Salt(${nginxHosts},cmd.script,salt://${saltScriptPath}/nginx_upstream.sh \enabled ${targetHosts} ${nginx_config_file}\)sh /usr/bin/python3 ${scriptAbsPath}/view_saltoutput.py ${WORKSPACE}/saltOutput.json ${nginxHosts} cmd.scriptbuildDescription 手动取消构建}}}
}saltstack server端主机上的脚本nginx_reload.sh#!/bin/bashsource /etc/profilenginx -t nginx -s reload sleep 3 ; ps -ef | grep nginx: | grep -v grepnginx_upstream.sh#!/bin/bash
# Filename : nginx_upstream.sh
# Date : 2021/08/26
# Author : beiguohao
# Email : oct_hao163.com
# Description: 本脚本用于jenkins pipeline发布代码过程中的禁用启用nginx主机操作SWITCH$1
HOSTS$2
FILE_NAME$3usage(){echo $0: [enable|enabled|disable|disabled] [hosts] [NGINX_UPSTREAM_CONFIG_FILE]
}# 启用nginx主机
enabled(){hosts$1file_name$2IFS$,for h in ${hosts}dosed -ri s/.*($h)/ server \1/g ${file_name}donecat ${file_name}
}# 禁用nginx主机
disabled(){hosts$1file_name$2IFS$,for h in ${hosts}dosed -ri s/.*($h)/# server \1/g ${file_name}donecat ${file_name}
}# 传递参数不为3或者任意参数为空则退出脚本
if [ $# -ne 3 -o -z $SWITCH -o -z $HOSTS -o -z $FILE_NAME ];thenusageexit 1
ficase $SWITCH inenable|enabled)enabled $HOSTS $FILE_NAME ;;disable|disabled)disabled $HOSTS $FILE_NAME;;*)usageexit 1;;
esacsync_app_package.sls{% set package_name pillar[package_name] %}sync_app_package:file.managed:- name: /tmp/{{ package_name }}- source: salt://jenkins_deploy/jenkins_war_packages/target/{{ package_name }}- user: zf- group: zf- mode: 644dubbo.py 此脚本放在saltstack base/_modules目录下面# -*- coding: utf-8 -*-
# Filename : nginx_upstream.sh
# Date : 2021/08/26
# Author : beiguohao
# Email : oct_hao163.com
# Description: 此脚本用于saltstack自定义模块from os.path import join as p_join
import telnetlib
import re__virtualname__ dubbo
finish dubbodef __virtual__():return __virtualname__def ls(port, host127.0.0.1, timeout5):获取应用注册到dubbo的所有服务的详细信息tn telnetlib.Telnet(hosthost, portport, timeouttimeout)tn.write(ls -l\n)res tn.read_until(finish).strip(finish)tn.close()res_l []# 遍历分割成列表的结果for service in res.strip(\r\n).split(\r\n):serviceName re.split(\s-, service)[0]pattern re.compile(group(.*?)\)d pattern.search(service)if d:res_l.append(p_join(d.group(1), serviceName))continueres_l.append(serviceName)return ,.join(res_l)test_url.sh#!/bin/bash
# Filename : test_url.sh
# Date : 2021/08/26
# Author : beiguohao
# Email : oct_hao163.com
# Description: 此脚本用于测试url是否可以访问URL$1
N0
MAX_N${2-9999}usage(){if [ $1 -gt 2 ] || [ $1 -lt 1 ];thenecho $0 [URL] [REQUEST_NUMBER]exit 1fi
}Curl(){code$(curl -s -o /dev/null -w %{http_code} $URL)while truedoif [ $N -ge $MAX_N ];thenecho Test Fail.exit 1elseif [ $code 200 ];thenecho Test $URL success, Code: $code.exit 0fifisleep 1N$((N1))done
}usage $#
Curl3. jenkins server端主机上面的脚本dubbo.py# -*- coding: utf-8 -*-
# Filename : dubbo.py
# Date : 2021/08/26
# Author : beiguohao
# Email : oct_hao163.com
# Description: 此脚本操作dubbo-admin中服务的启用和禁用from check_port_connect import check_port
from zookeeper import Zk
from os import path
import urllib.parse
import sysclass Dubbo:def __init__(self, zk_host, dubbo_host, dubbo_port, dubbo_all_service, dubbo_path/dubbo):self.zk_host zk_hostself.dubbo_host dubbo_hostself.dubbo_port dubbo_portself.dubbo_host_port [host :%s % self.dubbo_port for host in self.dubbo_host]self.dubbo_path dubbo_pathself.zk Zk(self.zk_host)self.all_node dubbo_all_service# 写入zookeeper对应dubbo的服务配置模板self.template override://%s/%s?categoryconfiguratorsdisabledtruedynamicfalseenabledtrueself._check_port()def _check_port(self):# 检测dubbo主机的端口连通性for host in self.dubbo_host:dubbo_port_test check_port(host, self.dubbo_port)if dubbo_port_test ! 0: sys.exit(1) def _format(self, zk_node, dubbo_host_port):# 分割服务名group/service 根号分割的名称是有组的服务l zk_node.split(/)# 大于1就是有组的服务if len(l) 1:zk_node l[1]override self.template % (dubbo_host_port, l[1])override group l[0]else:zk_node l[0]override self.template % (dubbo_host_port, l[0])return zk_node, overridedef disable(self):# 循环所有node并禁用所有dubbo主机的服务for zk_node in self.all_node:for host_port in self.dubbo_host_port:zk_node, override self._format(zk_node, host_port)d urllib.parse.urlencode({name: override}).split(name)[1]self.zk.create(%s/%s/configurators/%s % (self.dubbo_path, zk_node, d), b[])print(dubbo %s的服务已禁用请查看WEB页面 % ,.join(self.dubbo_host_port))self.zk.stop()def enable(self):# 循环所有node并禁用所有dubbo主机的服务for zk_node in self.all_node:for host_port in self.dubbo_host_port:zk_node, override self._format(zk_node, host_port)d urllib.parse.urlencode({name: override}).split(name)[1]self.zk.delete(%s/%s/configurators/%s % (self.dubbo_path, zk_node, d))print(dubbo %s的服务已启用请查看WEB页面 % ,.join(self.dubbo_host_port))self.zk.stop()def servic_status(self):for host_port in self.dubbo_host_port:n 0for zk_node in self.all_node:zk_node, override self._format(zk_node, host_port)d urllib.parse.urlencode({name: override}).split(name)[1]res self.zk.ls(%s/%s/configurators % (self.dubbo_path, zk_node))# 如果格式化后的值存在于列表中即为禁用if d not in res:n 1print(Host: %s, Total: %s, Enable: %s % (host_port, len(self.all_node), n))if __name__ __main__:usage usgae: %s [enable|disable|service_status] [zk_host:port,...] [dubbo_host] [dubbo_port] [dubbo_all_service] % sys.argv[0]if len(sys.argv) ! 6:sys.exit(usage)method sys.argv[1] zk_host sys.argv[2]dubbo_host sys.argv[3].split(,)dubbo_port sys.argv[4]dubbo_all_service sys.argv[5].split(,)dubbo Dubbo(zk_hostzk_host, dubbo_hostdubbo_host, dubbo_portdubbo_port, dubbo_all_servicedubbo_all_service)if method disable:dubbo.disable()elif method enable:dubbo.enable()elif method service_status:dubbo.servic_status()else:sys.exit(usage)zookeeper.py# -*- coding: utf-8 -*-
# Filename : zookeeper.py
# Date : 2021/08/26
# Author : beiguohao
# Email : oct_hao163.com
# Description: 此脚本用于连接、操作zookeeperfrom kazoo.client import KazooClient
from kazoo.exceptions import NoNodeError
from check_port_connect import check_port
from sys import exitclass Zk:def __init__(self, hosts, timeout10):self.hosts hostsself.timeout timeoutself._zk KazooClient(hostsself.hosts, timeoutself.timeout)self._zk.start()def ls(self, path/):获取Zk执行path中所有node:return: 所有nodetry:all_node self._zk.get_children(path)return all_nodeexcept Exception as e:print(repr(e))exit(1) def create(self, path, data):Zk创建:return:try:res self._zk.create(path, data, makepathTrue)return resexcept Exception as e:print(repr(e))exit(1)def delete(self, path):Zk 删除节点:param path: 要删除的节点:return:try:res self._zk.delete(path)return resexcept Exception as e:return repr(e)def stop(self):self._zk.stop()view_saltoutput.py # -*- coding: utf-8 -*-
# Filename : nginx_upstream.sh
# Date : 2021/08/26
# Author : beiguohao
# Email : oct_hao163.com
# Description: 此脚本用于查看saltstack api的输出结果因为saltapi中定义了saveFile: trueimport json, sysdef outPutJson(file_path, hosts, module):解析saltapi执行结果输出的json文件切记不同模块输出格式有所不同with open(file_path, r) as read_f:data json.load(read_f)for host in hosts:try:if module cmd.script:res (host, data[0][host][ret][stdout])elif module cmd.run:res (host (stdout), data[0][host][ret])elif module cp.get_file:res (host, data[0][host])if not res[1]:sys.exit(res)elif module dubbo.ls:res data[0][host]print(res)breakelse:res print(%s {\n%s\n} % res)except KeyError as e:print(KeyError: %s % e)sys.exit(1)if __name__ __main__:usage usage: %s [SALT_OUTPUT_FILE] [HOSTS] [MODULE] % sys.argv[0]if len(sys.argv) ! 4:sys.exit(usage) file_path sys.argv[1]hosts sys.argv[2].split(,)module sys.argv[3]outPutJson(file_path, hosts, module)ftp_upload.py# -*- coding: utf-8 -*-
# Filename : ftp_upload.py
# Date : 2021/08/26
# Author : beiguohao
# Email : oct_hao163.com
# Description: 此脚本用于操作ftpfrom ftplib import FTP
from os import path
import sysclass FTPClient:def __init__(self, host, user, passwd, port21):self.host hostself.user userself.passwd passwdself.port portself._ftp Noneself._login()def _login(self):登录FTP服务器:return: 连接或登录出现异常时返回错误信息try:self._ftp FTP()self._ftp.connect(self.host, self.port, timeout30)self._ftp.login(self.user, self.passwd)except Exception as e:print(str(e))sys.exit(1)def upload(self, localpath, remotepathNone):上传ftp文件:param localpath: local file path:param remotepath: remote file path:return:if not localpath: return Please select a local file. # 读取本地文件fp open(localpath, rb)# 如果未传递远程文件路径则上传到当前目录文件名称同本地文件if not remotepath:remotepath path.basename(localpath)# 上传文件try:self._ftp.storbinary(STOR remotepath, fp)except Exception as e:print(str(e))def nlst(self, dir/):查看目录下的内容:return: 以列表形式返回目录下的所有内容files_list self._ftp.nlst(dir)return files_listdef del_file(self, filenameNone):删除文件:param filename: 文件名称:return: 执行结果if not filename: return Please input filenametry:del_f self._ftp.delete(filename)except Exception as e:print(str(e))sys.exit(1)def close(self):退出ftp连接:return:try:# 向服务器发送quit命令self._ftp.quit()except Exception:print(No response from server)sys.exit(1)finally:# 客户端单方面关闭连接self._ftp.close()if __name__ __main__:usage usage: %s [FILE_NAME] [TARGET_PATH] % sys.argv[0]ftp_host 172.85.10.31ftp_user jenkins_deployftp_password jenkinsdeploy_2021if len(sys.argv) ! 3:sys.exit(usage)ftp FTPClient(hostftp_host, userftp_user, passwdftp_password)file_path sys.argv[1]target_path sys.argv[2]all_file ftp.nlst(target)if file_path.startswith(/):file_path path.join(/, file_path)f_path target/ path.basename(file_path)if f_path in all_file: ftp.del_file(f_path)ftp.upload(file_path, target_path)ftp.close()check_port_connect.py# -*- coding: utf-8 -*-
# Filename : check_port_connect.py
# Date : 2021/08/26
# Author : beiguohao
# Email : oct_hao163.com
# Description: 此脚本用于检测端口连通性import socket
import syssocket.setdefaulttimeout(10) def check_port(ip, port):port int(port)sk socket.socket(socket.AF_INET, socket.SOCK_STREAM)res sk.connect_ex((ip, port))if res 0:# print(%s %s port is Open % (ip, port))return 0else:print(%s %s port is Not Open % (ip, port))sys.exit(1) if __name__ __main__:usage usage: %s [IP|HSOTNAME] [PORT]if len(sys.argv) ! 3:sys.exit(usage) ip sys.argv[1] port sys.argv[2]check_port(ip, port)
