Pipeline 集成 Helm 发布微服务项目
一、Helm部署到K8S
通过刚刚的配置，如上4个阶段已经完成；
接下来：
二、推送代码到gitlab
# git clone http://192.168.171.10:9999/root/ms.git # 拉取代码
# cp simple-microservice-dev4/* ms/ -rf # copy代码
# cd ms
# git add .
# git commit -m "all"
# git push origin master
三、jenkins配置
3.1、安装组件
jenkins 组件搜索：
- Git Parameter
- Git
- Config File Provider
- Extended Choice Parameter3.2、添加凭据
添加凭据后保存，然后再次点击进入复制ID即可；
另外还需要添加一个kubeconfig配置文件：
“Manage Jenkins” ==> “Managed files” ==> “Add a new Config” ==> “Custom file” ==> 复制ID ==> “submit”
如果是kubeadm部署的，配置文件：
# cat /root/.kube/config# cd ansible-install-k8s-master/ssl/k8s/
或者手动生成：进入master的/opt/kubernetes/ssl目录：
一、生成管理员证书
# cat > admin-csr.json << EOF
{
"CN": "admin",
"hosts": [],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "BeiJing",
"ST": "BeiJing",
"O": "system:masters",
"OU": "System"
}
]
}
EOF
# cat > ca-config.json << EOF
{
"signing": {
"default": {
"expiry": "8760h"
},
"profiles": {
"kubernetes": {
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
],
"expiry": "8760h"
}
}
}
}
EOF
# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes admin-csr.json | cfssljson -bare admin
二、创建kubeconfig文件
# 设置集群参数
kubectl config set-cluster kubernetes \
--server=https://192.168.171.11:6443 \
--certificate-authority=ca.pem \
--embed-certs=true \
--kubeconfig=config
# 设置客户端认证参数
kubectl config set-credentials cluster-admin \
--certificate-authority=ca.pem \
--embed-certs=true \
--client-key=admin-key.pem \
--client-certificate=admin.pem \
--kubeconfig=config
# 设置上下文参数
kubectl config set-context default \
--cluster=kubernetes \
--user=cluster-admin \
--kubeconfig=config
# 设置默认上下文
kubectl config use-context default --kubeconfig=config3.3、Jenkins Pipeline配置
#!/usr/bin/env groovy
// 所需插件: Git Parameter/Git/Pipeline/Config File Provider/kubernetes/Extended Choice Parameter
// 公共
def registry = "192.168.171.10" # git仓库地址
// 项目
def project = "microservice" # 项目名称
def git_url = "http://192.168.171.10:9999/root/ms.git" # 项目的git地址
def gateway_domain_name = "gateway.zhdya.cn" # 项目对应的gateway域名（无，删除即可）
def portal_domain_name = "portal.zhdya.cn" # 项目对应的portal域名
// 认证（如果是放在公开的镜像目录中<library>则不需要，且如下均是保存在jenkins中引用的凭据ID）
def image_pull_secret = "registry-pull-secret"
def harbor_registry_auth = "7bb681c5-369b-4268-b35b-3d422e661fd4"
def git_auth = "43223e39-46d8-4d51-acb5-5a3a9c90400b"
// ConfigFileProvider ID
def k8s_auth = "7f92b4f5-9e3a-48fd-835b-c22935942317"
pipeline {
agent {
kubernetes {
label "jenkins-slave"
yaml """
kind: Pod
metadata:
name: jenkins-slave
spec:
containers:
- name: jnlp
image: "${registry}/library/jenkins-slave:jdk-1.8"
imagePullPolicy: Always
volumeMounts:
- name: docker-cmd # 引用外部的docker服务（数据卷方式）
mountPath: /usr/bin/docker
- name: docker-sock
mountPath: /var/run/docker.sock
- name: maven-cache
mountPath: /root/.m2 # 将应用编译所需的依赖全部存放在如下/tmp/m2中，后期的编译优先走缓存，提高速度！
volumes:
- name: docker-cmd
hostPath:
path: /usr/bin/docker
- name: docker-sock
hostPath:
path: /var/run/docker.sock
- name: maven-cache
hostPath:
path: /tmp/m2
"""
}
}
parameters {
gitParameter branch: '', branchFilter: '.*', defaultValue: '', description: '选择发布的分支', name: 'Branch', quickFilterEnabled: false, selectedValue: 'NONE', sortMode: 'NONE', tagFilter: '*', type: 'PT_BRANCH' # 动态获取git分支等；
extendedChoice defaultValue: 'none', description: '选择发布的微服务', \
multiSelectDelimiter: ',', name: 'Service', type: 'PT_CHECKBOX', \
value: 'gateway-service:9999,portal-service:8080,product-service:8010,order-service:8020,stock-service:8030' # 扩展多选参数，可以同时选择多个微服务部署；
choice (choices: ['ms', 'demo'], description: '部署模板', name: 'Template') # chart模板
choice (choices: ['1', '3', '5', '7'], description: '副本数', name: 'ReplicaCount')
choice (choices: ['ms'], description: '命名空间', name: 'Namespace')
}
stages {
stage('拉取代码'){
steps {
checkout([$class: 'GitSCM',
branches: [[name: "${params.Branch}"]],
doGenerateSubmoduleConfigurations: false,
extensions: [], submoduleCfg: [],
userRemoteConfigs: [[credentialsId: "${git_auth}", url: "${git_url}"]]
])
}
}
stage('代码编译') {
// 编译指定服务
steps {
sh """
mvn clean package -Dmaven.test.skip=true
"""
}
}
stage('构建镜像') {
steps {
withCredentials([usernamePassword(credentialsId: "${harbor_registry_auth}", passwordVariable: 'password', usernameVariable: 'username')]) {
sh """
docker login -u ${username} -p '${password}' ${registry}
for service in \$(echo ${Service} |sed 's/,/ /g'); do
service_name=\${service%:*} # 使用for进行多服务的循环
image_name=${registry}/${project}/\${service_name}:${BUILD_NUMBER}
cd \${service_name}
if ls |grep biz &>/dev/null; then
cd \${service_name}-biz
fi
docker build -t \${image_name} .
docker push \${image_name}
cd ${WORKSPACE}
done
"""
configFileProvider([configFile(fileId: "${k8s_auth}", targetLocation: "admin.kubeconfig")]){
sh """
# 添加镜像拉取认证
kubectl create secret docker-registry ${image_pull_secret} --docker-username=${username} --docker-password=${password} --docker-server=${registry} -n ${Namespace} --kubeconfig admin.kubeconfig |true
# 添加私有chart仓库
helm repo add --username ${username} --password ${password} myrepo http://${registry}/chartrepo/${project}
"""
}
}
}
}
stage('Helm部署到K8S') {
steps {
sh """
common_args="-n ${Namespace} --kubeconfig admin.kubeconfig"
for service in \$(echo ${Service} |sed 's/,/ /g'); do
service_name=\${service%:*}
service_port=\${service#*:}
image=${registry}/${project}/\${service_name}
tag=${BUILD_NUMBER}
helm_args="\${service_name} --set image.repository=\${image} --set image.tag=\${tag} --set replicaCount=${replicaCount} --set imagePullSecrets[0].name=${image_pull_secret} --set service.targetPort=\${service_port} myrepo/${Template}"
# 判断是否为新部署
if helm history \${service_name} \${common_args} &>/dev/null;then
action=upgrade
else
action=install
fi
# 针对服务启用ingress
if [ \${service_name} == "gateway-service" ]; then
helm \${action} \${helm_args} \
--set ingress.enabled=true \
--set ingress.host=${gateway_domain_name} \
\${common_args}
elif [ \${service_name} == "portal-service" ]; then
helm \${action} \${helm_args} \
--set ingress.enabled=true \
--set ingress.host=${portal_domain_name} \
\${common_args}
else
helm \${action} \${helm_args} \${common_args}
fi
done
# 查看Pod状态
sleep 10
kubectl get pods \${common_args}
"""
}
}
}
}插件注解：
Git Parameter 动态从git中获取所有分支
Git 拉取代码
Pipeline 流水线
Config File Provider 将配置文件由jenkins存储（kubeconfig文件<为什么不直接打在镜像中？不安全！>），pipeline引用（kubectl，helm）
kubernetes 动态创建代理
Extended Choice Parameter 扩展参数构建（多选）一旦配置之后，第一次的构建jenkins并不会真正的走逻辑，而是先检查其中的pipeline语法，一旦第一次完全成功后才出现如上发布的信息：
假如开发在git中创建了个分支： 
先来部署一个gateway服务，我来截取一个gateway发布的过程：（看了这个对应如上的pipeline就明白其中的配置了）
[INFO] BUILD SUCCESS
[INFO] ------------------------------------------------------------------------
[INFO] Total time: 15.645s
[INFO] Finished at: Mon Jan 27 11:54:20 UTC 2020
[INFO] Final Memory: 63M/151M
[INFO] ------------------------------------------------------------------------
[Pipeline] }
[Pipeline] // stage
[Pipeline] stage
[Pipeline] { (构建镜像)
[Pipeline] withCredentials
Masking supported pattern matches of $username or $password
[Pipeline] {
[Pipeline] sh
+ docker login -u **** -p **** 192.168.171.10
WARNING! Using --password via the CLI is insecure. Use --password-stdin.
WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
Configure a credential helper to remove this warning. See
https://docs.docker.com/engine/reference/commandline/login/#credentials-store
Login Succeeded
++ echo gateway-service:9999
++ sed 's/,/ /g'
+ for service in '$(echo gateway-service:9999 |sed '\''s/,/ /g'\'')'
+ service_name=gateway-service
+ image_name=192.168.171.10/microservice/gateway-service:4
+ cd gateway-service
+ grep biz
+ ls
+ docker build -t 192.168.171.10/microservice/gateway-service:4 .
Sending build context to Docker daemon 62.39MB
Step 1/6 : FROM java:8-jdk-alpine
8-jdk-alpine: Pulling from library/java
709515475419: Pulling fs layer
38a1c0aaa6fd: Pulling fs layer
5b58c996e33e: Pulling fs layer
38a1c0aaa6fd: Verifying Checksum
38a1c0aaa6fd: Download complete
709515475419: Verifying Checksum
709515475419: Download complete
709515475419: Pull complete
38a1c0aaa6fd: Pull complete
5b58c996e33e: Verifying Checksum
5b58c996e33e: Download complete
5b58c996e33e: Pull complete
Digest: sha256:d49bf8c44670834d3dade17f8b84d709e7db47f1887f671a0e098bafa9bae49f
Status: Downloaded newer image for java:8-jdk-alpine
---> 3fd9dd82815c
Step 2/6 : RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
---> Running in 8a24c4b5ba21
Removing intermediate container 8a24c4b5ba21
---> 31ab66935f61
Step 3/6 : COPY ./target/gateway-service.jar ./
---> ed025020603a
Step 4/6 : COPY pinpoint /pinpoint
---> c054f97b4364
Step 5/6 : EXPOSE 9999
---> Running in 09e4460fdade
Removing intermediate container 09e4460fdade
---> 74a14e74bdaf
Step 6/6 : CMD java -jar -javaagent:/pinpoint/pinpoint-bootstrap-1.8.3.jar -Dpinpoint.agentId=${HOSTNAME} -Dpinpoint.applicationName=ms-gateway /gateway-service.jar
---> Running in d3b908274346
Removing intermediate container d3b908274346
---> 11d1aa338ce0
Successfully built 11d1aa338ce0
Successfully tagged 192.168.171.10/microservice/gateway-service:4
+ docker push 192.168.171.10/microservice/gateway-service:4
The push refers to repository [192.168.171.10/microservice/gateway-service]
3b602c9dd0e1: Preparing
7d9a36094781: Preparing
b35120dbb2ab: Preparing
a1e7033f082e: Preparing
78075328e0da: Preparing
9f8566ee5135: Preparing
9f8566ee5135: Waiting
78075328e0da: Mounted from microservice/gateway
a1e7033f082e: Mounted from microservice/gateway
b35120dbb2ab: Pushed
9f8566ee5135: Mounted from microservice/gateway
3b602c9dd0e1: Pushed
7d9a36094781: Pushed
4: digest: sha256:6779c5a4cec9c97a573aa51de0acdfbe4422afc8091dd9986c72e6394798939c size: 1578
+ cd /home/jenkins/agent/workspace/YMS
[Pipeline] wrap
provisioning config files...
copy managed file [K8S-auth] to file:/home/jenkins/agent/workspace/YMS/****.kubeconfig
[Pipeline] {
[Pipeline] sh
+ true
+ kubectl create secret docker-registry registry-pull-secret --docker-username=**** --docker-password=**** --docker-server=192.168.171.10 -n ms --kubeconfig ****.kubeconfig
Error from server (AlreadyExists): secrets "registry-pull-secret" already exists
+ helm repo add --username **** --password **** myrepo http://192.168.171.10/chartrepo/microservice
"myrepo" has been added to your repositories
[Pipeline] }
[Pipeline] // wrap
[Pipeline] }
[Pipeline] // withCredentials
[Pipeline] }
[Pipeline] // stage
[Pipeline] stage
[Pipeline] { (Helm部署到K8S)
[Pipeline] sh
+ common_args='-n ms --kubeconfig admin.kubeconfig'
++ echo gateway-service:9999
++ sed 's/,/ /g'
+ for service in '$(echo gateway-service:9999 |sed '\''s/,/ /g'\'')'
+ service_name=gateway-service
+ service_port=9999
+ image=192.168.171.10/microservice/gateway-service
+ tag=4
+ helm_args='gateway-service --set image.repository=192.168.171.10/microservice/gateway-service --set image.tag=4 --set replicaCount=1 --set imagePullSecrets[0].name=registry-pull-secret --set service.targetPort=9999 myrepo/ms'
+ helm history gateway-service -n ms --kubeconfig admin.kubeconfig
+ action=install
+ '[' gateway-service == gateway-service ']'
+ helm install gateway-service --set image.repository=192.168.171.10/microservice/gateway-service --set image.tag=4 --set replicaCount=1 --set 'imagePullSecrets[0].name=registry-pull-secret' --set service.targetPort=9999 myrepo/ms --set ingress.enabled=true --set ingress.host=gateway.zhdya.cn -n ms --kubeconfig admin.kubeconfig
NAME: gateway-service
LAST DEPLOYED: Mon Jan 27 11:57:36 2020
NAMESPACE: ms
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
URL:
http://gateway.zhdya.cn
+ sleep 10
+ kubectl get pods -n ms --kubeconfig admin.kubeconfig
NAME READY STATUS RESTARTS AGE
eureka-0 1/1 Running 0 10h
eureka-1 1/1 Running 1 10h
eureka-2 1/1 Running 0 10h
ms-gateway-service-b48d77495-vfcjj 0/1 Running 0 11s
[Pipeline] }
[Pipeline] // stage
[Pipeline] }
[Pipeline] // node
[Pipeline] }
[Pipeline] // podTemplate
[Pipeline] End of Pipeline
Finished: SUCCESS
部署剩余4个服务：
[root@k8s-master1 gateway-service]# kubectl get po -n ms -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
eureka-0 1/1 Running 0 10h 10.244.36.71 k8s-node1 <none> <none>
eureka-1 1/1 Running 1 10h 10.244.159.180 k8s-master1 <none> <none>
eureka-2 1/1 Running 0 10h 10.244.169.142 k8s-node2 <none> <none>
ms-gateway-service-b48d77495-vfcjj 1/1 Running 0 14m 10.244.36.98 k8s-node1 <none> <none>
ms-order-service-5c98669484-cg2s4 1/1 Running 0 3m23s 10.244.36.103 k8s-node1 <none> <none>
ms-portal-service-5cd676978d-lpz6m 1/1 Running 0 3m28s 10.244.36.104 k8s-node1 <none> <none>
ms-product-service-b4f87c7d-cmrj2 1/1 Running 0 3m26s 10.244.169.147 k8s-node2 <none> <none>
ms-stock-service-6465887d5c-qbxb8 1/1 Running 0 3m15s 10.244.159.190 k8s-master1 <none> <none>
本博客所有文章除特别声明外，均采用 CC BY-SA 4.0 协议 ，转载请注明出处！