基于Vagrant+K8S+Harbor+gitlab的CI/CD(4）

CI/CD实践

把harbor拉取的凭证secret给创建好，命令如下：

kubectl -n test create secret docker-registry boge-secret --docker-server=harbor.boge.com --docker-username=admin --docker-password=boge666 --docker-email=admin@boge.com

在代码仓库变量配置里面配置如下变量值
Type Key Value State Masked
Variable DOCKER_USER admin 下面都关闭 下面都关闭
Variable DOCKER_PASS boge666
Variable REGISTRY_URL harbor.boge.com
Variable REGISTRY_NS product
File KUBE_CONFIG_TEST k8s相关config配置文件内容
k8s相关config配置文件内容 来自于/etc/kubernetes/admin.conf

flask相关的文件

app.py

from flask import Flask
app = Flask(__name__)@app.route('/')
def hello_world():return 'Hello, boge! 21.04.11.01'@app.route('/gg/<username>')
def hello(username):return 'welcome' + ': ' + username + '!'

gunicorn_config.py

bind = '0.0.0.0:5000'
graceful_timeout = 3600
timeout = 1200
max_requests = 1200
workers = 1
worker_class = 'gevent'

Dockerfile

FROM harbor.boge.com/library/python:3.5-slim-stretch
MAINTAINER bogeWORKDIR /kae/appCOPY requirements.txt .RUN  sed -i 's/deb.debian.org/ftp.cn.debian.org/g' /etc/apt/sources.list \&& sed -i 's/security.debian.org/ftp.cn.debian.org/g' /etc/apt/sources.list \&& apt-get update -y \&& apt-get install -y wget gcc libsm6 libxext6 libglib2.0-0 libxrender1 make \&& apt-get clean && apt-get autoremove -y && rm -rf /var/lib/apt/lists/*
RUN pip install --no-cache-dir -i https://mirrors.aliyun.com/pypi/simple -r requirements.txt \&& rm requirements.txtCOPY . .EXPOSE 5000
HEALTHCHECK CMD curl --fail http://localhost:5000 || exit 1ENTRYPOINT ["gunicorn", "app:app", "-c", "gunicorn_config.py"]

自动化配置文件.gitlab-ci.yml

stages:- build- deploy- rollback# tag name need: 20.11.21.01
variables:namecb: "flask-test"svcport: "5000"replicanum: "2"ingress: "flask-test.boge.com"certname: "mytls"CanarylIngressNum: "20".deploy_k8s: &deploy_k8s |if [ $CANARY_CB -eq 1 ];then cp -arf .project-name-canary.yaml ${namecb}-${CI_COMMIT_TAG}.yaml; sed -ri "s+CanarylIngressNum+${CanarylIngressNum}+g" ${namecb}-${CI_COMMIT_TAG}.yaml; sed -ri "s+NomalIngressNum+$(expr 100 - ${CanarylIngressNum})+g" ${namecb}-${CI_COMMIT_TAG}.yaml ;else cp -arf .project-name.yaml ${namecb}-${CI_COMMIT_TAG}.yaml;fised -ri "s+projectnamecb.boge.com+${ingress}+g" ${namecb}-${CI_COMMIT_TAG}.yamlsed -ri "s+projectnamecb+${namecb}+g" ${namecb}-${CI_COMMIT_TAG}.yamlsed -ri "s+5000+${svcport}+g" ${namecb}-${CI_COMMIT_TAG}.yamlsed -ri "s+replicanum+${replicanum}+g" ${namecb}-${CI_COMMIT_TAG}.yamlsed -ri "s+mytls+${certname}+g" ${namecb}-${CI_COMMIT_TAG}.yamlsed -ri "s+mytagcb+${CI_COMMIT_TAG}+g" ${namecb}-${CI_COMMIT_TAG}.yamlsed -ri "s+harbor.boge.com/library+${IMG_URL}+g" ${namecb}-${CI_COMMIT_TAG}.yamlcat ${namecb}-${CI_COMMIT_TAG}.yaml[ -d ~/.kube ] || mkdir ~/.kubeecho "$KUBE_CONFIG" > ~/.kube/configif [ $NORMAL_CB -eq 1 ];then if kubectl get deployments.|grep -w ${namecb}-canary &>/dev/null;then kubectl delete deployments.,svc ${namecb}-canary ;fi;fikubectl apply -f ${namecb}-${CI_COMMIT_TAG}.yaml --recordechoechoecho "============================================================="echo "                    Rollback Indx List"echo "============================================================="kubectl rollout history deployment ${namecb}|tail -5|awk -F"[ =]+" '{print $1"\t"$5}'|sed '$d'|sed '$d'|sort -r|awk '{print $NF}'|awk '$0=""NR".   "$0'.rollback_k8s: &rollback_k8s |[ -d ~/.kube ] || mkdir ~/.kubeecho "$KUBE_CONFIG" > ~/.kube/configlast_version_command=$( kubectl rollout history deployment ${namecb}|tail -5|awk -F"[ =]+" '{print $1"\t"$5}'|sed '$d'|sed '$d'|tail -${ROLL_NUM}|head -1 )last_version_num=$( echo ${last_version_command}|awk '{print $1}' )last_version_name=$( echo ${last_version_command}|awk '{print $2}' )kubectl rollout undo deployment ${namecb} --to-revision=$last_version_numecho $last_version_numecho $last_version_namekubectl rollout history deployment ${namecb}build:stage: buildretry: 2variables:# use dind.yaml to depoy dind'service on k8sDOCKER_HOST: tcp://172.28.128.10:2375/DOCKER_DRIVER: overlay2DOCKER_TLS_CERTDIR: ""##services:##- docker:dindbefore_script:- docker login ${REGISTRY_URL} -u "$DOCKER_USER" -p "$DOCKER_PASS"script:- docker pull ${REGISTRY_URL}/${REGISTRY_NS}/${namecb}:latest || true- docker build --network host --cache-from ${REGISTRY_URL}/${REGISTRY_NS}/${namecb}:latest --tag ${REGISTRY_URL}/${REGISTRY_NS}/${namecb}:$CI_COMMIT_TAG --tag ${REGISTRY_URL}/${REGISTRY_NS}/${namecb}:latest .- docker push ${REGISTRY_URL}/${REGISTRY_NS}/${namecb}:$CI_COMMIT_TAG- docker push ${REGISTRY_URL}/${REGISTRY_NS}/${namecb}:latestafter_script:- docker logout ${REGISTRY_URL}tags:- "docker"only:- tags#--------------------------K8S DEPLOY--------------------------------------------------BOGE-deploy:stage: deployimage: harbor.boge.com/library/kubectl:v1.19.9variables:KUBE_CONFIG: "$KUBE_CONFIG_TEST"IMG_URL: "${REGISTRY_URL}/${REGISTRY_NS}"NORMAL_CB: 1script:- *deploy_k8swhen: manualonly:- tags# canary start
BOGE-canary-deploy:stage: deployimage: harbor.boge.com/library/kubectl:v1.19.9variables:KUBE_CONFIG: "$KUBE_CONFIG_TEST"IMG_URL: "${REGISTRY_URL}/${REGISTRY_NS}"CANARY_CB: 1script:- *deploy_k8swhen: manualonly:- tags
# canary endBOGE-rollback-1:stage: rollbackimage: harbor.boge.com/library/kubectl:v1.19.9variables:KUBE_CONFIG: "$KUBE_CONFIG_TEST"ROLL_NUM: 1script:- *rollback_k8swhen: manualonly:- tagsBOGE-rollback-2:stage: rollbackimage: harbor.boge.com/library/kubectl:v1.19.9variables:KUBE_CONFIG: "$KUBE_CONFIG_TEST"ROLL_NUM: 2script:- *rollback_k8swhen: manualonly:- tagsBOGE-rollback-3:stage: rollbackimage: harbor.boge.com/library/kubectl:v1.19.9variables:KUBE_CONFIG: "$KUBE_CONFIG_TEST"ROLL_NUM: 3script:- *rollback_k8swhen: manualonly:- tags

正常部署

k8s的deployment模板文件 .project-name.yaml

---
# SVC
kind: Service
apiVersion: v1
metadata:labels:kae: "true"kae-app-name: projectnamecbkae-type: appname: projectnamecb
spec:selector:kae: "true"kae-app-name: projectnamecbkae-type: appports:- name: http-portport: 80protocol: TCPtargetPort: 5000
#      nodePort: 12345
#  type: NodePort---
# Ingress
apiVersion: extensions/v1beta1
kind: Ingress
metadata:labels:kae: "true"kae-app-name: projectnamecbkae-type: appname: projectnamecb
spec:tls:- hosts:- projectnamecb.boge.comsecretName: mytlsrules:- host: projectnamecb.boge.comhttp:paths:- path: /backend:serviceName: projectnamecbservicePort: 80---
# Deployment
apiVersion: apps/v1
kind: Deployment
metadata:name: projectnamecblabels:kae: "true"kae-app-name: projectnamecbkae-type: app
spec:replicas: replicanumselector:matchLabels:kae-app-name: projectnamecbtemplate:metadata:labels:kae: "true"kae-app-name: projectnamecbkae-type: appspec:containers:- name: projectnamecbimage: harbor.boge.com/library/projectnamecb:mytagcbenv:- name: TZvalue: Asia/Shanghaiports:- containerPort: 5000readinessProbe:httpGet:scheme: HTTPpath: /port: 5000initialDelaySeconds: 10periodSeconds: 5timeoutSeconds: 3successThreshold: 1failureThreshold: 3livenessProbe:httpGet:scheme: HTTPpath: /port: 5000initialDelaySeconds: 10periodSeconds: 5timeoutSeconds: 3successThreshold: 1failureThreshold: 3resources:requests:cpu: 0.3memory: 0.5Gilimits:cpu: 0.3memory: 0.5GiimagePullSecrets:- name: boge-secret

金私雀部署

John Scott Haldane 于 1895 年提出，因为小型恒温动物的呼吸交换比人类更快,矿井中的一氧化碳等有毒气体或甲烷等窒息性气体会先影响它们。

比如,同样的一氧化碳浓度,老鼠会在几分钟内受到一氧化碳的影响，人类需要 20 倍的时间才会受到影响，于是 1896 年左右开始，老鼠被用作井下有毒气体预警的物种。

一段时间后，人们发现金丝雀这种生物对于有毒气体更加敏感。在 1900 年开始有记录显示，一些矿井开始把金丝雀作为井下有毒气体的预警物种。

后来，人们为了能重复使用金丝雀，发明了一个用于金丝雀毒气探测的专用笼子。笼子可以主动充氧，前方设有一个通气孔，通气孔可以通过密闭窗进行开启和关闭。在需要金丝雀进行预警的时候,把通气孔打开。如果笼子中的金丝雀被毒气毒晕，关上通气孔的窗口并让笼子充满氧气。如果金丝雀如果没有被毒死，就有可能活过来。

因为科技的不断发展，有毒气体探测器被发明了出来。这种用生命来探测的方式开始逐步退出历史舞台。直到1986 年英国和美国完全停止使用金丝雀来作为预警生物使用。

金丝雀部署这种部署方式的目标与逻辑和使用金丝雀来预警非常类似(通过开关通气孔/流量的方式来控制危害与恢复能力)，我猜想可能大家也希望能纪念一下在 20 世纪为矿工献出生命的金色小鸟们，所以这种方式被冠上了金丝雀的名称。

金丝雀部署的模板文件 .project-name-canary.yaml

---
# SVC
kind: Service
apiVersion: v1
metadata:labels:kae: "true"kae-app-name: projectnamecb-canarykae-type: appname: projectnamecb-canary
spec:selector:kae: "true"kae-app-name: projectnamecb-canarykae-type: appports:- name: http-portport: 80protocol: TCPtargetPort: 5000
#      nodePort: 12345
#  type: NodePort---
# Ingress
apiVersion: extensions/v1beta1
kind: Ingress
metadata:labels:kae: "true"kae-app-name: projectnamecb-canarykae-type: appname: projectnamecbannotations:nginx.ingress.kubernetes.io/service-weight: |projectnamecb: NomalIngressNum, projectnamecb-canary: CanarylIngressNum
spec:tls:- hosts:- projectnamecb.boge.comsecretName: mytlsrules:- host: projectnamecb.boge.comhttp:paths:- path: /backend:serviceName: projectnamecbservicePort: 80- path: /backend:serviceName: projectnamecb-canaryservicePort: 80---
# Deployment
apiVersion: apps/v1
kind: Deployment
metadata:name: projectnamecb-canarylabels:kae: "true"kae-app-name: projectnamecb-canarykae-type: app
spec:replicas: replicanumselector:matchLabels:kae-app-name: projectnamecb-canarytemplate:metadata:labels:kae: "true"kae-app-name: projectnamecb-canarykae-type: appspec:containers:- name: projectnamecb-canaryimage: harbor.boge.com/library/projectnamecb:mytagcbenv:- name: TZvalue: Asia/Shanghaiports:- containerPort: 5000readinessProbe:httpGet:scheme: HTTPpath: /port: 5000initialDelaySeconds: 10periodSeconds: 5timeoutSeconds: 3successThreshold: 1failureThreshold: 3livenessProbe:httpGet:scheme: HTTPpath: /port: 5000initialDelaySeconds: 10periodSeconds: 5timeoutSeconds: 3successThreshold: 1failureThreshold: 3resources:requests:cpu: 0.3memory: 0.5Gilimits:cpu: 0.3memory: 0.5GiimagePullSecrets:- name: boge-secret