python全程自动化跑数据迭代

1、底层镜像

root@nx-eks-ctl:/usr/local/src/zhdya/pythontest# cat Dockerfile
FROM python:3.9
RUN mkdir -p /usr/local/ck/
COPY requirements.txt /usr/local/ck/
RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
WORKDIR /usr/local/ck/
RUN pip install -i https://pypi.mirrors.ustc.edu.cn/simple/ -r requirements.txt

root@nx-eks-ctl:/usr/local/src/zhdya/pythontest# cat requirements.txt
clickhouse_driver==0.2.4
numpy==1.23.0
pandas==1.4.3
pymysql==1.0.2
sqlalchemy==1.4.39

2、deploy.yml

apiVersion: batch/v1beta1
kind: CronJob
metadata:
  name: bigdatajob
  namespace: cron-jobs
spec:
  schedule: "*/5 * * * *"
  failedJobHistoryLimit: 2
  concurrencyPolicy: Forbid
  successfulJobHistoryLimit: 2
  startingDeadlineSeconds: 600
  jobTemplate:
    spec:
      template:
        spec:
          containers:
          - name: bigdatajob
            image: $IMAGE_NAME
            imagePullPolicy: IfNotPresent
            args:
            - "/bin/sh"
            - "-c"
            - "python /usr/local/ck/app.py"
          restartPolicy: OnFailure
          imagePullSecrets:
          - name: harborsecret

备注：

concurrencyPolicy：
Allow: 允许Jobs并发运行(默认)
Forbid: 禁止并发运行，如果上一次运行尚未完成，则跳过下一次运行
Replace: 替换，取消当前正在运行的作业并用新作业替换它
startingDeadlineSeconds: # 启动作业错误的超时时长

Jenkins自动化（HK线上）：

def git_address = "https://gitlab.devops.xxxx.com/xxxx/go-demo.git" 
def git_auth = "56e9bfd7-7bae-4943-xxxx-51c9e14a0d91"

//Harbor私服地址
def harbor_url = "harbor-devops.xxxx.com"
//Harbor的项目名称
def harbor_project_name = "sparkx"
//Harbor的凭证
def harbor_auth = "c5391591-fa9f-4347-xxxx-56e50df47150"
//k8s的凭证
def k8s_auth = "9211c0f5-8ff7-439d-xxx-d9a15065c46e"
//启动时间
def start = new Date().format('yyyy-MM-dd HH:mm:ss')

//创建一个Pod的模板，label为jenkins-slave
podTemplate(label: 'jenkins-slave-bigdata', cloud: 'kubernetes', containers: [
    containerTemplate( 
        name: 'build-image',
        image: "harbor-devops.xxxx.com/library/docker:stable",
        ttyEnabled: true,
        command: 'cat'
        ),
    ],
    //需要将docker和kubectl挂在到pod中这样才可以在pod中与k8s进行联通操作。
    volumes: [
        hostPathVolume(mountPath: '/var/run/docker.sock', hostPath: '/var/run/docker.sock'),
    ]
)
{
    node("jenkins-slave-bigdata") {
        stage('Pull'){
            checkout([$class: 'GitSCM', branches: [[name: "${branch}"]], extensions: [], userRemoteConfigs: [[credentialsId: "${git_auth}", url: "${git_address}"]]])
        }
        stage('Build-Image') {
            //定义newTag
            def newTag = sh(returnStdout: true,script: 'echo `date +"%Y%m%d%H%M"_``git describe --tags --always`').trim()
            
            container('build-image') {
                //build
                sh "docker build -t ${harbor_url}/${harbor_project_name}/bigdatajob:${newTag} ."
                
                //登录Harbor，并上传镜像
                withCredentials([usernamePassword(credentialsId: "${harbor_auth}", passwordVariable: 'password', usernameVariable: 'username')])
                {
                    //登录
                    sh "docker login -u ${username} -p ${password} ${harbor_url}"
                    //上传镜像
                    sh "docker push ${harbor_url}/${harbor_project_name}/bigdatajob:${newTag}"
                }
                
                //删除本地镜像
                sh "docker rmi -f ${harbor_url}/${harbor_project_name}/bigdatajob:${newTag}"
            }
            
            //更新相关参数至deploy文件
            def deploy_image_name = "${harbor_url}/${harbor_project_name}/bigdatajob:${newTag}"
            
            sh """
                sed -i 's#\$IMAGE_NAME#${deploy_image_name}#' deploy.yml
                cat deploy.yml
            """
        }
        stage('Deploy'){
            kubernetesDeploy(
                kubeconfigId: '9211c0f5-8ff7-xxx-a9d2-d9a15065c46e',
                configs: 'deploy.yml' 
            )
        }
        stage('Webhook') {
            wrap([$class: 'BuildUser']) {
                script {
                    BUILD_USER = "${env.BUILD_USER}"
                }
            }
            def head = "\"构建${JOB_NAME}：#${env.BUILD_ID}<font color=\\\"info\\\">成功</font>，详细信息如下:"
            def s1 = ">部署分支：<font color=\\\"comment\\\">${branch}</font>"
            def s2 = ">部署服务：<font color=\\\"comment\\\">cronJob-bigdata</font>"
            def execTime = start+"   "+new Date().format('yyyy-MM-dd HH:mm:ss')
            def s3 = ">部署时间：<font color=\\\"comment\\\">${execTime}</font>"
            def s4 = ">部署人：<font color=\\\"comment\\\">${BUILD_USER}</font>"
            def msg = "${head}"+"\n"+ "${s1}"+"\n"+"${s2}"+"\n"+"${s3}"+"\n"+"${s4}\""
            echo "${msg}"
            def body = "{ \"msgtype\": \"markdown\", \"markdown\": { \"content\": ${msg} } }"
            echo "${body}"
            httpRequest contentType: 'APPLICATION_JSON_UTF8', httpMode: 'POST', requestBody: "${body}", responseHandle: 'NONE', url: 'https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=aa84fd2f-6346-xxxx-baa7-ae91b865d216', wrapAsMultipart: false
        }
    }
}