Kuboard-K8S多集群管理

一、部署Yaml文件

未启用默认的nodePort模式,采用ingress进行访问

root@hk-eks-ctl:/usr/local/src/zhdya/kuboard# cat kuboard-v3.yaml
---
apiVersion: v1
kind: Namespace
metadata:
  name: kuboard

---
apiVersion: v1
kind: ConfigMap
metadata:
  name: kuboard-v3-config
  namespace: kuboard
data:
  # 关于如下参数的解释,请参考文档 https://kuboard.cn/install/v3/install-built-in.html
  # [common]
  KUBOARD_SERVER_NODE_PORT: '30080'
  KUBOARD_AGENT_SERVER_UDP_PORT: '30081'
  KUBOARD_AGENT_SERVER_TCP_PORT: '30081'
  KUBOARD_SERVER_LOGRUS_LEVEL: info  # error / debug / trace
  # KUBOARD_AGENT_KEY 是 Agent 与 Kuboard 通信时的密钥,请修改为一个任意的包含字母、数字的32位字符串,此密钥变更后,需要删除 Kuboard Agent 重新导入。
  KUBOARD_AGENT_KEY: 32b7d6572c6255211b4eec9009e4a816
  KUBOARD_AGENT_IMAG: eipwork/kuboard-agent
  KUBOARD_QUESTDB_IMAGE: questdb/questdb:6.0.5
  KUBOARD_DISABLE_AUDIT: 'false' # 如果要禁用 Kuboard 审计功能,将此参数的值设置为 'true',必须带引号。

  # 关于如下参数的解释,请参考文档 https://kuboard.cn/install/v3/install-gitlab.html
  # [gitlab login]
  # KUBOARD_LOGIN_TYPE: "gitlab"
  # KUBOARD_ROOT_USER: "your-user-name-in-gitlab"
  # GITLAB_BASE_URL: "http://gitlab.mycompany.com"
  # GITLAB_APPLICATION_ID: "7c10882aa46810a0402d17c66103894ac5e43d6130b81c17f7f2d8ae182040b5"
  # GITLAB_CLIENT_SECRET: "77c149bd3a4b6870bffa1a1afaf37cba28a1817f4cf518699065f5a8fe958889"

  # 关于如下参数的解释,请参考文档 https://kuboard.cn/install/v3/install-github.html
  # [github login]
  # KUBOARD_LOGIN_TYPE: "github"
  # KUBOARD_ROOT_USER: "your-user-name-in-github"
  # GITHUB_CLIENT_ID: "17577d45e4de7dad88e0"
  # GITHUB_CLIENT_SECRET: "ff738553a8c7e9ad39569c8d02c1d85ec19115a7"

  # 关于如下参数的解释,请参考文档 https://kuboard.cn/install/v3/install-ldap.html
  # [ldap login]
  # KUBOARD_LOGIN_TYPE: "ldap"
  # KUBOARD_ROOT_USER: "your-user-name-in-ldap"
  # LDAP_HOST: "ldap-ip-address:389"
  # LDAP_BIND_DN: "cn=admin,dc=example,dc=org"
  # LDAP_BIND_PASSWORD: "admin"
  # LDAP_BASE_DN: "dc=example,dc=org"
  # LDAP_FILTER: "(objectClass=posixAccount)"
  # LDAP_ID_ATTRIBUTE: "uid"
  # LDAP_USER_NAME_ATTRIBUTE: "uid"
  # LDAP_EMAIL_ATTRIBUTE: "mail"
  # LDAP_DISPLAY_NAME_ATTRIBUTE: "cn"
  # LDAP_GROUP_SEARCH_BASE_DN: "dc=example,dc=org"
  # LDAP_GROUP_SEARCH_FILTER: "(objectClass=posixGroup)"
  # LDAP_USER_MACHER_USER_ATTRIBUTE: "gidNumber"
  # LDAP_USER_MACHER_GROUP_ATTRIBUTE: "gidNumber"
  # LDAP_GROUP_NAME_ATTRIBUTE: "cn"

---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: kuboard-boostrap
  namespace: kuboard

---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: kuboard-boostrap-crb
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-admin
subjects:
- kind: ServiceAccount
  name: kuboard-boostrap
  namespace: kuboard

---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  labels:
    k8s.kuboard.cn/name: kuboard-etcd
  name: kuboard-etcd
  namespace: kuboard
spec:
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      k8s.kuboard.cn/name: kuboard-etcd
  template:
    metadata:
      labels:
        k8s.kuboard.cn/name: kuboard-etcd
    spec:
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
              - matchExpressions:
                  - key: node-role.kubernetes.io/master
                    operator: Exists
              - matchExpressions:
                  - key: node-role.kubernetes.io/control-plane
                    operator: Exists
              - matchExpressions:
                  - key: k8s.kuboard.cn/role
                    operator: In
                    values:
                      - etcd
      containers:
        - env:
            - name: HOSTNAME
              valueFrom:
                fieldRef:
                  apiVersion: v1
                  fieldPath: spec.nodeName
            - name: HOSTIP
              valueFrom:
                fieldRef:
                  apiVersion: v1
                  fieldPath: status.hostIP
          image: 'eipwork/etcd-host:3.4.16-1'
          imagePullPolicy: Always
          name: etcd
          ports:
            - containerPort: 2381
              hostPort: 2381
              name: server
              protocol: TCP
            - containerPort: 2382
              hostPort: 2382
              name: peer
              protocol: TCP
          livenessProbe:
            failureThreshold: 3
            httpGet:
              path: /health
              port: 2381
              scheme: HTTP
            initialDelaySeconds: 30
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 1
          volumeMounts:
            - mountPath: /data
              name: data
      dnsPolicy: ClusterFirst
      hostNetwork: true
      restartPolicy: Always
      serviceAccount: kuboard-boostrap
      serviceAccountName: kuboard-boostrap
      tolerations:
        - key: node-role.kubernetes.io/master
          operator: Exists
        - key: node-role.kubernetes.io/control-plane
          operator: Exists
      volumes:
        - hostPath:
            path: /usr/share/kuboard/etcd
          name: data
  updateStrategy:
    rollingUpdate:
      maxUnavailable: 1
    type: RollingUpdate

---
apiVersion: apps/v1
kind: Deployment
metadata:
  annotations: {}
  labels:
    k8s.kuboard.cn/name: kuboard-v3
  name: kuboard-v3
  namespace: kuboard
spec:
  replicas: 1
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      k8s.kuboard.cn/name: kuboard-v3
  template:
    metadata:
      labels:
        k8s.kuboard.cn/name: kuboard-v3
    spec:
      affinity:
        nodeAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
            - preference:
                matchExpressions:
                  - key: node-role.kubernetes.io/master
                    operator: Exists
              weight: 100
            - preference:
                matchExpressions:
                  - key: node-role.kubernetes.io/control-plane
                    operator: Exists
              weight: 100
      containers:
        - env:
            - name: HOSTIP
              valueFrom:
                fieldRef:
                  apiVersion: v1
                  fieldPath: status.hostIP
            - name: HOSTNAME
              valueFrom:
                fieldRef:
                  apiVersion: v1
                  fieldPath: spec.nodeName
          envFrom:
            - configMapRef:
                name: kuboard-v3-config
          image: 'eipwork/kuboard:v3'
          imagePullPolicy: Always
          livenessProbe:
            failureThreshold: 3
            httpGet:
              path: /kuboard-resources/version.json
              port: 80
              scheme: HTTP
            initialDelaySeconds: 30
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 1
          name: kuboard
          ports:
            - containerPort: 80
              name: web
              protocol: TCP
            - containerPort: 443
              name: https
              protocol: TCP
            - containerPort: 10081
              name: peer
              protocol: TCP
            - containerPort: 10081
              name: peer-u
              protocol: UDP
          readinessProbe:
            failureThreshold: 3
            httpGet:
              path: /kuboard-resources/version.json
              port: 80
              scheme: HTTP
            initialDelaySeconds: 30
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 1
          resources: {}
          # startupProbe:
          #   failureThreshold: 20
          #   httpGet:
          #     path: /kuboard-resources/version.json
          #     port: 80
          #     scheme: HTTP
          #   initialDelaySeconds: 5
          #   periodSeconds: 10
          #   successThreshold: 1
          #   timeoutSeconds: 1
      dnsPolicy: ClusterFirst
      restartPolicy: Always
      serviceAccount: kuboard-boostrap
      serviceAccountName: kuboard-boostrap
      tolerations:
        - key: node-role.kubernetes.io/master
          operator: Exists

---
apiVersion: v1
kind: Service
metadata:
  annotations: {}
  labels:
    k8s.kuboard.cn/name: kuboard-v3
  name: kuboard-v3
  namespace: kuboard
spec:
  ports:
    - name: web
#      nodePort: 30080
      port: 80
      protocol: TCP
      targetPort: 80
    - name: tcp
#      nodePort: 30081
      port: 10081
      protocol: TCP
      targetPort: 10081
    - name: udp
#      nodePort: 30081
      port: 10081
      protocol: UDP
      targetPort: 10081
  selector:
    k8s.kuboard.cn/name: kuboard-v3
  sessionAffinity: None
#  type: NodePort
  type: ClusterIP

二、ingress

root@hk-eks-ctl:/usr/local/src/zhdya/kuboard# cat kuboard-ingress.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: kuboard-ingress
  namespace: kuboard
spec:
  ingressClassName: kong
  rules:
  - host: kuboard-devops.xxxxx.com
    http:
      paths:
      - backend:
          service:
            name: kuboard-v3
            port:
              number: 80
        path: /
        pathType: ImplementationSpecific
status:
  loadBalancer:
    ingress:
    - hostname: a922238e9xxxxx.ap-xxx-1.elb.amazonaws.com

三、agent连接集群

默认agent连接采用nodePort的方式,所以 KUBOARD_ENDPOINTKUBORAD_AGENT_HOST 需要修改为

root@hk-eks-ctl:/usr/local/src/zhdya/kuboard# cat kuboard-agent.yaml
---
apiVersion: v1
kind: Namespace
metadata:
  name: kuboard

---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: kuboard-admin
  namespace: kuboard

---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: kuboard-admin-crb
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-admin
subjects:
- kind: ServiceAccount
  name: kuboard-admin
  namespace: kuboard

---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: kuboard-viewer
  namespace: kuboard

---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: kuboard-viewer-crb
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: view
subjects:
- kind: ServiceAccount
  name: kuboard-viewer
  namespace: kuboard

---
apiVersion: apps/v1
kind: Deployment
metadata:
  annotations:
    k8s.kuboard.cn/ingress: "false"
    k8s.kuboard.cn/service: none
    k8s.kuboard.cn/workload: kuboard-agent-1lccwv7
  labels:
    k8s.kuboard.cn/name: kuboard-agent-1lccwv7
  name: kuboard-agent-1lccwv7
  namespace: kuboard
spec:
  replicas: 1
  selector:
    matchLabels:
      k8s.kuboard.cn/name: kuboard-agent-1lccwv7
  template:
    metadata:
      labels:
        k8s.kuboard.cn/name: kuboard-agent-1lccwv7
    spec:
      affinity:
        nodeAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
            - preference:
                matchExpressions:
                  - key: node-role.kubernetes.io/master
                    operator: Exists
              weight: 100
        podAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
            - podAffinityTerm:
                labelSelector:
                  matchLabels:
                    k8s.kuboard.cn/name: kuboard-v3
                namespaces:
                  - kuboard
                topologyKey: kubernetes.io/hostname
              weight: 100
      serviceAccountName: kuboard-admin
      tolerations:
        - effect: NoSchedule
          key: node-role.kubernetes.io/master
          operator: Exists
      containers:
        - env:
            - name: KUBOARD_ENDPOINT
              value: "http://kuboard-v3.kuboard"		## 修改为svc的name
            - name: KUBOARD_AGENT_HOST
              value: "172.20.141.5"				## 修改为svc的IP地址
            - name: KUBOARD_AGENT_PORT
              value: "10081"			## 修改为tcp的端口
            - name: KUBOARD_AGENT_REMOTE_PORT
              value: "35002"
            - name: KUBOARD_AGENT_PROTOCOL
              value: "tcp"
            - name: KUBOARD_AGENT_PROXY
              value: ""
            - name: KUBOARD_K8S_CLUSTER_NAME
              value: "SparkX-K8S"
            - name: KUBOARD_AGENT_KEY
              value: "32b7d6572c6255211b4eec9009e4a816"
            - name: KUBERNETES_TOKEN_NAME
              value: "kuboard-admin"
            - name: KUBOARD_ANONYMOUS_TOKEN
              value: "FDeJRgm09SBeJ1G1BC7Wyx8cxzXCR4jz"
          image: "eipwork/kuboard-agent:v3"
          imagePullPolicy: Always
          livenessProbe:
            exec:
              command:
                - /health.sh
            failureThreshold: 3
            initialDelaySeconds: 5
            periodSeconds: 60
            successThreshold: 1
            timeoutSeconds: 1
          name: kuboard-agent
      restartPolicy: Always

---
apiVersion: apps/v1
kind: Deployment
metadata:
  annotations:
    k8s.kuboard.cn/ingress: "false"
    k8s.kuboard.cn/service: none
    k8s.kuboard.cn/workload: kuboard-agent-1lccwv7-2
  labels:
    k8s.kuboard.cn/name: kuboard-agent-1lccwv7-2
  name: kuboard-agent-1lccwv7-2
  namespace: kuboard
spec:
  replicas: 1
  selector:
    matchLabels:
      k8s.kuboard.cn/name: kuboard-agent-1lccwv7-2
  template:
    metadata:
      labels:
        k8s.kuboard.cn/name: kuboard-agent-1lccwv7-2
    spec:
      affinity:
        nodeAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
            - preference:
                matchExpressions:
                  - key: node-role.kubernetes.io/master
                    operator: Exists
              weight: 100
        podAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
            - podAffinityTerm:
                labelSelector:
                  matchLabels:
                    k8s.kuboard.cn/name: kuboard-v3
                namespaces:
                  - kuboard
                topologyKey: kubernetes.io/hostname
              weight: 100
      serviceAccountName: kuboard-viewer
      tolerations:
        - effect: NoSchedule
          key: node-role.kubernetes.io/master
          operator: Exists
      containers:
        - env:
            - name: KUBOARD_ENDPOINT
              value: "http://kuboard-v3.kuboard"		## 修改为svc的name
            - name: KUBOARD_AGENT_HOST
              value: "172.20.141.5"		## 修改为svc的IP地址
            - name: KUBOARD_AGENT_PORT
              value: "10081"		## 修改为tcp的端口
            - name: KUBOARD_AGENT_REMOTE_PORT
              value: "35002"
            - name: KUBOARD_AGENT_PROTOCOL
              value: "tcp"
            - name: KUBOARD_AGENT_PROXY
              value: ""
            - name: KUBOARD_K8S_CLUSTER_NAME
              value: "SparkX-K8S"
            - name: KUBOARD_AGENT_KEY
              value: "32b7d6572c6255211b4eec9009e4a816"
            - name: KUBERNETES_TOKEN_NAME
              value: "kuboard-viewer"
            - name: KUBOARD_ANONYMOUS_TOKEN
              value: "FDeJRgm09SBeJ1G1BC7Wyx8cxzXCR4jz"
          image: "eipwork/kuboard-agent:v3"
          imagePullPolicy: Always
          livenessProbe:
            exec:
              command:
                - /health.sh
            failureThreshold: 3
            initialDelaySeconds: 5
            periodSeconds: 60
            successThreshold: 1
            timeoutSeconds: 1
          name: kuboard-agent
      restartPolicy: Always

四、部署后状态

root@hk-eks-ctl:/usr/local/src/zhdya/kuboard# kubectl get po -nkuboard
NAME                                      READY   STATUS    RESTARTS   AGE
kuboard-agent-1lccwv7-2-d8b6ff456-7jktw   1/1     Running   0          68m
kuboard-agent-1lccwv7-5f54f57554-sfw95    1/1     Running   0          68m
kuboard-etcd-hcr6t                        1/1     Running   0          174m
kuboard-etcd-sz7sg                        1/1     Running   0          174m
kuboard-questdb-5794cd4bcd-2qhvj          1/1     Running   0          174m
kuboard-v3-5fc46b5557-j8ps6               1/1     Running   0          174m

五、截个图吧