本文档讲解部署一个三实例 kube-apiserver 集群的步骤.

注意：如果没有特殊指明，本文档的所有操作均在 zhangjun-k8s-01 节点上执行。

创建 kubernetes-master 证书和私钥

创建证书签名请求：

cd /opt/k8s/worksource /opt/k8s/bin/environment.shcat > kubernetes-csr.json <<EOF{  "CN": "kubernetes-master",  "hosts": [    "127.0.0.1",    "172.27.138.251",    "172.27.137.229",    "172.27.138.239",    "${CLUSTER_KUBERNETES_SVC_IP}",    "kubernetes",    "kubernetes.default",    "kubernetes.default.svc",    "kubernetes.default.svc.cluster",    "kubernetes.default.svc.cluster.local.",    "kubernetes.default.svc.${CLUSTER_DNS_DOMAIN}."  ],  "key": {    "algo": "rsa",    "size": 2048  },  "names": [    {      "C": "CN",      "ST": "BeiJing",      "L": "BeiJing",      "O": "k8s",      "OU": "opsnull"    }  ]}EOF

• hosts 字段指定授权使用该证书的 IP 和域名列表，这里列出了 master 节点 IP、kubernetes 服务的 IP 和域名；

生成证书和私钥：

cfssl gencert -ca=/opt/k8s/work/ca.pem \  -ca-key=/opt/k8s/work/ca-key.pem \  -config=/opt/k8s/work/ca-config.json \  -profile=kubernetes kubernetes-csr.json | cfssljson -bare kubernetesls kubernetes*pem

将生成的证书和私钥文件拷贝到所有 master 节点：

cd /opt/k8s/worksource /opt/k8s/bin/environment.shfor node_ip in ${NODE_IPS[@]}  do    echo ">>> ${node_ip}"    ssh root@${node_ip} "mkdir -p /etc/kubernetes/cert"    scp kubernetes*.pem root@${node_ip}:/etc/kubernetes/cert/  done

创建加密配置文件

cd /opt/k8s/worksource /opt/k8s/bin/environment.shcat > encryption-config.yaml <<EOFkind: EncryptionConfigapiVersion: v1resources:  - resources:      - secrets    providers:      - aescbc:          keys:            - name: key1              secret: ${ENCRYPTION_KEY}      - identity: {}EOF

将加密配置文件拷贝到 master 节点的 /etc/kubernetes 目录下：

cd /opt/k8s/worksource /opt/k8s/bin/environment.shfor node_ip in ${NODE_IPS[@]}  do    echo ">>> ${node_ip}"    scp encryption-config.yaml root@${node_ip}:/etc/kubernetes/  done

创建审计策略文件

cd /opt/k8s/worksource /opt/k8s/bin/environment.shcat > audit-policy.yaml <<EOFapiVersion: audit.k8s.io/v1beta1kind: Policyrules:  # The following requests were manually identified as high-volume and low-risk, so drop them.  - level: None    resources:      - group: ""        resources:          - endpoints          - services          - services/status    users:      - 'system:kube-proxy'    verbs:      - watch  - level: None    resources:      - group: ""        resources:          - nodes          - nodes/status    userGroups:      - 'system:nodes'    verbs:      - get  - level: None    namespaces:      - kube-system    resources:      - group: ""        resources:          - endpoints    users:      - 'system:kube-controller-manager'      - 'system:kube-scheduler'      - 'system:serviceaccount:kube-system:endpoint-controller'    verbs:      - get      - update  - level: None    resources:      - group: ""        resources:          - namespaces          - namespaces/status          - namespaces/finalize    users:      - 'system:apiserver'    verbs:      - get  # Don't log HPA fetching metrics.  - level: None    resources:      - group: metrics.k8s.io    users:      - 'system:kube-controller-manager'    verbs:      - get      - list  # Don't log these read-only URLs.  - level: None    nonResourceURLs:      - '/healthz*'      - /version      - '/swagger*'  # Don't log events requests.  - level: None    resources:      - group: ""        resources:          - events  # node and pod status calls from nodes are high-volume and can be large, don't log responses  # for expected updates from nodes  - level: Request    omitStages:      - RequestReceived    resources:      - group: ""        resources:          - nodes/status          - pods/status    users:      - kubelet      - 'system:node-problem-detector'      - 'system:serviceaccount:kube-system:node-problem-detector'    verbs:      - update      - patch  - level: Request    omitStages:      - RequestReceived    resources:      - group: ""        resources:          - nodes/status          - pods/status    userGroups:      - 'system:nodes'    verbs:      - update      - patch  # deletecollection calls can be large, don't log responses for expected namespace deletions  - level: Request    omitStages:      - RequestReceived    users:      - 'system:serviceaccount:kube-system:namespace-controller'    verbs:      - deletecollection  # Secrets, ConfigMaps, and TokenReviews can contain sensitive & binary data,  # so only log at the Metadata level.  - level: Metadata    omitStages:      - RequestReceived    resources:      - group: ""        resources:          - secrets          - configmaps      - group: authentication.k8s.io        resources:          - tokenreviews  # Get repsonses can be large; skip them.  - level: Request    omitStages:      - RequestReceived    resources:      - group: ""      - group: admissionregistration.k8s.io      - group: apiextensions.k8s.io      - group: apiregistration.k8s.io      - group: apps      - group: authentication.k8s.io      - group: authorization.k8s.io      - group: autoscaling      - group: batch      - group: certificates.k8s.io      - group: extensions      - group: metrics.k8s.io      - group: networking.k8s.io      - group: policy      - group: rbac.authorization.k8s.io      - group: scheduling.k8s.io      - group: settings.k8s.io      - group: storage.k8s.io    verbs:      - get      - list      - watch  # Default level for known APIs  - level: RequestResponse    omitStages:      - RequestReceived    resources:      - group: ""      - group: admissionregistration.k8s.io      - group: apiextensions.k8s.io      - group: apiregistration.k8s.io      - group: apps      - group: authentication.k8s.io      - group: authorization.k8s.io      - group: autoscaling      - group: batch      - group: certificates.k8s.io      - group: extensions      - group: metrics.k8s.io      - group: networking.k8s.io      - group: policy      - group: rbac.authorization.k8s.io      - group: scheduling.k8s.io      - group: settings.k8s.io      - group: storage.k8s.io  # Default level for all other requests.  - level: Metadata    omitStages:      - RequestReceivedEOF

分发审计策略文件：

cd /opt/k8s/worksource /opt/k8s/bin/environment.shfor node_ip in ${NODE_IPS[@]}  do    echo ">>> ${node_ip}"    scp audit-policy.yaml root@${node_ip}:/etc/kubernetes/audit-policy.yaml  don