Skip to content

Latest commit

 

History

History
253 lines (243 loc) · 9.43 KB

07.kubelet-node部署.md

File metadata and controls

253 lines (243 loc) · 9.43 KB
Raw

部署 kubelet worker 节点

1.1 创建 kubelet 证书及签名请求

这边使用 Node authorizer 来认证节点的 kubelet 能够存取如 services、endpoints 等 API，而使用 Node authorizer 需要定义 system:nodes 群组（凭证的Organization），并且包含 system:node:<nodeName> 的使用者名称（凭证的Common Name）。

创建 kubelet 证书请求文件
cat > /root/pki/ssl/kubelet-csr.json <<EOF
{
  "CN": "system:node:##NODE_IP##",
  "hosts": [
    "127.0.0.1",
    "##NODE_IP##"
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "BeiJing",
      "L": "BeiJing",
      "O": "system:nodes",
      "OU": "4Paradigm"
    }
  ]
}
EOF
签发各节点证书请求
$ export NODE_IPS=(192.168.133.128 192.168.133.129 192.168.133.130 192.168.133.131)
$ for node_ip in ${NODE_IPS[@]}
  do
    echo ">>> ${node_ip}"
    \cp -f /root/pki/ssl/kubelet-csr.json /root/pki/ssl/kubelet-csr-${node_ip}.json
    sed -i "s@##NODE_IP##@${node_ip}@" kubelet-csr-${node_ip}.json
    cfssl gencert -ca=/etc/kubernetes/ssl/ca.pem -ca-key=/etc/kubernetes/ssl/ca-key.pem -config=/etc/kubernetes/ssl/ca-config.json -profile=kubernetes kubelet-csr-${node_ip}.json | cfssljson -bare kubelet-${node_ip}
    scp kubelet-${node_ip}* ${node_ip}:/etc/kubernetes/ssl
    ssh ${node_ip} "mv /etc/kubernetes/ssl/kubelet-${node_ip}.csr /etc/kubernetes/ssl/kubelet.csr"
    ssh ${node_ip} "mv /etc/kubernetes/ssl/kubelet-${node_ip}.pem /etc/kubernetes/ssl/kubelet.pem"
    ssh ${node_ip} "mv /etc/kubernetes/ssl/kubelet-${node_ip}-key.pem /etc/kubernetes/ssl/kubelet-key.pem"
    ssh ${node_ip} "chmod 755 /etc/kubernetes/ssl/kubelet*.pem"
  done
创建 kubelet.kubeconfig
$ export MASTER_VIP=192.168.133.200
$ export KUBE_APISERVER="https://${MASTER_VIP}:8443"
$ export NODE_IPS=(192.168.133.128 192.168.133.129 192.168.133.130 192.168.133.131)
$ for node_ip in ${NODE_IPS[@]}
  do
    echo ">>> ${node_ip}"
    echo "设置集群参数"
    kubectl config set-cluster kubernetes --certificate-authority=/etc/kubernetes/ssl/ca.pem --embed-certs=true --server=${KUBE_APISERVER} --kubeconfig=kubelet-${node_ip}.kubeconfig

    echo "设置客户端认证参数"
    kubectl config set-credentials system:node:${node_ip} --client-certificate=/root/pki/ssl/kubelet-${node_ip}.pem --embed-certs=true --client-key=/root/pki/ssl/kubelet-${node_ip}-key.pem --kubeconfig=kubelet-${node_ip}.kubeconfig

    echo "设置上下文参数"
    kubectl config set-context default --cluster=kubernetes --user=system:node:${node_ip} --kubeconfig=kubelet-${node_ip}.kubeconfig

    echo "选择默认上下文"
    kubectl config use-context default --kubeconfig=kubelet-${node_ip}.kubeconfig
    scp kubelet-${node_ip}.kubeconfig ${node_ip}:/etc/kubernetes/kubelet.kubeconfig
  done

提示:

  • 首先说明该操作全部在 k8s-m01 主机执行；
  • 其次主要是考虑安全原因，只提供 master 节点主机具有 kubectl 命令执行操作；
  • 在"设置客户端认证参数"中所指定的证书均在 /root/pki/ssl 目录中，我们创建证书的所有文件也在该目录中生成；

1.2 拷贝 kubelet 所需二进制命令

$ cd
$ export NODE_IPS=(192.168.133.128 192.168.133.129 192.168.133.130 192.168.133.131)
$ for node_ip in ${NODE_IPS[@]};
do
  echo ">>> ${node_ip}"
  scp kubernetes/server/bin/{kubelet,kube-proxy} ${node_ip}:/usr/sbin/
done

1.3 创建 kubelet 配置文件模板

$ export CLUSTER_DNS_DOMAIN="cluster.local."
$ export CLUSTER_DNS_SVC_IP="10.254.0.2"
$ export CLUSTER_CIDR="172.30.0.0/16"

$ cat > /tmp/kubelet.config.json <<EOF
{
  "kind": "KubeletConfiguration",
  "apiVersion": "kubelet.config.k8s.io/v1beta1",
  "authentication": {
    "x509": {
      "clientCAFile": "/etc/kubernetes/ssl/ca.pem"
    },
    "webhook": {
      "enabled": true,
      "cacheTTL": "2m0s"
    },
    "anonymous": {
      "enabled": false
    }
  },
  "authorization": {
    "mode": "Webhook",
    "webhook": {
      "cacheAuthorizedTTL": "5m0s",
      "cacheUnauthorizedTTL": "30s"
    }
  },
  "failSwapOn": false,
  "address": "##NODE_IP##",
  "port": 10250,
  "readOnlyPort": 0,
  "cgroupDriver": "cgroupfs",
  "hairpinMode": "promiscuous-bridge",
  "serializeImagePulls": false,
  "featureGates": {
    "RotateKubeletClientCertificate": true,
    "RotateKubeletServerCertificate": true
  },
  "clusterDomain": "${CLUSTER_DNS_DOMAIN}",
  "clusterDNS": ["${CLUSTER_DNS_SVC_IP}"],
  "podCIDR": "${CLUSTER_CIDR}",
  "resolvConf": "/etc/resolv.conf",
  "runtimeRequestTimeout": "15m"
}
EOF
  • address：API 监听地址，不能为 127.0.0.1，否则 kube-apiserver、heapster 等不能调用 kubelet 的 API；
  • readOnlyPort=0：关闭只读端口(默认 10255)，等效为未指定；
  • authentication.anonymous.enabled：设置为 false，不允许匿名访问 10250 端口；
  • authentication.x509.clientCAFile：指定签名客户端证书的 CA 证书，开启 HTTP 证书认证；
  • authentication.webhook.enabled=true：开启 HTTPs bearer token 认证；
  • 对于未通过 x509 证书和 webhook 认证的请求(kube-apiserver 或其他客户端)，将被拒绝，提示 Unauthorized；
  • authroization.mode=Webhook：kubelet 使用 SubjectAccessReview API 查询 kube-apiserver 某 user、group 是否具有操作资源的权限(RBAC)；
  • featureGates.RotateKubeletClientCertificate/featureGates.RotateKubeletServerCertificate：自动 rotate 证书，证书的有效期取决于 kube-controller-manager 的 --experimental-cluster-signing-duration 参数；
  • 需要 root 账户运行；
为各节点创建和分发 kubelet 配置文件
$ export NODE_IPS=(192.168.133.128 192.168.133.129 192.168.133.130 192.168.133.131)
$ for node_ip in ${NODE_IPS[@]}
  do 
    echo ">>> ${node_ip}"
    scp /tmp/kubelet.config.json root@${node_ip}:/etc/kubernetes/kubelet.config.json
    ssh root@${node_ip} "sed -i 's/##NODE_IP##/${node_ip}/' /etc/kubernetes/kubelet.config.json"
  done
创建和分发 kubelet systemd 服务启动文件
$ cat > /tmp/kubelet.service <<EOF
[Unit]
Description=Kubernetes Kubelet
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=docker.service
Requires=docker.service

[Service]
WorkingDirectory=/var/lib/kubelet
ExecStart=/usr/sbin/kubelet \\
  --cloud-provider= \\
  --cert-dir=/etc/kubernetes/ssl \\
  --kubeconfig=/etc/kubernetes/kubelet.kubeconfig \\
  --config=/etc/kubernetes/kubelet.config.json \\
  --hostname-override=##NODE_IP## \\
  --pod-infra-container-image=hexun/pause-amd64:3.1 \\
  --cni-conf-dir=/etc/cni/net.d \\
  --cni-bin-dir=/opt/cni/bin \\
  --network-plugin=cni \\
  --alsologtostderr=true \\
  --logtostderr=false \\
  --log-dir=/var/log/kubernetes \\
  --register-node=true \\
  --v=2
Restart=on-failure
RestartSec=5

[Install]
WantedBy=multi-user.target
EOF
  • 如果设置了 --hostname-override 选项，则 kube-proxy 也需要设置该选项，否则会出现找不到 Node 的情况；
  • k8s approve kubelet 的 csr 请求后，在 --cert-dir 目录创建证书和私钥文件，然后写入 --kubeconfig文件；

为各节点创建和分发 kubelet systemd unit 文件

$ export NODE_IPS=(192.168.133.128 192.168.133.129 192.168.133.130 192.168.133.131)
$ for node_ip in ${NODE_IPS[@]}
  do 
    echo ">>> ${node_ip}"
    scp /tmp/kubelet.service root@${node_ip}:/etc/systemd/system/kubelet.service
    ssh root@${node_ip} "sed -i 's/##NODE_IP##/${node_ip}/' /etc/systemd/system/kubelet.service"
    ssh root@${node_ip} "mkdir -p {/etc/cni/net.d,/var/log/kubernetes,/var/lib/kube-proxy,/var/lib/kubernetes,/var/run/kubernetes}"
  done

1.4 下载安装 cni 插件

$ export NODE_IPS=(192.168.133.128 192.168.133.129 192.168.133.130 192.168.133.131)
$ for node_ip in ${NODE_IPS[@]}
 do
   echo ">>> ${node_ip}"
   ssh root@${node_ip} "mkdir -p {/etc/cni/net.d,/opt/cni/bin,/var/lib/kube-proxy,/var/lib/kubernetes,/var/run/kubernetes}"
   ssh root@${node_ip} "wget https://github.com/containernetworking/plugins/releases/download/v0.7.4/cni-plugins-amd64-v0.7.4.tgz && tar zxf cni-plugins-amd64-v0.7.4.tgz -C /opt/cni/bin/"
   ssh root@${node_ip} "rm -rf cni-plugins-amd64-v0.7.4.tgz"
 done
创建 cni 网络插件模板
$ export CLUSTER_CIDR="172.30.0.0/16"
$ cat > /tmp/10-default.conf <<EOF
{
   "name": "mynet",
   "type": "bridge",
   "bridge": "mynet0",
   "isDefaultGateway": true,
   "ipMasq": true,
   "hairpinMode": true,
   "ipam": {
       "type": "host-local",
       "subnet": "{{ CLUSTER_CIDR }}"
   }
}
EOF
将 cni 网络配置分发至个节点
$ export NODE_IPS=(192.168.133.128 192.168.133.129 192.168.133.130 192.168.133.131)
$ for node_ip in ${NODE_IPS[@]}
 do
   echo ">>> ${node_ip}"
   scp /tmp/10-default.conf root@${node_ip}:/etc/cni/net.d/
   ssh ${node_ip} "systemctl daemon-reload && systemctl enable kubelet && systemctl start kubelet"
 done

提示:
如果没有指定 cni 命令以及配置文件，kubelet 如果指定了 cni 是无法启动的。

启动 kubelet 服务

$ /usr/sbin/swapoff -a
$ systemctl daemon-reload && systemctl enable kubelet && systemctl restart kubelet

验证节点状态

$ kubectl get node
NAME              STATUS     ROLES    AGE   VERSION
192.168.133.128   Ready    <none>   22h   v1.13.0
192.168.133.129   Ready    <none>   22h   v1.13.0
192.168.133.130   Ready    <none>   22h   v1.13.0
192.168.133.131   Ready    <none>   22h   v1.13.0