参考文档node
1.下载etcd最新版本linux
官方地址git
[root@k8s-node1 etcd]# wget https://github.com/coreos/etcd/releases/download/v3.3.7/etcd-v3.3.7-linux-amd64.tar.gz --2019-11-03 21:42:33-- https://github.com/coreos/etcd/releases/download/v3.3.7/etcd-v3.3.7-linux-amd64.tar.gz Resolving github.com (github.com)... 140.82.114.3 Connecting to github.com (github.com)|140.82.114.3|:443... connected. HTTP request sent, awaiting response... 301 Moved Permanently Location: https://github.com/etcd-io/etcd/releases/download/v3.3.7/etcd-v3.3.7-linux-amd64.tar.gz [following] --2019-11-03 21:42:34-- https://github.com/etcd-io/etcd/releases/download/v3.3.7/etcd-v3.3.7-linux-amd64.tar.gz Reusing existing connection to github.com:443. HTTP request sent, awaiting response... 302 Found Location: https://github-production-release-asset-2e65be.s3.amazonaws.com/11225014/dd736838-6974-11e8-8737-c3613eec99eb?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIWNJYAX4CSVEH53A%2F20191104%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20191104T024234Z&X-Amz-Expires=300&X-Amz-Signature=c45bc2504edea625b70821e017939952af01e41fc87e44ce6b82acae00bd3cb3&X-Amz-SignedHeaders=host&actor_id=0&response-content-disposition=attachment%3B%20filename%3Detcd-v3.3.7-linux-amd64.tar.gz&response-content-type=application%2Foctet-stream [following] --2019-11-03 21:42:34-- https://github-production-release-asset-2e65be.s3.amazonaws.com/11225014/dd736838-6974-11e8-8737-c3613eec99eb?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIWNJYAX4CSVEH53A%2F20191104%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20191104T024234Z&X-Amz-Expires=300&X-Amz-Signature=c45bc2504edea625b70821e017939952af01e41fc87e44ce6b82acae00bd3cb3&X-Amz-SignedHeaders=host&actor_id=0&response-content-disposition=attachment%3B%20filename%3Detcd-v3.3.7-linux-amd64.tar.gz&response-content-type=application%2Foctet-stream Resolving github-production-release-asset-2e65be.s3.amazonaws.com (github-production-release-asset-2e65be.s3.amazonaws.com)... 52.216.114.171 Connecting to github-production-release-asset-2e65be.s3.amazonaws.com (github-production-release-asset-2e65be.s3.amazonaws.com)|52.216.114.171|:443... connected. HTTP request sent, awaiting response... 200 OK Length: 11271567 (11M) [application/octet-stream] Saving to: ‘etcd-v3.3.7-linux-amd64.tar.gz’ 100%[==============================================================================================>] 11,271,567 1.01MB/s in 13s 2019-11-03 21:42:49 (816 KB/s) - ‘etcd-v3.3.7-linux-amd64.tar.gz’ saved [11271567/11271567] [root@k8s-node1 etcd]#
[root@k8s-node1 etcd]# ls etcd-v3.3.7-linux-amd64.tar.gz
解包github
[root@k8s-node1 etcd]# tar -zxvf etcd-v3.3.7-linux-amd64.tar.gz
2.分发etcd文件到全部节点web
[root@k8s-node1 etcd]# cp etcd-v3.3.7-linux-amd64/etcd* /opt/k8s/bin [root@k8s-node1 etcd]# scp etcd-v3.3.7-linux-amd64/etcd* root@k8s-node2:/opt/k8s/bin etcd 100% 18MB 93.7MB/s 00:00 etcdctl 100% 15MB 95.9MB/s 00:00 [root@k8s-node1 etcd]# scp etcd-v3.3.7-linux-amd64/etcd* root@k8s-node3:/opt/k8s/bin etcd 100% 18MB 88.5MB/s 00:00 etcdctl 100% 15MB 64.3MB/s 00:00 [root@k8s-node1 etcd]#
3.添加etcd文件的执行权限json
[root@k8s-node1 etcd]# chmod +x /opt/k8s/bin/* [root@k8s-node1 etcd]# ssh k8s-node2 "chmod +x /opt/k8s/bin/*" [root@k8s-node1 etcd]# ssh k8s-node3 "chmod +x /opt/k8s/bin/*"
4.建立etcd证书和密钥app
建立签名请求ssh
hosts 字段指定受权使用该证书的 etcd 节点 IP 或域名列表,这里将 etcd 集群的三个节点 IP 都列在其中.ide
[root@k8s-node1 etcd]# pwd
/opt/k8s/k8s_software/etcd
[root@k8s-node1 etcd]# cat etcd-csr.json
{
"CN": "etcd",
"hosts": [
"127.0.0.1",
"192.168.174.128",
"192.168.174.129",
"192.168.174.130"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "SZ",
"L": "SZ",
"O": "k8s",
"OU": "4Paradigm"
}
]
}
[root@k8s-node1 etcd]#
生成证书和密钥测试
[root@k8s-node1 etcd]# cfssl gencert -ca=/etc/kubernetes/cert/ca.pem -ca-key=/etc/kubernetes/cert/ca-key.pem -config=/etc/kubernetes/cert/ca-config.json -profile=kubernetes etcd-csr.json | cfssljson -bare etcd
2019/10/30 03:27:04 [INFO] generate received request
2019/10/30 03:27:04 [INFO] received CSR
2019/10/30 03:27:04 [INFO] generating key: rsa-2048
2019/10/30 03:27:05 [INFO] encoded CSR
2019/10/30 03:27:05 [INFO] signed certificate with serial number 578718875980449416776664445360856794460879035803
2019/10/30 03:27:05 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").
[root@k8s-node1 etcd]# ls
etcd.csr etcd-csr.json etcd-key.pem etcd.pem
分发证书和密钥
[root@k8s-node1 etcd]# cp *.pem /etc/etcd/cert/ [root@k8s-node1 etcd]# scp *.pem root@k8s-node2:/etc/etcd/cert/ etcd-key.pem 100% 1679 1.3MB/s 00:00 etcd.pem 100% 1415 1.1MB/s 00:00 [root@k8s-node1 etcd]# scp *.pem root@k8s-node3:/etc/etcd/cert/ etcd-key.pem 100% 1679 1.8MB/s 00:00 etcd.pem 100% 1415 1.3MB/s 00:00 [root@k8s-node1 etcd]#
5.建立etcd的systemd unit文件
[root@k8s-node1 etcd]# pwd
/opt/k8s/k8s_software/etcd
[root@k8s-node1 etcd]# cat etcd.service.template
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
Documentation=https://github.com/coreos
[Service]
User=k8s
Type=notify
WorkingDirectory=/var/lib/etcd/
ExecStart=/opt/k8s/bin/etcd \
--data-dir=/var/lib/etcd \
--name=##NODE_NAME## \
--cert-file=/etc/etcd/cert/etcd.pem \
--key-file=/etc/etcd/cert/etcd-key.pem \\
--trusted-ca-file=/etc/kubernetes/cert/ca.pem \
--peer-cert-file=/etc/etcd/cert/etcd.pem \
--peer-key-file=/etc/etcd/cert/etcd-key.pem \
--peer-trusted-ca-file=/etc/kubernetes/cert/ca.pem \
--peer-client-cert-auth \
--client-cert-auth \
--listen-peer-urls=https://##NODE_IP##:2380 \
--initial-advertise-peer-urls=https://##NODE_IP##:2380 \
--listen-client-urls=https://##NODE_IP##:2379,http://127.0.0.1:2379\
--advertise-client-urls=https://##NODE_IP##:2379 \
--initial-cluster-token=etcd-cluster-0 \
--initial-cluster=${ETCD_NODES} \
--initial-cluster-state=new
Restart=on-failure
RestartSec=5
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
[root@k8s-node1 etcd]#
##User:指定以 k8s 帐户运行.
##WorkingDirectory --data-dir:指定工做目录和数据目录为/var/lib/etcd,需在启动服务前建立这个目录.
##--name:指定节点名称,当 --initial-cluster-state 值为 new 时,--name 的参数值必须位于 --initial-cluster 列表中.
##--cert-file 、 --key-file:etcd server 与 client 通讯时使用的证书和私钥.
##--trusted-ca-file:签名 client 证书的 CA 证书,用于验证 client 证书.
##--peer-cert-file --peer-key-file:etcd 与 peer 通讯使用的证书和私钥.
##--peer-trusted-ca-file:签名 peer 证书的 CA 证书,用于验证 peer 证书.
6.分发生成的 systemd unit 文件,并修改好各节点配置文件里的##NODE_NAME##和##NODE_IP##
[root@k8s-node1 etcd]# cp etcd.service.template /etc/systemd/system/etcd.service [root@k8s-node1 etcd]# scp etcd.service.template root@k8s-node2:/etc/systemd/system/etcd.service etcd.service.template 100% 1020 972.4KB/s 00:00 [root@k8s-node1 etcd]# scp etcd.service.template root@k8s-node3:/etc/systemd/system/etcd.service etcd.service.template 100% 1020 786.5KB/s 00:00 [root@k8s-node1 etcd]#
各个节点修改下,修改为对应的name和ip.
[root@k8s-node1 etcd]# sed -i 's/##NODE_NAME##/k8s-node1/' /etc/systemd/system/etcd.service
[root@k8s-node1 etcd]# sed -i 's/##NODE_IP##/192\.168\.174\.128/g' /etc/systemd/system/etcd.service
[root@k8s-node1 etcd]# sed -i 's/\${ETCD_NODES}/k8s-node1=https:\/\/192\.168\.174\.128:2380,k8s-node2=https:\/\/192\.168\.174\.129:2380,k8s-node3=https:\/\/192\.168\.174\.130:2380/' /etc/systemd/system/etcd.service
改好后参考见下:
[root@k8s-node1 etcd]# cat /etc/systemd/system/etcd.service [Unit] Description=Etcd Server After=network.target After=network-online.target Wants=network-online.target Documentation=https://github.com/coreos [Service] User=k8s Type=notify WorkingDirectory=/var/lib/etcd/ ExecStart=/opt/k8s/bin/etcd \ --data-dir=/var/lib/etcd \ --name=k8s-node1 \ --cert-file=/etc/etcd/cert/etcd.pem \ --key-file=/etc/etcd/cert/etcd-key.pem \ --trusted-ca-file=/etc/kubernetes/cert/ca.pem \ --peer-cert-file=/etc/etcd/cert/etcd.pem \ --peer-key-file=/etc/etcd/cert/etcd-key.pem \ --peer-trusted-ca-file=/etc/kubernetes/cert/ca.pem \ --peer-client-cert-auth \ --client-cert-auth \ --listen-peer-urls=https://192.168.174.128:2380 \ --initial-advertise-peer-urls=https://192.168.174.128:2380 \ --listen-client-urls=https://192.168.174.128:2379,http://127.0.0.1:2379\ --advertise-client-urls=https://192.168.174.128:2379 \ --initial-cluster-token=etcd-cluster-0 \ --initial-cluster=k8s-node1=https://192.168.174.128:2380,k8s-node2=https://192.168.174.129:2380,k8s-node3=https://192.168.174.130:2380 \ --initial-cluster-state=new Restart=on-failure RestartSec=5 LimitNOFILE=65536 [Install] WantedBy=multi-user.target [root@k8s-node1 etcd]#
7.检查并添加权限,全部节点都须要操做
[root@k8s-node1 etcd]# ll /etc/etcd/cert/ total 8 -rw------- 1 root root 1679 Oct 30 03:29 etcd-key.pem -rw-r--r-- 1 root root 1415 Oct 30 03:29 etcd.pem [root@k8s-node1 etcd]# chown -R k8s /etc/etcd/cert/ [root@k8s-node1 etcd]# chmod +x -R /etc/etcd/cert/
[root@k8s-node1 etcd]# cd /etc/kubernetes/cert/ [root@k8s-node1 cert]# ll total 20 -rw-r--r-- 1 root root 292 Oct 30 00:22 ca-config.json -rw-r--r-- 1 root root 993 Oct 30 00:22 ca.csr -rw-r--r-- 1 root root 201 Oct 30 00:22 ca-csr.json -rw------- 1 root root 1675 Oct 30 00:22 ca-key.pem -rw-r--r-- 1 root root 1338 Oct 30 00:22 ca.pem [root@k8s-node1 cert]# chown -R k8s /etc/kubernetes/cert/ [root@k8s-node1 cert]# chmod -R +x /etc/kubernetes/cert
8.启动etcd
systemctl daemon-reload && systemctl enable etcd && systemctl restart etcd
整个节点都重启下,再验证状态
[root@k8s-node3 ~]# etcdctl --version etcdctl version: 3.3.7 API version: 2 [root@k8s-node3 ~]# etcdctl cluster-health failed to check the health of member 1c83ad9421d77430 on https://192.168.174.130:2379: Get https://192.168.174.130:2379/health: x509: certificate signed by unknown authority member 1c83ad9421d77430 is unreachable: [https://192.168.174.130:2379] are all unreachable failed to check the health of member 5eec694677c3c515 on https://192.168.174.129:2379: Get https://192.168.174.129:2379/health: x509: certificate signed by unknown authority member 5eec694677c3c515 is unreachable: [https://192.168.174.129:2379] are all unreachable failed to check the health of member 65f8d952bfce7d85 on https://192.168.174.128:2379: Get https://192.168.174.128:2379/health: x509: certificate signed by unknown authority member 65f8d952bfce7d85 is unreachable: [https://192.168.174.128:2379] are all unreachable cluster is unavailable [root@k8s-node3 ~]# etcdctl member list client: etcd cluster is unavailable or misconfigured; error #0: x509: certificate signed by unknown authority ; error #1: x509: certificate signed by unknown authority ; error #2: x509: certificate signed by unknown authority [root@k8s-node3 ~]#
带上证书后正常
[root@k8s-node1 ~]# etcdctl --ca-file=/etc/kubernetes/cert/ca.pem --cert-file=/etc/etcd/cert/etcd.pem --key-file=/etc/etcd/cert/etcd-key.pem cluster-health member 1c83ad9421d77430 is healthy: got healthy result from https://192.168.174.130:2379 member 5eec694677c3c515 is healthy: got healthy result from https://192.168.174.129:2379 member 65f8d952bfce7d85 is healthy: got healthy result from https://192.168.174.128:2379 cluster is healthy [root@k8s-node1 ~]# etcdctl --ca-file=/etc/kubernetes/cert/ca.pem --cert-file=/etc/etcd/cert/etcd.pem --key-file=/etc/etcd/cert/etcd-key.pem member list 1c83ad9421d77430: name=k8s-node3 peerURLs=https://192.168.174.130:2380 clientURLs=https://192.168.174.130:2379 isLeader=false 5eec694677c3c515: name=k8s-node2 peerURLs=https://192.168.174.129:2380 clientURLs=https://192.168.174.129:2379 isLeader=true 65f8d952bfce7d85: name=k8s-node1 peerURLs=https://192.168.174.128:2380 clientURLs=https://192.168.174.128:2379 isLeader=false
测试下
[root@k8s-node1 ~]# etcdctl --ca-file=/etc/kubernetes/cert/ca.pem --cert-file=/etc/etcd/cert/etcd.pem --key-file=/etc/etcd/cert/etcd-key.pem mkdir /test [root@k8s-node1 ~]# etcdctl --ca-file=/etc/kubernetes/cert/ca.pem --cert-file=/etc/etcd/cert/etcd.pem --key-file=/etc/etcd/cert/etcd-key.pem mk /test/t1 00 00 [root@k8s-node1 ~]# etcdctl --ca-file=/etc/kubernetes/cert/ca.pem --cert-file=/etc/etcd/cert/etcd.pem --key-file=/etc/etcd/cert/etcd-key.pem ls /test /test/t1 [root@k8s-node1 ~]# etcdctl --ca-file=/etc/kubernetes/cert/ca.pem --cert-file=/etc/etcd/cert/etcd.pem --key-file=/etc/etcd/cert/etcd-key.pem get /test/t1 00 [root@k8s-node1 ~]#
[root@k8s-node2 ~]# etcdctl --ca-file=/etc/kubernetes/cert/ca.pem --cert-file=/etc/etcd/cert/etcd.pem --key-file=/etc/etcd/cert/etcd-key.pem get /test/t1 00 [root@k8s-node2 ~]#
[root@k8s-node3 ~]# etcdctl --ca-file=/etc/kubernetes/cert/ca.pem --cert-file=/etc/etcd/cert/etcd.pem --key-file=/etc/etcd/cert/etcd-key.pem get /test/t1 00 [root@k8s-node3 ~]#