etcd 虚拟机与docker搭建etcd集群环境方式

方式一、

虚拟机

主机信息：

主机名	系统	IP
master	CentOS 7.3	10.191.80.55
node01	CentOS 7.3	10.191.80.56
node02	CentOS 7.3	10.191.80.57

环境配置：

设置主机名

 # 分别在每台执行
 hostnamectl set-hostname master
 hostnamectl set-hostname node01
 hostnamectl set-hostname node02

配置本地解析

# 每台都执行
cat >> /etc/hosts << EOF
10.191.80.55    master
10.191.80.56    node01
10.191.80.57    node02
EOF

关闭防火墙，selinux

 # 每台都执行
 systemctl stop firewalld
 systemctl disable firewalld --now
 setenforce 0
 sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config

配置时间同步

 # 每台都执行
 yum install chrony -y && systemctl enable --now chronyd

集群部署

下载etcd二进制包

 # 每台都执行
 cd /opt
 wget https://github.com/etcd-io/etcd/releases/download/v3.5.8/etcd-v3.5.8-linux-amd64.tar.gz
 tar -zxvf etcd-v3.5.8-linux-amd64.tar.gz

编写etcd.conf配置

# master
name: 'master'
data-dir: '/data/etcd'
listen-client-urls: 'http://10.191.80.55:2379'
listen-peer-urls: 'http://10.191.80.55:2380'
advertise-client-urls: 'http://10.191.80.55:2379'
initial-advertise-peer-urls: 'http://10.191.80.55:2380'
initial-cluster: 'master=http://10.191.80.55:2380,node01=http://10.191.80.56:2380,node02=http://10.191.80.57:2380'
initial-cluster-token: 'etcd-cluster'
initial-cluster-state: 'new'

logger: 'zap'
log-level: 'info'
log-rotate-enable: 'true'
log-rotate-size-limit: '100'
log-rotate-age: '30'
log-outputs: [/var/log/etcd.log]

#node01
name: 'node01'
data-dir: '/data/etcd'
listen-client-urls: 'http://10.191.80.56:2379'
listen-peer-urls: 'http://10.191.80.56:2380'
advertise-client-urls: 'http://10.191.80.56:2379'
initial-advertise-peer-urls: 'http://10.191.80.56:2380'
initial-cluster: 'master=http://10.191.80.55:2380,node01=http://10.191.80.56:2380,node02=http://10.191.80.57:2380'
initial-cluster-token: 'etcd-cluster'
initial-cluster-state: 'new'

logger: 'zap'
log-level: 'info'
log-rotate-enable: 'true'
log-rotate-size-limit: '100'
log-rotate-age: '30'
log-outputs: [/var/log/etcd.log]

#node02
name: 'node02'
data-dir: '/data/etcd'
listen-client-urls: 'http://10.191.80.57:2379'
listen-peer-urls: 'http://10.191.80.57:2380'
advertise-client-urls: 'http://10.191.80.57:2379'
initial-advertise-peer-urls: 'http://10.191.80.57:2380'
initial-cluster: 'master=http://10.191.80.55:2380,node01=http://10.191.80.56:2380,node02=http://10.191.80.57:2380'
initial-cluster-token: 'etcd-cluster'
initial-cluster-state: 'new'

logger: 'zap'
log-level: 'info'
log-rotate-enable: 'true'
log-rotate-size-limit: '100'
log-rotate-age: '30'
log-outputs: [/var/log/etcd.log]

启动时的参数含义

--name	      etcd的节点名字,自定义名字,etcd节点的名字不能相同
--data-dir    etcd的数据存储目录
--listen-client-urls	用于当前节点与客户端交互的URL地址，每个节点同样可以向客户端提供多个URL地址，多个地址使用道号分隔节课,端口一般保持默认2379即可
--listen-peer-urls	   用于集群内各个节点之间通信的URL地址，每个节点可以监听多个URL地址,集群内部将通过这些URL地址进行数据交互,例如，Leader节点的选举、Message消息传输或是快照传输等,端口一般保持默认2380即可
--advertise-client-urls	建议使用的客户端通信url,该值用于etcd 代理或etcd成员与etcd节点通信,与listen-client-urls参数值保持一致即可
--initial-advertise-peer-urls	建议用于集群内部节点之间交互的URL地址,节点间将以该值进行通信,与listen-peer-urls参数值保持一致即可
--initial-cluster	集群中所有的initial-advertise-peer-urls的合集,etcd启动的时候,会通过这个配置找到其他etcd节点的列表
--initial-cluster-token	 节点的token值,该值可自定义,设置该值后集群将生成唯一id,并为每个节点也生成唯一id,当使用相同配置文件再启动一个集群时,只要该token值不一样,etcd 集群就不会相互影响
--initial-cluster-state	初始化时集群的状态,可取值:new和existing,new代表新建的集群,existing 代表加入已经存在的集群

--logger：为结构化日志记录指定'zap'或'capnslog'。
--log-level：日志级别，可以是debug、info、warning、error或者critical。
--log-rotate-enable：启用日志轮换。
--log-rotate-size-limit：设置轮换文件大小限制为100MB
--log-rotate-age：轮换文件的最大保留时间为30天
--log-outputs：日志输出方式，可以是stdout、stderr或者指定一个文件路径。 例如：[stdout\stderr\自定义路径]

编写etcd.service配置文件，加入到systemd系统进程（3台机器都要添加）

# master\node01\node02
cat>/usr/lib/systemd/system/etcd.service<<'EOF'
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target

[Service]
Type=notify
EnvironmentFile=/etc/etcd/etcd.conf
ExecStart=/opt/etcd-v3.5.8-linux-amd64/etcd --config-file /etc/etcd/etcd.conf
Restart=on-failure
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

启动etcd集群

因为已经将etcd加入到了systemd系统进程，所以需要重新加载一下配置，在启动。
注意的：最先启动的etcd，会成为集群的leader。(主)

启动etcd服务
先启动master、然后依次执行node01、node02

# 启动etcd服务
systemctl daemon-reload
systemctl start etcd
systemctl enable etcd

查看etcd集群状态

[root@master etcd-v3.5.8-linux-amd64]# ./etcdctl --endpoints="http://master:2379,http://node01:2379,http://node02:2379" endpoint status  --write-out=table
+--------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
|      ENDPOINT      |        ID        | VERSION | DB SIZE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS |
+--------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| http://master:2379 |  e25edc71ef30b4d |   3.5.8 |   20 kB |      true |      false |        13 |         34 |                 34 |        |
| http://node01:2379 | 619cf852ff2000ae |   3.5.8 |   20 kB |     false |      false |        13 |         34 |                 34 |        |
| http://node02:2379 | a8ad0f1fb32790e8 |   3.5.8 |   20 kB |     false |      false |        13 |         34 |                 34 |        |
+--------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
# 可以看出 master节点是leader

查看etcd集群健康状态

[root@master etcd-v3.5.8-linux-amd64]# ./etcdctl --endpoints="http://master:2379,http://node01:2379,http://node02:2379" endpoint health  --write-out=table
+--------------------+--------+------------+-------+
|      ENDPOINT      | HEALTH |    TOOK    | ERROR |
+--------------------+--------+------------+-------+
| http://master:2379 |   true | 2.498557ms |       |
| http://node02:2379 |   true |  2.71546ms |       |
| http://node01:2379 |   true | 2.672862ms |       |
+--------------------+--------+------------+-------+

列出etcd集群成员节点

[root@master etcd-v3.5.8-linux-amd64]# ./etcdctl --endpoints="http://master:2379,http://node01:2379,http://node02:2379" member list  --write-out=table
+------------------+---------+--------+--------------------------+--------------------------+------------+
|        ID        | STATUS  |  NAME  |        PEER ADDRS        |       CLIENT ADDRS       | IS LEARNER |
+------------------+---------+--------+--------------------------+--------------------------+------------+
|  e25edc71ef30b4d | started | master | http://10.191.80.55:2380 | http://10.191.80.55:2379 |      false |
| 619cf852ff2000ae | started | node01 | http://10.191.80.56:2380 | http://10.191.80.56:2379 |      false |
| a8ad0f1fb32790e8 | started | node02 | http://10.191.80.57:2380 | http://10.191.80.57:2379 |      false |
+------------------+---------+--------+--------------------------+--------------------------+------------+

方式二、

docker

ps：使用docker安装可以在同一台机器安装也可以在三台机器拆分安装（本文docker安装方式是在一台机器进行）

创建 docker 网络

docker network create app-etcd

mkdir /root/etcd-n1
mkdir /root/etcd-n2
mkdir /root/etcd-n3

拉取 docker 镜像

docker pull quay.io/coreos/etcd:v3.5.1

创建节点 1

docker run -d \
  -p 10379:2379 \
  -p 10380:2380 \
  --net app-etcd \
  --volume=/root/etcd-n1:/etcd-data \
  --name etcd-n1 \
  quay.io/coreos/etcd:v3.5.1 \
  /usr/local/bin/etcd \
  --name etcd-n1 \
  --data-dir /etcd-data \
  --listen-client-urls http://0.0.0.0:2379 \
  --advertise-client-urls http://etcd-n1:2379 \
  --listen-peer-urls http://0.0.0.0:2380 \
  --initial-advertise-peer-urls http://etcd-n1:2380 \
  --initial-cluster etcd-n1=http://etcd-n1:2380,etcd-n2=http://etcd-n2:2380,etcd-n3=http://etcd-n3:2380 \
  --initial-cluster-token myetcdToken \
  --initial-cluster-state new \
  --log-level info \
  --logger zap \
  --log-outputs stderr

创建节点 2

 docker run -d \
  -p 20379:2379 \
  -p 20380:2380 \
  --net app-etcd \
  --volume=/root/etcd-n2:/etcd-data \
  --name etcd-n2 \
  quay.io/coreos/etcd:v3.5.1 \
  /usr/local/bin/etcd \
  --name etcd-n2 \
  --data-dir /etcd-data \
  --listen-client-urls http://0.0.0.0:2379 \
  --advertise-client-urls http://etcd-n2:2379 \
  --listen-peer-urls http://0.0.0.0:2380 \
  --initial-advertise-peer-urls http://etcd-n2:2380 \
  --initial-cluster etcd-n1=http://etcd-n1:2380,etcd-n2=http://etcd-n2:2380,etcd-n3=http://etcd-n3:2380 \
  --initial-cluster-token myetcdToken \
  --initial-cluster-state new \
  --log-level info \
  --logger zap \
  --log-outputs stderr

创建节点 3

 docker run -d \
  -p 30379:2379 \
  -p 30380:2380 \
  --net app-etcd \
  --volume=/root/etcd-n3:/etcd-data \
  --name etcd-n3 \
  quay.io/coreos/etcd:v3.5.1 \
  /usr/local/bin/etcd \
  --name etcd-n3 \
  --data-dir /etcd-data \
  --listen-client-urls http://0.0.0.0:2379 \
  --advertise-client-urls http://etcd-n3:2379 \
  --listen-peer-urls http://0.0.0.0:2380 \
  --initial-advertise-peer-urls http://etcd-n3:2380 \
  --initial-cluster etcd-n1=http://etcd-n1:2380,etcd-n2=http://etcd-n2:2380,etcd-n3=http://etcd-n3:2380 \
  --initial-cluster-token myetcdToken \
  --initial-cluster-state new \
  --log-level info \
  --logger zap \
  --log-outputs stderr

测试：

# 搭建好的 docker 服务列表
[root@vmCentos ~]# docker ps
CONTAINER ID   IMAGE                        COMMAND                  CREATED              STATUS              PORTS                                                                                      NAMES
9195d985bc07   quay.io/coreos/etcd:v3.5.1   "/usr/local/bin/etcd…"   12 seconds ago       Up 11 seconds       0.0.0.0:20379->2379/tcp, :::20379->2379/tcp, 0.0.0.0:20380->2380/tcp, :::20380->2380/tcp   etcd-n2
09c91092fa51   quay.io/coreos/etcd:v3.5.1   "/usr/local/bin/etcd…"   17 seconds ago       Up 16 seconds       0.0.0.0:30379->2379/tcp, :::30379->2379/tcp, 0.0.0.0:30380->2380/tcp, :::30380->2380/tcp   etcd-n3
8a376c0ed897   quay.io/coreos/etcd:v3.5.1   "/usr/local/bin/etcd…"   About a minute ago   Up About a minute   0.0.0.0:10379->2379/tcp, :::10379->2379/tcp, 0.0.0.0:10380->2380/tcp, :::10380->2380/tcp   etcd-n1

# 执行客户端调用测试
[root@vmCentos etcd-v3.5.1-linux-amd64]# ./etcdctl --endpoints=http://127.0.0.1:20380 put 123 key
OK
[root@vmCentos etcd-v3.5.1-linux-amd64]# ./etcdctl --endpoints=http://127.0.0.1:30380 put 123 key1
OK
[root@vmCentos etcd-v3.5.1-linux-amd64]# ./etcdctl --endpoints=http://127.0.0.1:10380 get 123
123
key1