一、前置准备

启动redis服务

# 查看redis进程
ps -ef |grep redis# 启动redis服务
/usr/local/redis/bin/redis-server /usr/local/redis/conf/redis.conf#查看主从节点信息
/usr/local/redis/bin/redis-cli -a 123456
info replication

启动sentinel服务

声明：在启动3个redis服务的前提下，再依次启动3个sentinel服务

/usr/local/redis/bin/redis-sentinel /usr/local/redis/conf/sentiel.conf

查看sentiel启动日志

tail -f /usr/local/redis/log/sentinel.log

二、实战故障转移

2.1. 主节点停止

[root@bigdata01 redis-6.2.0]# /usr/local/redis/bin/redis-cli -a 123456
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
127.0.0.1:6379> info replcation
127.0.0.1:6379> info replication
# Replication
role:master
connected_slaves:2
slave0:ip=xxx.xxx.92.101,port=6379,state=online,offset=3xxx,lag=1
slave1:ip=xxx.xxx.92.102,port=6379,state=online,offset=3xxx,lag=1
master_failover_state:no-failover
master_replid:4f07bacde969998356194ad698c6dd7e2f6f7da2
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:3xxx
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1391
repl_backlog_histlen:1778
127.0.0.1:6379> SHUTDOWN
not connected> 

2.2. Sentienl监控日志

mymaster 主节点 xxx.xxx.92.100 6379 被添加主观下线标记，其他的sentinel每秒一次和xxx.xxx.92.100 6379的通信
# +sdown master mymaster xxx.xxx.92.100 6379
启动一个新选举流程，记录当前选举的次数
# +new-epoch 1
投票选举一个sentinel去做故障转移这件事
# +vote-for-leader a20913bfcbeb3a5d8d58ce1bd372cc40455xxx10 1
客观下线：当前环境内多个sentinel节点对我们的某一个服务器标记为主观下线，满足quorum 仲裁的选项值之后，标记为客观下线
# +odown master mymaster xxx.xxx.92.100 6379 #quorum 3/2
# Next failover delay: I will not start a failover before Sun Jul  4 22:39:56 2021
# +config-update-from sentinel a20913bfcbeb3a5d8d58ce1bd372cc40455xxx10 xxx.xxx.92.1026379
# +switch-master mymaster xxx.xxx.92.100 6379 xxx.xxx.92.101 6379
* +slave slave xxx.xxx.92.102:6379 xxx.xxx.92.102 6379 @ mymaster xxx.xxx.92.101 6379
* +slave slave xxx.xxx.92.100:6379 xxx.xxx.92.100 6379 @ mymaster xxx.xxx.92.101 6379
# +sdown slave xxx.xxx.92.100:6379 xxx.xxx.92.100 6379 @ mymaster xxx.xxx.92.101 6379

2.3. 节点信息监控

查看故障转移后的节点信息
01节点信息

[root@bigdata02 ~]# /usr/local/redis/bin/redis-cli -a 123456
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
127.0.0.1:6379> info replication
# Replication
role:slave
master_host:xxx.xxx.92.100
master_port:6379
master_link_status:up
master_last_io_seconds_ago:6
master_sync_in_progress:0
slave_repl_offset:3196
slave_priority:100
slave_read_only:1
connected_slaves:0
master_failover_state:no-failover
master_replid:4f07bacde969998356194ad698c6dd7e2f6f7da2
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:3196
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1391
repl_backlog_histlen:1806# 故障转移后 -------------------------------------------------------------------
127.0.0.1:6379> info replication
# Replication
role:master
connected_slaves:1
slave0:ip=xxx.xxx.92.102,port=6379,state=online,offset=380674,lag=1
master_failover_state:no-failover
master_replid:424069affa171e62af3e2c65a2ee1cb27ecb6c2f
master_replid2:4f07bacde969998356194ad698c6dd7e2f6f7da2
master_repl_offset:380674
second_repl_offset:156702
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1391
repl_backlog_histlen:379284
127.0.0.1:6379> 

02节点信息

[root@bigdata03 ~]# /usr/local/redis/bin/redis-cli -a 123456
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
127.0.0.1:6379> info replication
# Replication
role:slave
master_host:xxx.xxx.92.100
master_port:6379
master_link_status:up
master_last_io_seconds_ago:4
master_sync_in_progress:0
slave_repl_offset:3210
slave_priority:100
slave_read_only:1
connected_slaves:0
master_failover_state:no-failover
master_replid:4f07bacde969998356194ad698c6dd7e2f6f7da2
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:3210
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1433
repl_backlog_histlen:1778# 故障转移后 -------------------------------------------------------------------
127.0.0.1:6379> info replication
# Replication
role:slave
master_host:xxx.xxx.92.101
master_port:6379
master_link_status:up
master_last_io_seconds_ago:0
master_sync_in_progress:0
slave_repl_offset:381546
slave_priority:100
slave_read_only:1
connected_slaves:0
master_failover_state:no-failover
master_replid:424069affa171e62af3e2c65a2ee1cb27ecb6c2f
master_replid2:4f07bacde969998356194ad698c6dd7e2f6f7da2
master_repl_offset:381546
second_repl_offset:156702
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1433
repl_backlog_histlen:380114
127.0.0.1:6379> 

2.4. 故障转移后变化

01节点由slave节点变成了master节点，02仍然后slave节点

2.5. 重新启动

故障的 100节点(原主节点)重新启动

/usr/local/redis/bin/redis-server /usr/local/redis/conf/redis.conf

2.6. sentine日志

* +convert-to-slave slave xxx.xxx.92.100:6379 xxx.xxx.92.100 6379 @ mymaster xxx.xxx.92.101 6379

2.7. 节点信息监控

[root@bigdata01 redis-6.2.0]# /usr/local/redis/bin/redis-cli -a 123456
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
127.0.0.1:6379> info replication
# Replication
role:slave
master_host:xxx.xxx.92.101
master_port:6379
master_link_status:up
master_last_io_seconds_ago:0
master_sync_in_progress:0
slave_repl_offset:528510
slave_priority:100
slave_read_only:1
connected_slaves:0
master_failover_state:no-failover
master_replid:424069affa171e62af3e2c65a2ee1cb27ecb6c2f
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:528510
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:518136
repl_backlog_histlen:10375
127.0.0.1:6379>

01节点信息

[root@bigdata02 ~]# /usr/local/redis/bin/redis-cli -a 123456
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
127.0.0.1:6379> info replication
# Replication
role:slave
master_host:xxx.xxx.92.100
master_port:6379
master_link_status:up
master_last_io_seconds_ago:6
master_sync_in_progress:0
slave_repl_offset:3196
slave_priority:100
slave_read_only:1
connected_slaves:0
master_failover_state:no-failover
master_replid:4f07bacde969998356194ad698c6dd7e2f6f7da2
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:3196
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1391
repl_backlog_histlen:1806# 故障转移后 -------------------------------------------------------------------
127.0.0.1:6379> info replication
# Replication
role:master
connected_slaves:1
slave0:ip=xxx.xxx.92.102,port=6379,state=online,offset=380674,lag=1
master_failover_state:no-failover
master_replid:424069affa171e62af3e2c65a2ee1cb27ecb6c2f
master_replid2:4f07bacde969998356194ad698c6dd7e2f6f7da2
master_repl_offset:380674
second_repl_offset:156702
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1391
repl_backlog_histlen:379284#原主节点重新启动后 -------------------------------------------------------------------
127.0.0.1:6379> info replication
# Replication
role:master
connected_slaves:2
slave0:ip=xxx.xxx.92.102,port=6379,state=online,offset=537589,lag=1
slave1:ip=xxx.xxx.92.100,port=6379,state=online,offset=537732,lag=0
master_failover_state:no-failover
master_replid:424069affa171e62af3e2c65a2ee1cb27ecb6c2f
master_replid2:4f07bacde969998356194ad698c6dd7e2f6f7da2
master_repl_offset:537875
second_repl_offset:156702
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1391
repl_backlog_histlen:536485
127.0.0.1:6379> 

02节点信息

[root@bigdata03 ~]# /usr/local/redis/bin/redis-cli -a 123456
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
127.0.0.1:6379> info replication
# Replication
role:slave
master_host:xxx.xxx.92.100
master_port:6379
master_link_status:up
master_last_io_seconds_ago:4
master_sync_in_progress:0
slave_repl_offset:3210
slave_priority:100
slave_read_only:1
connected_slaves:0
master_failover_state:no-failover
master_replid:4f07bacde969998356194ad698c6dd7e2f6f7da2
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:3210
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1433
repl_backlog_histlen:1778# 故障转移后 -------------------------------------------------------------------
127.0.0.1:6379> info replication
# Replication
role:slave
master_host:xxx.xxx.92.101
master_port:6379
master_link_status:up
master_last_io_seconds_ago:0
master_sync_in_progress:0
slave_repl_offset:381546
slave_priority:100
slave_read_only:1
connected_slaves:0
master_failover_state:no-failover
master_replid:424069affa171e62af3e2c65a2ee1cb27ecb6c2f
master_replid2:4f07bacde969998356194ad698c6dd7e2f6f7da2
master_repl_offset:381546
second_repl_offset:156702
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1433
repl_backlog_histlen:380114#原主节点重新启动后 -------------------------------------------------------------------
127.0.0.1:6379> info replication
# Replication
role:slave
master_host:xxx.xxx.92.101
master_port:6379
master_link_status:up
master_last_io_seconds_ago:0
master_sync_in_progress:0
slave_repl_offset:556005
slave_priority:100
slave_read_only:1
connected_slaves:0
master_failover_state:no-failover
master_replid:424069affa171e62af3e2c65a2ee1cb27ecb6c2f
master_replid2:4f07bacde969998356194ad698c6dd7e2f6f7da2
master_repl_offset:556005
second_repl_offset:156702
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1433
repl_backlog_histlen:554573
127.0.0.1:6379> 

2.8. 配置文件变化

原100主节点

# 放行访问IP的限制 多个用空格隔开
# 0.0.0.0 代表无限制
bind 0.0.0.0# 后台启动开启
daemonize yes# 日志存储目录以及日志文件名
logfile "/usr/local/redis/log/redis.log"# rdb数据文件名
dbfilename "dump.rdb"# aof模式开启
appendonly no# aof数据文件名
appendfilename "appendonly.aof"#rdb数据文件和aof数据文件存储目录
dir "/usr/local/redis/data"# 设置密码
requirepass 123456# 从节点访问主节点的密码(必须和requirepass 一致)
masterauth "123456"#从节点只读模式
replica-read-only yes
# Generated by CONFIG REWRITE
pidfile "/var/run/redis.pid"
save 3600 1
save 300 100
save 60 10000
user default on #8d969eef6ecad3c29a3a629280e686cf0c3f5d5a86aff3ca12020c923adc6c92 ~* &* +@all
replicaof xxx.xxx.92.101 6379

变化：最下面添加了主节点信息

原01节点

# 放行访问IP的限制 多个用空格隔开
# 0.0.0.0 代表无限制
bind 0.0.0.0# 后台启动开启
daemonize yes# 日志存储目录以及日志文件名
logfile "/usr/local/redis/log/redis.log"# rdb数据文件名
dbfilename "dump.rdb"# aof模式开启
appendonly no# aof数据文件名
appendfilename "appendonly.aof"#rdb数据文件和aof数据文件存储目录
dir "/usr/local/redis/data"# 设置密码
requirepass 123456# 从节点访问主节点的密码(必须和requirepass 一致)
masterauth "123456"#从节点只读模式
replica-read-only yes# 从节点从属于那个主节点# Generated by CONFIG REWRITE
pidfile "/var/run/redis.pid"
save 3600 1
save 300 100
save 60 10000
user default on #8d969eef6ecad3c29a3a629280e686cf0c3f5d5a86aff3ca12020c923adc6c92 ~* &* +@all

变化：原来配置的主节点信息没了

02节点信息

# 放行访问IP的限制 多个用空格隔开
# 0.0.0.0 代表无限制
bind 0.0.0.0# 后台启动开启
daemonize yes# 日志存储目录以及日志文件名
logfile "/usr/local/redis/log/redis.log"# rdb数据文件名
dbfilename "dump.rdb"# aof模式开启
appendonly no# aof数据文件名
appendfilename "appendonly.aof"#rdb数据文件和aof数据文件存储目录
dir "/usr/local/redis/data"# 设置密码
requirepass 123456# 从节点访问主节点的密码(必须和requirepass 一致)
masterauth "123456"#从节点只读模式
replica-read-only yes# 从节点从属于那个主节点
replicaof xxx.xxx.92.101 6379
# Generated by CONFIG REWRITE
pidfile "/var/run/redis.pid"
save 3600 1
save 300 100
save 60 10000
user default on #8d969eef6ecad3c29a3a629280e686cf0c3f5d5a86aff3ca12020c923adc6c92 ~* &* +@all

2.9. 读写验证

# 登陆redis客户端
/usr/local/redis/bin/redis-cli --user default --pass 123456

#故障转移后的100机器现在为slave节点，可读不可写
[root@bigdata01 redis-6.2.0]# /usr/local/redis/bin/redis-cli --user default --pass 123456
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
127.0.0.1:6379> set name lisi
(error) READONLY You can't write against a read only replica.
127.0.0.1:6379>

#故障转移后的101机器现在为master节点，可以读写
[root@bigdata02 ~]# /usr/local/redis/bin/redis-cli --user default --pass 123456
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
127.0.0.1:6379> set name lisi
OK
127.0.0.1:6379> get name
"lisi"
127.0.0.1:6379> 

#故障转移后的102机器现在为slave节点，可读不可写
[root@bigdata03 ~]# /usr/local/redis/bin/redis-cli --user default --pass 123456
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
127.0.0.1:6379> set name wangwu 
(error) READONLY You can't write against a read only replica.
127.0.0.1:6379> get name
"lisi"
127.0.0.1:6379>