Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
ruanhaishen
redis
Commits
3a56013a
Commit
3a56013a
authored
Nov 18, 2013
by
antirez
Browse files
Sentinel: state machine and timeouts simplified.
parent
4be53b1c
Changes
1
Hide whitespace changes
Inline
Side-by-side
src/sentinel.c
View file @
3a56013a
...
@@ -85,7 +85,7 @@ typedef struct sentinelAddr {
...
@@ -85,7 +85,7 @@ typedef struct sentinelAddr {
#define SENTINEL_SLAVE_RECONF_RETRY_PERIOD 10000
#define SENTINEL_SLAVE_RECONF_RETRY_PERIOD 10000
#define SENTINEL_DEFAULT_PARALLEL_SYNCS 1
#define SENTINEL_DEFAULT_PARALLEL_SYNCS 1
#define SENTINEL_MIN_LINK_RECONNECT_PERIOD 15000
#define SENTINEL_MIN_LINK_RECONNECT_PERIOD 15000
#define SENTINEL_DEFAULT_FAILOVER_TIMEOUT (60*
1
5*1000)
#define SENTINEL_DEFAULT_FAILOVER_TIMEOUT (60*5*1000)
#define SENTINEL_MAX_PENDING_COMMANDS 100
#define SENTINEL_MAX_PENDING_COMMANDS 100
#define SENTINEL_ELECTION_TIMEOUT 10000
#define SENTINEL_ELECTION_TIMEOUT 10000
...
@@ -105,8 +105,7 @@ typedef struct sentinelAddr {
...
@@ -105,8 +105,7 @@ typedef struct sentinelAddr {
#define SENTINEL_FAILOVER_STATE_WAIT_NEXT_SLAVE 6
/* wait replication */
#define SENTINEL_FAILOVER_STATE_WAIT_NEXT_SLAVE 6
/* wait replication */
#define SENTINEL_FAILOVER_STATE_ALERT_CLIENTS 7
/* Run user script. */
#define SENTINEL_FAILOVER_STATE_ALERT_CLIENTS 7
/* Run user script. */
#define SENTINEL_FAILOVER_STATE_WAIT_ALERT_SCRIPT 8
/* Wait script exec. */
#define SENTINEL_FAILOVER_STATE_WAIT_ALERT_SCRIPT 8
/* Wait script exec. */
#define SENTINEL_FAILOVER_STATE_DETECT_END 9
/* Check for failover end. */
#define SENTINEL_FAILOVER_STATE_UPDATE_CONFIG 9
/* Monitor promoted slave. */
#define SENTINEL_FAILOVER_STATE_UPDATE_CONFIG 10
/* Monitor promoted slave. */
#define SENTINEL_MASTER_LINK_STATUS_UP 0
#define SENTINEL_MASTER_LINK_STATUS_UP 0
#define SENTINEL_MASTER_LINK_STATUS_DOWN 1
#define SENTINEL_MASTER_LINK_STATUS_DOWN 1
...
@@ -1693,10 +1692,6 @@ void sentinelRefreshInstanceInfo(sentinelRedisInstance *ri, const char *info) {
...
@@ -1693,10 +1692,6 @@ void sentinelRefreshInstanceInfo(sentinelRedisInstance *ri, const char *info) {
ri
->
flags
&=
~
SRI_RECONF_INPROG
;
ri
->
flags
&=
~
SRI_RECONF_INPROG
;
ri
->
flags
|=
SRI_RECONF_DONE
;
ri
->
flags
|=
SRI_RECONF_DONE
;
sentinelEvent
(
REDIS_NOTICE
,
"+slave-reconf-done"
,
ri
,
"%@"
);
sentinelEvent
(
REDIS_NOTICE
,
"+slave-reconf-done"
,
ri
,
"%@"
);
/* If we are moving forward (a new slave is now configured)
* we update the change_time as we are conceptually passing
* to the next slave. */
ri
->
failover_state_change_time
=
mstime
();
}
}
}
}
}
}
...
@@ -1968,7 +1963,6 @@ const char *sentinelFailoverStateStr(int state) {
...
@@ -1968,7 +1963,6 @@ const char *sentinelFailoverStateStr(int state) {
case
SENTINEL_FAILOVER_STATE_WAIT_PROMOTION
:
return
"wait_promotion"
;
case
SENTINEL_FAILOVER_STATE_WAIT_PROMOTION
:
return
"wait_promotion"
;
case
SENTINEL_FAILOVER_STATE_RECONF_SLAVES
:
return
"reconf_slaves"
;
case
SENTINEL_FAILOVER_STATE_RECONF_SLAVES
:
return
"reconf_slaves"
;
case
SENTINEL_FAILOVER_STATE_ALERT_CLIENTS
:
return
"alert_clients"
;
case
SENTINEL_FAILOVER_STATE_ALERT_CLIENTS
:
return
"alert_clients"
;
case
SENTINEL_FAILOVER_STATE_DETECT_END
:
return
"detect_end"
;
case
SENTINEL_FAILOVER_STATE_UPDATE_CONFIG
:
return
"update_config"
;
case
SENTINEL_FAILOVER_STATE_UPDATE_CONFIG
:
return
"update_config"
;
default:
return
"unknown"
;
default:
return
"unknown"
;
}
}
...
@@ -2816,17 +2810,20 @@ void sentinelFailoverWaitStart(sentinelRedisInstance *ri) {
...
@@ -2816,17 +2810,20 @@ void sentinelFailoverWaitStart(sentinelRedisInstance *ri) {
/* If I'm not the leader, I can't continue with the failover. */
/* If I'm not the leader, I can't continue with the failover. */
if
(
!
isleader
)
{
if
(
!
isleader
)
{
int
election_timeout
=
SENTINEL_ELECTION_TIMEOUT
;
/* The election timeout is the MIN between SENTINEL_ELECTION_TIMEOUT
* and the configured failover timeout. */
if
(
election_timeout
>
ri
->
failover_timeout
)
election_timeout
=
ri
->
failover_timeout
;
/* Abort the failover if I'm not the leader after some time. */
/* Abort the failover if I'm not the leader after some time. */
if
(
mstime
()
-
ri
->
failover_start_time
>
SENTINEL_ELECTION_TIMEOUT
)
{
if
(
mstime
()
-
ri
->
failover_start_time
>
election_timeout
)
{
sentinelEvent
(
REDIS_WARNING
,
"-failover-abort-not-elected"
,
ri
,
"%@"
);
sentinelEvent
(
REDIS_WARNING
,
"-failover-abort-not-elected"
,
ri
,
"%@"
);
sentinelAbortFailover
(
ri
);
sentinelAbortFailover
(
ri
);
}
}
return
;
return
;
}
}
sentinelEvent
(
REDIS_WARNING
,
"+elected-leader"
,
ri
,
"%@"
);
sentinelEvent
(
REDIS_WARNING
,
"+elected-leader"
,
ri
,
"%@"
);
/* Start the failover going to the next state if enough time has
* elapsed. */
ri
->
failover_state
=
SENTINEL_FAILOVER_STATE_SELECT_SLAVE
;
ri
->
failover_state
=
SENTINEL_FAILOVER_STATE_SELECT_SLAVE
;
ri
->
failover_state_change_time
=
mstime
();
ri
->
failover_state_change_time
=
mstime
();
sentinelEvent
(
REDIS_WARNING
,
"+failover-state-select-slave"
,
ri
,
"%@"
);
sentinelEvent
(
REDIS_WARNING
,
"+failover-state-select-slave"
,
ri
,
"%@"
);
...
@@ -2835,6 +2832,8 @@ void sentinelFailoverWaitStart(sentinelRedisInstance *ri) {
...
@@ -2835,6 +2832,8 @@ void sentinelFailoverWaitStart(sentinelRedisInstance *ri) {
void
sentinelFailoverSelectSlave
(
sentinelRedisInstance
*
ri
)
{
void
sentinelFailoverSelectSlave
(
sentinelRedisInstance
*
ri
)
{
sentinelRedisInstance
*
slave
=
sentinelSelectSlave
(
ri
);
sentinelRedisInstance
*
slave
=
sentinelSelectSlave
(
ri
);
/* We don't handle the timeout in this state as the function aborts
* the failover or go forward in the next state. */
if
(
slave
==
NULL
)
{
if
(
slave
==
NULL
)
{
sentinelEvent
(
REDIS_WARNING
,
"-failover-abort-no-good-slave"
,
ri
,
"%@"
);
sentinelEvent
(
REDIS_WARNING
,
"-failover-abort-no-good-slave"
,
ri
,
"%@"
);
sentinelAbortFailover
(
ri
);
sentinelAbortFailover
(
ri
);
...
@@ -2852,7 +2851,16 @@ void sentinelFailoverSelectSlave(sentinelRedisInstance *ri) {
...
@@ -2852,7 +2851,16 @@ void sentinelFailoverSelectSlave(sentinelRedisInstance *ri) {
void
sentinelFailoverSendSlaveOfNoOne
(
sentinelRedisInstance
*
ri
)
{
void
sentinelFailoverSendSlaveOfNoOne
(
sentinelRedisInstance
*
ri
)
{
int
retval
;
int
retval
;
if
(
ri
->
promoted_slave
->
flags
&
SRI_DISCONNECTED
)
return
;
/* We can't send the command to the promoted slave if it is now
* disconnected. Retry again and again with this state until the timeout
* is reached, then abort the failover. */
if
(
ri
->
promoted_slave
->
flags
&
SRI_DISCONNECTED
)
{
if
(
mstime
()
-
ri
->
failover_state_change_time
>
ri
->
failover_timeout
)
{
sentinelEvent
(
REDIS_WARNING
,
"-failover-abort-slave-timeout"
,
ri
,
"%@"
);
sentinelAbortFailover
(
ri
);
}
return
;
}
/* Send SLAVEOF NO ONE command to turn the slave into a master.
/* Send SLAVEOF NO ONE command to turn the slave into a master.
* We actually register a generic callback for this command as we don't
* We actually register a generic callback for this command as we don't
...
@@ -2869,16 +2877,11 @@ void sentinelFailoverSendSlaveOfNoOne(sentinelRedisInstance *ri) {
...
@@ -2869,16 +2877,11 @@ void sentinelFailoverSendSlaveOfNoOne(sentinelRedisInstance *ri) {
/* We actually wait for promotion indirectly checking with INFO when the
/* We actually wait for promotion indirectly checking with INFO when the
* slave turns into a master. */
* slave turns into a master. */
void
sentinelFailoverWaitPromotion
(
sentinelRedisInstance
*
ri
)
{
void
sentinelFailoverWaitPromotion
(
sentinelRedisInstance
*
ri
)
{
mstime_t
elapsed
=
mstime
()
-
ri
->
failover_state_change_time
;
/* Just handle the timeout. Switching to the next state is handled
* by the function parsing the INFO command of the promoted slave. */
if
(
elapsed
>=
SENTINEL_PROMOTION_RETRY_PERIOD
)
{
if
(
mstime
()
-
ri
->
failover_state_change_time
>
ri
->
failover_timeout
)
{
sentinelEvent
(
REDIS_WARNING
,
"-promotion-timeout"
,
ri
->
promoted_slave
,
sentinelEvent
(
REDIS_WARNING
,
"-failover-abort-slave-timeout"
,
ri
,
"%@"
);
"%@"
);
sentinelAbortFailover
(
ri
);
sentinelEvent
(
REDIS_WARNING
,
"+failover-state-select-slave"
,
ri
,
"%@"
);
ri
->
failover_state
=
SENTINEL_FAILOVER_STATE_SELECT_SLAVE
;
ri
->
failover_state_change_time
=
mstime
();
ri
->
promoted_slave
->
flags
&=
~
SRI_PROMOTED
;
ri
->
promoted_slave
=
NULL
;
}
}
}
}
...
@@ -3002,6 +3005,8 @@ void sentinelFailoverReconfNextSlave(sentinelRedisInstance *master) {
...
@@ -3002,6 +3005,8 @@ void sentinelFailoverReconfNextSlave(sentinelRedisInstance *master) {
}
}
}
}
dictReleaseIterator
(
di
);
dictReleaseIterator
(
di
);
/* Check if all the slaves are reconfigured and handle timeout. */
sentinelFailoverDetectEnd
(
master
);
sentinelFailoverDetectEnd
(
master
);
}
}
...
@@ -3049,50 +3054,46 @@ void sentinelFailoverStateMachine(sentinelRedisInstance *ri) {
...
@@ -3049,50 +3054,46 @@ void sentinelFailoverStateMachine(sentinelRedisInstance *ri) {
case
SENTINEL_FAILOVER_STATE_RECONF_SLAVES
:
case
SENTINEL_FAILOVER_STATE_RECONF_SLAVES
:
sentinelFailoverReconfNextSlave
(
ri
);
sentinelFailoverReconfNextSlave
(
ri
);
break
;
break
;
case
SENTINEL_FAILOVER_STATE_DETECT_END
:
sentinelFailoverDetectEnd
(
ri
);
break
;
}
}
}
}
/* Abort a failover in progress with the following steps:
/* Abort a failover in progress:
* 1) Set the master back to the original one, increment the config epoch.
*
* 2) Reconfig slaves to replicate to the old master.
* This function can only be called before the promoted slave acknowledged
* 3) Reconfig the promoted slave as a slave as well. */
* the slave -> master switch. Otherwise the failover can't be aborted and
* will reach its end.
*
* If there is a promoted slave and we already got acknowledge of the
* slave -> master switch, we clear our flags and redirect to the
* new master. Eventually the config will be propagated if it is the one
* with the greater config epoch for this master.
*
* Otherwise if we still did not received the acknowledgement from the
* promoted slave, or there is no promoted slave at all, we just clear the
* failover-in-progress state as there is nothing to do (if the promoted
* slave for some reason actually received our "SLAVEOF NO ONE" command
* even if we did not received the ACK, it will be reverted to slave again
* by one of the Sentinels). */
void
sentinelAbortFailover
(
sentinelRedisInstance
*
ri
)
{
void
sentinelAbortFailover
(
sentinelRedisInstance
*
ri
)
{
dictIterator
*
di
;
dictIterator
*
di
;
dictEntry
*
de
;
dictEntry
*
de
;
int
sentinel_role
;
redisAssert
(
ri
->
flags
&
SRI_FAILOVER_IN_PROGRESS
);
redisAssert
(
ri
->
flags
&
SRI_FAILOVER_IN_PROGRESS
);
redisAssert
(
ri
->
failover_state
<=
SENTINEL_FAILOVER_STATE_WAIT_PROMOTION
);
/* Clear failover related flags from slaves.
/* Clear failover related flags from slaves. */
* Also if we are the leader make sure to send SLAVEOF commands to all the
* already reconfigured slaves in order to turn them back into slaves of
* the original master. */
di
=
dictGetIterator
(
ri
->
slaves
);
di
=
dictGetIterator
(
ri
->
slaves
);
while
((
de
=
dictNext
(
di
))
!=
NULL
)
{
while
((
de
=
dictNext
(
di
))
!=
NULL
)
{
sentinelRedisInstance
*
slave
=
dictGetVal
(
de
);
sentinelRedisInstance
*
slave
=
dictGetVal
(
de
);
if
(
!
(
slave
->
flags
&
SRI_DISCONNECTED
)
&&
(
slave
->
flags
&
(
SRI_PROMOTED
|
SRI_RECONF_SENT
|
SRI_RECONF_INPROG
|
SRI_RECONF_DONE
)))
{
int
retval
;
retval
=
sentinelSendSlaveOf
(
slave
,
ri
->
addr
->
ip
,
ri
->
addr
->
port
);
if
(
retval
==
REDIS_OK
)
sentinelEvent
(
REDIS_NOTICE
,
"-slave-reconf-undo"
,
slave
,
"%@"
);
}
slave
->
flags
&=
~
(
SRI_RECONF_SENT
|
SRI_RECONF_INPROG
|
SRI_RECONF_DONE
);
slave
->
flags
&=
~
(
SRI_RECONF_SENT
|
SRI_RECONF_INPROG
|
SRI_RECONF_DONE
);
}
}
dictReleaseIterator
(
di
);
dictReleaseIterator
(
di
);
sentinel_role
=
SENTINEL_LEADER
;
ri
->
flags
&=
~
(
SRI_FAILOVER_IN_PROGRESS
|
SRI_FORCE_FAILOVER
);
ri
->
flags
&=
~
(
SRI_FAILOVER_IN_PROGRESS
|
SRI_FORCE_FAILOVER
);
ri
->
failover_state
=
SENTINEL_FAILOVER_STATE_NONE
;
ri
->
failover_state
=
SENTINEL_FAILOVER_STATE_NONE
;
ri
->
failover_state_change_time
=
mstime
();
ri
->
failover_state_change_time
=
mstime
();
if
(
ri
->
promoted_slave
)
{
if
(
ri
->
promoted_slave
)
{
sentinelCallClientReconfScript
(
ri
,
sentinel_role
,
"abort"
,
sentinelCallClientReconfScript
(
ri
,
SENTINEL_LEADER
,
"abort"
,
ri
->
promoted_slave
->
addr
,
ri
->
addr
);
ri
->
promoted_slave
->
addr
,
ri
->
addr
);
ri
->
promoted_slave
->
flags
&=
~
SRI_PROMOTED
;
ri
->
promoted_slave
->
flags
&=
~
SRI_PROMOTED
;
ri
->
promoted_slave
=
NULL
;
ri
->
promoted_slave
=
NULL
;
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment