Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
ruanhaishen
redis
Commits
9a867b68
You need to sign in or sign up before continuing.
Commit
9a867b68
authored
Oct 07, 2014
by
antirez
Browse files
Cluster: more chatty slaves when failover is stalled.
parent
eedfbdec
Changes
2
Hide whitespace changes
Inline
Side-by-side
src/cluster.c
View file @
9a867b68
...
...
@@ -417,6 +417,7 @@ void clusterInit(void) {
server
.
cluster
->
failover_auth_count
=
0
;
server
.
cluster
->
failover_auth_rank
=
0
;
server
.
cluster
->
failover_auth_epoch
=
0
;
server
.
cluster
->
cant_failover_reason
=
REDIS_CLUSTER_CANT_FAILOVER_NONE
;
server
.
cluster
->
lastVoteEpoch
=
0
;
server
.
cluster
->
stats_bus_messages_sent
=
0
;
server
.
cluster
->
stats_bus_messages_received
=
0
;
...
...
@@ -2430,6 +2431,68 @@ int clusterGetSlaveRank(void) {
return
rank
;
}
/* This function is called by clusterHandleSlaveFailover() in order to
* let the slave log why it is not able to failover. Sometimes there are
* not the conditions, but since the failover function is called again and
* again, we can't log the same things continuously.
*
* This function works by logging only if a given set of conditions are
* true:
*
* 1) The reason for which the failover can't be initiated changed.
* The reasons also include a NONE reason we reset the state to
* when the slave finds that its master is fine (no FAIL flag).
* 2) Also, the log is emitted again if the master is still down and
* the reason for not failing over is still the same, but more than
* REDIS_CLUSTER_CANT_FAILOVER_RELOG_PERIOD seconds elapsed.
* 3) Finally, the function only logs if the slave is down for more than
* five seconds + NODE_TIMEOUT. This way nothing is logged when a
* failover starts in a reasonable time.
*
* The function is called with the reason why the slave can't failover
* which is one of the integer macros REDIS_CLUSTER_CANT_FAILOVER_*.
*
* The function is guaranteed to be called only if 'myself' is a slave. */
void
clusterLogCantFailover
(
int
reason
)
{
char
*
msg
;
static
time_t
lastlog_time
=
0
;
mstime_t
nolog_fail_time
=
server
.
cluster_node_timeout
+
5000
;
/* Don't log if we have the same reason for some time. */
if
(
reason
==
server
.
cluster
->
cant_failover_reason
&&
time
(
NULL
)
-
lastlog_time
<
REDIS_CLUSTER_CANT_FAILOVER_RELOG_PERIOD
)
return
;
server
.
cluster
->
cant_failover_reason
=
reason
;
/* We also don't emit any log if the master failed no long ago, the
* goal of this function is to log slaves in a stalled condition for
* a long time. */
if
(
myself
->
slaveof
&&
nodeFailed
(
myself
->
slaveof
)
&&
(
mstime
()
-
myself
->
slaveof
->
fail_time
)
<
nolog_fail_time
)
return
;
switch
(
reason
)
{
case
REDIS_CLUSTER_CANT_FAILOVER_DATA_AGE
:
msg
=
"Disconnected from master for longer than allowed."
;
break
;
case
REDIS_CLUSTER_CANT_FAILOVER_WAITING_DELAY
:
msg
=
"Waiting the delay before I can start a new failover."
;
break
;
case
REDIS_CLUSTER_CANT_FAILOVER_EXPIRED
:
msg
=
"Failover attempt expired."
;
break
;
case
REDIS_CLUSTER_CANT_FAILOVER_WAITING_VOTES
:
msg
=
"Waiting for votes, but majority still not reached."
;
break
;
default:
msg
=
"Unknown reason code."
;
break
;
}
lastlog_time
=
time
(
NULL
);
redisLog
(
REDIS_WARNING
,
"Currently unable to failover: %s"
,
msg
);
}
/* This function is called if we are a slave node and our master serving
* a non-zero amount of hash slots is in FAIL state.
*
...
...
@@ -2468,7 +2531,13 @@ void clusterHandleSlaveFailover(void) {
if
(
nodeIsMaster
(
myself
)
||
myself
->
slaveof
==
NULL
||
(
!
nodeFailed
(
myself
->
slaveof
)
&&
!
manual_failover
)
||
myself
->
slaveof
->
numslots
==
0
)
return
;
myself
->
slaveof
->
numslots
==
0
)
{
/* There are no reasons to failover, so we set the reason why we
* are returning without failing over to NONE. */
server
.
cluster
->
cant_failover_reason
=
REDIS_CLUSTER_CANT_FAILOVER_NONE
;
return
;
}
/* Set data_age to the number of seconds we are disconnected from
* the master. */
...
...
@@ -2494,7 +2563,10 @@ void clusterHandleSlaveFailover(void) {
(((
mstime_t
)
server
.
repl_ping_slave_period
*
1000
)
+
(
server
.
cluster_node_timeout
*
server
.
cluster_slave_validity_factor
)))
{
if
(
!
manual_failover
)
return
;
if
(
!
manual_failover
)
{
clusterLogCantFailover
(
REDIS_CLUSTER_CANT_FAILOVER_DATA_AGE
);
return
;
}
}
/* If the previous failover attempt timedout and the retry time has
...
...
@@ -2550,10 +2622,16 @@ void clusterHandleSlaveFailover(void) {
}
/* Return ASAP if we can't still start the election. */
if
(
mstime
()
<
server
.
cluster
->
failover_auth_time
)
return
;
if
(
mstime
()
<
server
.
cluster
->
failover_auth_time
)
{
clusterLogCantFailover
(
REDIS_CLUSTER_CANT_FAILOVER_WAITING_DELAY
);
return
;
}
/* Return ASAP if the election is too old to be valid. */
if
(
auth_age
>
auth_timeout
)
return
;
if
(
auth_age
>
auth_timeout
)
{
clusterLogCantFailover
(
REDIS_CLUSTER_CANT_FAILOVER_EXPIRED
);
return
;
}
/* Ask for votes if needed. */
if
(
server
.
cluster
->
failover_auth_sent
==
0
)
{
...
...
@@ -2608,6 +2686,8 @@ void clusterHandleSlaveFailover(void) {
/* 6) If there was a manual failover in progress, clear the state. */
resetManualFailover
();
}
else
{
clusterLogCantFailover
(
REDIS_CLUSTER_CANT_FAILOVER_WAITING_VOTES
);
}
}
...
...
src/cluster.h
View file @
9a867b68
...
...
@@ -62,6 +62,14 @@ typedef struct clusterLink {
#define nodeTimedOut(n) ((n)->flags & REDIS_NODE_PFAIL)
#define nodeFailed(n) ((n)->flags & REDIS_NODE_FAIL)
/* Reasons why a slave is not able to failover. */
#define REDIS_CLUSTER_CANT_FAILOVER_NONE 0
#define REDIS_CLUSTER_CANT_FAILOVER_DATA_AGE 1
#define REDIS_CLUSTER_CANT_FAILOVER_WAITING_DELAY 2
#define REDIS_CLUSTER_CANT_FAILOVER_EXPIRED 3
#define REDIS_CLUSTER_CANT_FAILOVER_WAITING_VOTES 4
#define REDIS_CLUSTER_CANT_FAILOVER_RELOG_PERIOD (60*5)
/* seconds. */
/* This structure represent elements of node->fail_reports. */
typedef
struct
clusterNodeFailReport
{
struct
clusterNode
*
node
;
/* Node reporting the failure condition. */
...
...
@@ -107,6 +115,8 @@ typedef struct clusterState {
int
failover_auth_sent
;
/* True if we already asked for votes. */
int
failover_auth_rank
;
/* This slave rank for current auth request. */
uint64_t
failover_auth_epoch
;
/* Epoch of the current election. */
int
cant_failover_reason
;
/* Why a slave is currently not able to
failover. See the CANT_FAILOVER_* macros. */
/* Manual failover state in common. */
mstime_t
mf_end
;
/* Manual failover time limit (ms unixtime).
It is zero if there is no MF in progress. */
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment