Commit 51e049fd authored by Li Wei's avatar Li Wei Committed by Willem Thiart
Browse files

DAOS-279 server: Zero timeout_elapsed for leaders



If raft_become_candidate() assigns a negative value e (i.e.,
-election_timeout < e < 0) to timeout_elapsed, and if this replica
successfully becomes the leader, then this leader won't send heartbeats
for request_timeout - e milliseconds. This may be longer than
election_timeout and cause an unnecessary leadership change.

This patch zeroes timeout_elapsed in raft_become_leader(), randomizes
election timeouts in [election_timeout, 2 * election_timeout), and fixes
raft_recv_appendentries() to avoid zeroing timeout_elapsed if the AE
request has an older term.
Signed-off-by: default avatarLi Wei <wei.g.li@intel.com>
parent 14a4e6a2
...@@ -49,6 +49,7 @@ typedef struct { ...@@ -49,6 +49,7 @@ typedef struct {
int num_nodes; int num_nodes;
int election_timeout; int election_timeout;
int election_timeout_rand;
int request_timeout; int request_timeout;
/* what this node thinks is the node ID of the current leader, or NULL if /* what this node thinks is the node ID of the current leader, or NULL if
...@@ -80,6 +81,8 @@ void raft_vote(raft_server_t* me, raft_node_t* node); ...@@ -80,6 +81,8 @@ void raft_vote(raft_server_t* me, raft_node_t* node);
void raft_set_current_term(raft_server_t* me,int term); void raft_set_current_term(raft_server_t* me,int term);
void raft_randomize_election_timeout(raft_server_t* me_);
/** /**
* @return 0 on error */ * @return 0 on error */
int raft_send_requestvote(raft_server_t* me, raft_node_t* node); int raft_send_requestvote(raft_server_t* me, raft_node_t* node);
......
...@@ -41,6 +41,15 @@ static void __log(raft_server_t *me_, raft_node_t* node, const char *fmt, ...) ...@@ -41,6 +41,15 @@ static void __log(raft_server_t *me_, raft_node_t* node, const char *fmt, ...)
me->cb.log(me_, node, me->udata, buf); me->cb.log(me_, node, me->udata, buf);
} }
void raft_randomize_election_timeout(raft_server_t* me_)
{
raft_server_private_t* me = (raft_server_private_t*)me_;
/* [election_timeout, 2 * election_timeout) */
me->election_timeout_rand = me->election_timeout + rand() % me->election_timeout;
__log(me_, NULL, "randomize election timeout to %d", me->election_timeout_rand);
}
raft_server_t* raft_new() raft_server_t* raft_new()
{ {
raft_server_private_t* me = raft_server_private_t* me =
...@@ -52,6 +61,7 @@ raft_server_t* raft_new() ...@@ -52,6 +61,7 @@ raft_server_t* raft_new()
me->timeout_elapsed = 0; me->timeout_elapsed = 0;
me->request_timeout = 200; me->request_timeout = 200;
me->election_timeout = 1000; me->election_timeout = 1000;
raft_randomize_election_timeout((raft_server_t*)me);
me->log = log_new(); me->log = log_new();
if (!me->log) { if (!me->log) {
free(me); free(me);
...@@ -87,6 +97,7 @@ void raft_clear(raft_server_t* me_) ...@@ -87,6 +97,7 @@ void raft_clear(raft_server_t* me_)
me->current_term = 0; me->current_term = 0;
me->voted_for = -1; me->voted_for = -1;
me->timeout_elapsed = 0; me->timeout_elapsed = 0;
raft_randomize_election_timeout(me_);
me->voting_cfg_change_log_idx = -1; me->voting_cfg_change_log_idx = -1;
raft_set_state((raft_server_t*)me, RAFT_STATE_FOLLOWER); raft_set_state((raft_server_t*)me, RAFT_STATE_FOLLOWER);
me->current_leader = NULL; me->current_leader = NULL;
...@@ -115,7 +126,7 @@ void raft_election_start(raft_server_t* me_) ...@@ -115,7 +126,7 @@ void raft_election_start(raft_server_t* me_)
raft_server_private_t* me = (raft_server_private_t*)me_; raft_server_private_t* me = (raft_server_private_t*)me_;
__log(me_, NULL, "election starting: %d %d, term: %d ci: %d", __log(me_, NULL, "election starting: %d %d, term: %d ci: %d",
me->election_timeout, me->timeout_elapsed, me->current_term, me->election_timeout_rand, me->timeout_elapsed, me->current_term,
raft_get_current_idx(me_)); raft_get_current_idx(me_));
raft_become_candidate(me_); raft_become_candidate(me_);
...@@ -129,6 +140,7 @@ void raft_become_leader(raft_server_t* me_) ...@@ -129,6 +140,7 @@ void raft_become_leader(raft_server_t* me_)
__log(me_, NULL, "becoming leader term:%d", raft_get_current_term(me_)); __log(me_, NULL, "becoming leader term:%d", raft_get_current_term(me_));
raft_set_state(me_, RAFT_STATE_LEADER); raft_set_state(me_, RAFT_STATE_LEADER);
me->timeout_elapsed = 0;
for (i = 0; i < me->num_nodes; i++) for (i = 0; i < me->num_nodes; i++)
{ {
if (me->node == me->nodes[i]) if (me->node == me->nodes[i])
...@@ -155,11 +167,8 @@ void raft_become_candidate(raft_server_t* me_) ...@@ -155,11 +167,8 @@ void raft_become_candidate(raft_server_t* me_)
me->current_leader = NULL; me->current_leader = NULL;
raft_set_state(me_, RAFT_STATE_CANDIDATE); raft_set_state(me_, RAFT_STATE_CANDIDATE);
/* We need a random factor here to prevent simultaneous candidates. raft_randomize_election_timeout(me_);
* If the randomness is always positive it's possible that a fast node me->timeout_elapsed = 0;
* would deadlock the cluster by always gaining a headstart. To prevent
* this, we allow a negative randomness as a potential handicap. */
me->timeout_elapsed = me->election_timeout - 2 * (rand() % me->election_timeout);
for (i = 0; i < me->num_nodes; i++) for (i = 0; i < me->num_nodes; i++)
if (me->node != me->nodes[i] && raft_node_is_voting(me->nodes[i])) if (me->node != me->nodes[i] && raft_node_is_voting(me->nodes[i]))
...@@ -168,8 +177,12 @@ void raft_become_candidate(raft_server_t* me_) ...@@ -168,8 +177,12 @@ void raft_become_candidate(raft_server_t* me_)
void raft_become_follower(raft_server_t* me_) void raft_become_follower(raft_server_t* me_)
{ {
raft_server_private_t* me = (raft_server_private_t*)me_;
__log(me_, NULL, "becoming follower"); __log(me_, NULL, "becoming follower");
raft_set_state(me_, RAFT_STATE_FOLLOWER); raft_set_state(me_, RAFT_STATE_FOLLOWER);
raft_randomize_election_timeout(me_);
me->timeout_elapsed = 0;
} }
int raft_periodic(raft_server_t* me_, int msec_since_last_period) int raft_periodic(raft_server_t* me_, int msec_since_last_period)
...@@ -189,7 +202,7 @@ int raft_periodic(raft_server_t* me_, int msec_since_last_period) ...@@ -189,7 +202,7 @@ int raft_periodic(raft_server_t* me_, int msec_since_last_period)
if (me->request_timeout <= me->timeout_elapsed) if (me->request_timeout <= me->timeout_elapsed)
raft_send_appendentries_all(me_); raft_send_appendentries_all(me_);
} }
else if (me->election_timeout <= me->timeout_elapsed) else if (me->election_timeout_rand <= me->timeout_elapsed)
{ {
if (1 < raft_get_num_voting_nodes(me_) && if (1 < raft_get_num_voting_nodes(me_) &&
raft_node_is_voting(raft_get_my_node(me_))) raft_node_is_voting(raft_get_my_node(me_)))
...@@ -323,8 +336,6 @@ int raft_recv_appendentries( ...@@ -323,8 +336,6 @@ int raft_recv_appendentries(
raft_server_private_t* me = (raft_server_private_t*)me_; raft_server_private_t* me = (raft_server_private_t*)me_;
int e = 0; int e = 0;
me->timeout_elapsed = 0;
if (0 < ae->n_entries) if (0 < ae->n_entries)
__log(me_, node, "recvd appendentries t:%d ci:%d lc:%d pli:%d plt:%d #%d", __log(me_, node, "recvd appendentries t:%d ci:%d lc:%d pli:%d plt:%d #%d",
ae->term, ae->term,
...@@ -357,6 +368,8 @@ int raft_recv_appendentries( ...@@ -357,6 +368,8 @@ int raft_recv_appendentries(
/* update current leader because ae->term is up to date */ /* update current leader because ae->term is up to date */
me->current_leader = node; me->current_leader = node;
me->timeout_elapsed = 0;
/* Not the first appendentries we've received */ /* Not the first appendentries we've received */
/* NOTE: the log starts at 1 */ /* NOTE: the log starts at 1 */
if (0 < ae->prev_log_idx) if (0 < ae->prev_log_idx)
......
...@@ -23,6 +23,7 @@ void raft_set_election_timeout(raft_server_t* me_, int millisec) ...@@ -23,6 +23,7 @@ void raft_set_election_timeout(raft_server_t* me_, int millisec)
{ {
raft_server_private_t* me = (raft_server_private_t*)me_; raft_server_private_t* me = (raft_server_private_t*)me_;
me->election_timeout = millisec; me->election_timeout = millisec;
raft_randomize_election_timeout(me_);
} }
void raft_set_request_timeout(raft_server_t* me_, int millisec) void raft_set_request_timeout(raft_server_t* me_, int millisec)
......
...@@ -65,6 +65,11 @@ raft_cbs_t generic_funcs = { ...@@ -65,6 +65,11 @@ raft_cbs_t generic_funcs = {
.persist_vote = __raft_persist_vote, .persist_vote = __raft_persist_vote,
}; };
static int max_election_timeout(int election_timeout)
{
return 2 * election_timeout;
}
void TestRaft_server_voted_for_records_who_we_voted_for(CuTest * tc) void TestRaft_server_voted_for_records_who_we_voted_for(CuTest * tc)
{ {
void *r = raft_new(); void *r = raft_new();
...@@ -1615,8 +1620,8 @@ void TestRaft_follower_becomes_candidate_when_election_timeout_occurs( ...@@ -1615,8 +1620,8 @@ void TestRaft_follower_becomes_candidate_when_election_timeout_occurs(
raft_add_node(r, NULL, 1, 1); raft_add_node(r, NULL, 1, 1);
raft_add_node(r, NULL, 2, 0); raft_add_node(r, NULL, 2, 0);
/* 1.001 seconds have passed */ /* max election timeout have passed */
raft_periodic(r, 1001); raft_periodic(r, max_election_timeout(1000) + 1);
/* is a candidate now */ /* is a candidate now */
CuAssertTrue(tc, 1 == raft_is_candidate(r)); CuAssertTrue(tc, 1 == raft_is_candidate(r));
...@@ -1996,8 +2001,8 @@ void TestRaft_candidate_election_timeout_and_no_leader_results_in_new_election( ...@@ -1996,8 +2001,8 @@ void TestRaft_candidate_election_timeout_and_no_leader_results_in_new_election(
raft_become_candidate(r); raft_become_candidate(r);
CuAssertTrue(tc, 1 == raft_get_current_term(r)); CuAssertTrue(tc, 1 == raft_get_current_term(r));
/* clock over (ie. 1000 + 1), causing new election */ /* clock over (ie. max election timeout + 1), causing new election */
raft_periodic(r, 1001); raft_periodic(r, max_election_timeout(1000) + 1);
CuAssertTrue(tc, 2 == raft_get_current_term(r)); CuAssertTrue(tc, 2 == raft_get_current_term(r));
/* receiving this vote gives the server majority */ /* receiving this vote gives the server majority */
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment