Commit 51e049fd authored by Li Wei's avatar Li Wei Committed by Willem Thiart
Browse files

DAOS-279 server: Zero timeout_elapsed for leaders



If raft_become_candidate() assigns a negative value e (i.e.,
-election_timeout < e < 0) to timeout_elapsed, and if this replica
successfully becomes the leader, then this leader won't send heartbeats
for request_timeout - e milliseconds. This may be longer than
election_timeout and cause an unnecessary leadership change.

This patch zeroes timeout_elapsed in raft_become_leader(), randomizes
election timeouts in [election_timeout, 2 * election_timeout), and fixes
raft_recv_appendentries() to avoid zeroing timeout_elapsed if the AE
request has an older term.
Signed-off-by: default avatarLi Wei <wei.g.li@intel.com>
parent 14a4e6a2
......@@ -49,6 +49,7 @@ typedef struct {
int num_nodes;
int election_timeout;
int election_timeout_rand;
int request_timeout;
/* what this node thinks is the node ID of the current leader, or NULL if
......@@ -80,6 +81,8 @@ void raft_vote(raft_server_t* me, raft_node_t* node);
void raft_set_current_term(raft_server_t* me,int term);
void raft_randomize_election_timeout(raft_server_t* me_);
/**
* @return 0 on error */
int raft_send_requestvote(raft_server_t* me, raft_node_t* node);
......
......@@ -41,6 +41,15 @@ static void __log(raft_server_t *me_, raft_node_t* node, const char *fmt, ...)
me->cb.log(me_, node, me->udata, buf);
}
void raft_randomize_election_timeout(raft_server_t* me_)
{
raft_server_private_t* me = (raft_server_private_t*)me_;
/* [election_timeout, 2 * election_timeout) */
me->election_timeout_rand = me->election_timeout + rand() % me->election_timeout;
__log(me_, NULL, "randomize election timeout to %d", me->election_timeout_rand);
}
raft_server_t* raft_new()
{
raft_server_private_t* me =
......@@ -52,6 +61,7 @@ raft_server_t* raft_new()
me->timeout_elapsed = 0;
me->request_timeout = 200;
me->election_timeout = 1000;
raft_randomize_election_timeout((raft_server_t*)me);
me->log = log_new();
if (!me->log) {
free(me);
......@@ -87,6 +97,7 @@ void raft_clear(raft_server_t* me_)
me->current_term = 0;
me->voted_for = -1;
me->timeout_elapsed = 0;
raft_randomize_election_timeout(me_);
me->voting_cfg_change_log_idx = -1;
raft_set_state((raft_server_t*)me, RAFT_STATE_FOLLOWER);
me->current_leader = NULL;
......@@ -115,7 +126,7 @@ void raft_election_start(raft_server_t* me_)
raft_server_private_t* me = (raft_server_private_t*)me_;
__log(me_, NULL, "election starting: %d %d, term: %d ci: %d",
me->election_timeout, me->timeout_elapsed, me->current_term,
me->election_timeout_rand, me->timeout_elapsed, me->current_term,
raft_get_current_idx(me_));
raft_become_candidate(me_);
......@@ -129,6 +140,7 @@ void raft_become_leader(raft_server_t* me_)
__log(me_, NULL, "becoming leader term:%d", raft_get_current_term(me_));
raft_set_state(me_, RAFT_STATE_LEADER);
me->timeout_elapsed = 0;
for (i = 0; i < me->num_nodes; i++)
{
if (me->node == me->nodes[i])
......@@ -155,11 +167,8 @@ void raft_become_candidate(raft_server_t* me_)
me->current_leader = NULL;
raft_set_state(me_, RAFT_STATE_CANDIDATE);
/* We need a random factor here to prevent simultaneous candidates.
* If the randomness is always positive it's possible that a fast node
* would deadlock the cluster by always gaining a headstart. To prevent
* this, we allow a negative randomness as a potential handicap. */
me->timeout_elapsed = me->election_timeout - 2 * (rand() % me->election_timeout);
raft_randomize_election_timeout(me_);
me->timeout_elapsed = 0;
for (i = 0; i < me->num_nodes; i++)
if (me->node != me->nodes[i] && raft_node_is_voting(me->nodes[i]))
......@@ -168,8 +177,12 @@ void raft_become_candidate(raft_server_t* me_)
void raft_become_follower(raft_server_t* me_)
{
raft_server_private_t* me = (raft_server_private_t*)me_;
__log(me_, NULL, "becoming follower");
raft_set_state(me_, RAFT_STATE_FOLLOWER);
raft_randomize_election_timeout(me_);
me->timeout_elapsed = 0;
}
int raft_periodic(raft_server_t* me_, int msec_since_last_period)
......@@ -189,7 +202,7 @@ int raft_periodic(raft_server_t* me_, int msec_since_last_period)
if (me->request_timeout <= me->timeout_elapsed)
raft_send_appendentries_all(me_);
}
else if (me->election_timeout <= me->timeout_elapsed)
else if (me->election_timeout_rand <= me->timeout_elapsed)
{
if (1 < raft_get_num_voting_nodes(me_) &&
raft_node_is_voting(raft_get_my_node(me_)))
......@@ -323,8 +336,6 @@ int raft_recv_appendentries(
raft_server_private_t* me = (raft_server_private_t*)me_;
int e = 0;
me->timeout_elapsed = 0;
if (0 < ae->n_entries)
__log(me_, node, "recvd appendentries t:%d ci:%d lc:%d pli:%d plt:%d #%d",
ae->term,
......@@ -357,6 +368,8 @@ int raft_recv_appendentries(
/* update current leader because ae->term is up to date */
me->current_leader = node;
me->timeout_elapsed = 0;
/* Not the first appendentries we've received */
/* NOTE: the log starts at 1 */
if (0 < ae->prev_log_idx)
......
......@@ -23,6 +23,7 @@ void raft_set_election_timeout(raft_server_t* me_, int millisec)
{
raft_server_private_t* me = (raft_server_private_t*)me_;
me->election_timeout = millisec;
raft_randomize_election_timeout(me_);
}
void raft_set_request_timeout(raft_server_t* me_, int millisec)
......
......@@ -65,6 +65,11 @@ raft_cbs_t generic_funcs = {
.persist_vote = __raft_persist_vote,
};
static int max_election_timeout(int election_timeout)
{
return 2 * election_timeout;
}
void TestRaft_server_voted_for_records_who_we_voted_for(CuTest * tc)
{
void *r = raft_new();
......@@ -1615,8 +1620,8 @@ void TestRaft_follower_becomes_candidate_when_election_timeout_occurs(
raft_add_node(r, NULL, 1, 1);
raft_add_node(r, NULL, 2, 0);
/* 1.001 seconds have passed */
raft_periodic(r, 1001);
/* max election timeout have passed */
raft_periodic(r, max_election_timeout(1000) + 1);
/* is a candidate now */
CuAssertTrue(tc, 1 == raft_is_candidate(r));
......@@ -1996,8 +2001,8 @@ void TestRaft_candidate_election_timeout_and_no_leader_results_in_new_election(
raft_become_candidate(r);
CuAssertTrue(tc, 1 == raft_get_current_term(r));
/* clock over (ie. 1000 + 1), causing new election */
raft_periodic(r, 1001);
/* clock over (ie. max election timeout + 1), causing new election */
raft_periodic(r, max_election_timeout(1000) + 1);
CuAssertTrue(tc, 2 == raft_get_current_term(r));
/* receiving this vote gives the server majority */
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment