Commit 3c134468 authored by Yossi Gottlieb's avatar Yossi Gottlieb
Browse files

Merge remote-tracking branch 'upstream/master'

parents 876f8d7c 9b983183
...@@ -72,9 +72,7 @@ tests_full: ...@@ -72,9 +72,7 @@ tests_full:
.PHONY: test_virtraft .PHONY: test_virtraft
test_virtraft: test_virtraft:
cp src/*.c virtraft/deps/raft/ python3 tests/virtraft2.py --servers 7 -i 20000 --compaction_rate 50 --drop_rate 5 -P 10 --seed 1 -m 3
cp include/*.h virtraft/deps/raft/
cd virtraft; make clean; make; make tests
.PHONY: amalgamation .PHONY: amalgamation
amalgamation: amalgamation:
......
...@@ -27,12 +27,43 @@ Quality Assurance ...@@ -27,12 +27,43 @@ Quality Assurance
We use the following methods to ensure that the library is safe: We use the following methods to ensure that the library is safe:
* A `simulator <https://github.com/willemt/virtraft>`_ is used to test the Raft invariants on unreliable networks * A simulator (virtraft2) is used to test the Raft invariants on unreliable networks
* `Fuzzing/property-based-testing <https://github.com/willemt/virtraft/blob/master/tests/test_fuzzer.py>`_ via `Hypothesis <https://github.com/DRMacIver/hypothesis/>`_ * `Fuzzing/property-based-testing <https://github.com/willemt/virtraft/blob/master/tests/test_fuzzer.py>`_ via `Hypothesis <https://github.com/DRMacIver/hypothesis/>`_
* All bugs have regression tests * All bugs have regression tests
* Many unit tests * Many unit tests
* `Usage <https://github.com/willemt/ticketd>`_ * `Usage <https://github.com/willemt/ticketd>`_
virtraft2
---------
This cluster simulator checks the following:
* Log Matching (servers must have matching logs)
* State Machine Safety (applied entries have the same ID)
* Election Safety (only one valid leader per term)
* Current Index Validity (does the current index have an existing entry?)
* Entry ID Monotonicity (entries aren't appended out of order)
* Committed entry popping (committed entries are not popped from the log)
* Log Accuracy (does the server's log match mirror an independent log?)
* Deadlock detection (does the cluster continuously make progress?)
Chaos generated by virtraft2:
* Random bi-directional partitions between nodes
* Message dropping
* Message duplication
* Membership change injection
* Random compactions
Run the simulator using:
.. code-block:: bash
:class: ignore
make test_virtraft
virtraft2 succeeds `virtraft <https://github.com/willemt/virtraft>`_
Single file amalgamation Single file amalgamation
======================== ========================
...@@ -440,7 +471,7 @@ The process works like this: ...@@ -440,7 +471,7 @@ The process works like this:
2. Save the current membership details to the snapshot. 2. Save the current membership details to the snapshot.
3. Save the finite state machine to the snapshot. 3. Save the finite state machine to the snapshot.
4. End snapshotting with ``raft_end_snapshot``. 4. End snapshotting with ``raft_end_snapshot``.
5. When the ``send_snapshot`` callback fires, the user must propogate the snapshot to the other node. 5. When the ``send_snapshot`` callback fires, the user must propogate the snapshot to the peer.
6. Once the peer has the snapshot, they call ``raft_begin_load_snapshot``. 6. Once the peer has the snapshot, they call ``raft_begin_load_snapshot``.
7. Peer calls ``raft_add_node`` to add nodes as per the snapshot's membership info. 7. Peer calls ``raft_add_node`` to add nodes as per the snapshot's membership info.
8. Peer calls ``raft_node_set_voting`` to nodes as per the snapshot's membership info. 8. Peer calls ``raft_node_set_voting`` to nodes as per the snapshot's membership info.
......
...@@ -12,14 +12,21 @@ ...@@ -12,14 +12,21 @@
#include "raft_types.h" #include "raft_types.h"
#define RAFT_ERR_NOT_LEADER -2 typedef enum {
#define RAFT_ERR_ONE_VOTING_CHANGE_ONLY -3 RAFT_ERR_NOT_LEADER=-2,
#define RAFT_ERR_SHUTDOWN -4 RAFT_ERR_ONE_VOTING_CHANGE_ONLY=-3,
#define RAFT_ERR_NOMEM -5 RAFT_ERR_SHUTDOWN=-4,
#define RAFT_ERR_NEEDS_SNAPSHOT -6 RAFT_ERR_NOMEM=-5,
#define RAFT_ERR_SNAPSHOT_IN_PROGRESS -7 RAFT_ERR_NEEDS_SNAPSHOT=-6,
#define RAFT_ERR_SNAPSHOT_ALREADY_LOADED -8 RAFT_ERR_SNAPSHOT_IN_PROGRESS=-7,
#define RAFT_ERR_LAST -100 RAFT_ERR_SNAPSHOT_ALREADY_LOADED=-8,
RAFT_ERR_LAST=-100,
} raft_error_e;
typedef enum {
RAFT_MEMBERSHIP_ADD,
RAFT_MEMBERSHIP_REMOVE,
} raft_membership_e;
#define RAFT_REQUESTVOTE_ERR_GRANTED 1 #define RAFT_REQUESTVOTE_ERR_GRANTED 1
#define RAFT_REQUESTVOTE_ERR_NOT_GRANTED 0 #define RAFT_REQUESTVOTE_ERR_NOT_GRANTED 0
...@@ -64,7 +71,6 @@ typedef enum { ...@@ -64,7 +71,6 @@ typedef enum {
* Removing nodes is a 2 step process: first demote, then remove. * Removing nodes is a 2 step process: first demote, then remove.
*/ */
RAFT_LOGTYPE_REMOVE_NODE, RAFT_LOGTYPE_REMOVE_NODE,
RAFT_LOGTYPE_SNAPSHOT,
/** /**
* Users can piggyback the entry mechanism by specifying log types that * Users can piggyback the entry mechanism by specifying log types that
* are higher than RAFT_LOGTYPE_NUM. * are higher than RAFT_LOGTYPE_NUM.
...@@ -297,7 +303,7 @@ typedef int ( ...@@ -297,7 +303,7 @@ typedef int (
* @param[in] raft The Raft server making this callback * @param[in] raft The Raft server making this callback
* @param[in] user_data User data that is passed from Raft server * @param[in] user_data User data that is passed from Raft server
* @param[in] term Current term * @param[in] term Current term
* @param[in] vote The node value dicating we haven't voted for anybody * @param[in] vote The node value dictating we haven't voted for anybody
* @return 0 on success */ * @return 0 on success */
typedef int ( typedef int (
*func_persist_term_f *func_persist_term_f
...@@ -337,6 +343,25 @@ typedef int ( ...@@ -337,6 +343,25 @@ typedef int (
raft_index_t entry_idx raft_index_t entry_idx
); );
/** Callback for being notified of membership changes.
*
* Implementing this callback is optional.
*
* Remove notification happens before the node is about to be removed.
*
* @param[in] raft The Raft server making this callback
* @param[in] user_data User data that is passed from Raft server
* @param[in] node The node that is the subject of this log. Could be NULL.
* @param[in] type The type of membership change */
typedef void (
*func_membership_event_f
) (
raft_server_t* raft,
void *user_data,
raft_node_t *node,
raft_membership_e type
);
typedef struct typedef struct
{ {
/** Callback for sending request vote messages */ /** Callback for sending request vote messages */
...@@ -388,6 +413,8 @@ typedef struct ...@@ -388,6 +413,8 @@ typedef struct
/** Callback for detecting when a non-voting node has sufficient logs. */ /** Callback for detecting when a non-voting node has sufficient logs. */
func_node_has_sufficient_logs_f node_has_sufficient_logs; func_node_has_sufficient_logs_f node_has_sufficient_logs;
func_membership_event_f notify_membership_event;
/** Callback for catching debugging log messages /** Callback for catching debugging log messages
* This callback is optional */ * This callback is optional */
func_log_f log; func_log_f log;
...@@ -735,12 +762,12 @@ raft_node_id_t raft_node_get_id(raft_node_t* me_); ...@@ -735,12 +762,12 @@ raft_node_id_t raft_node_get_id(raft_node_t* me_);
* @return get state of type raft_state_e. */ * @return get state of type raft_state_e. */
int raft_get_state(raft_server_t* me_); int raft_get_state(raft_server_t* me_);
/** The the most recent log's term /** Get the most recent log's term
* @return the last log term */ * @return the last log term */
raft_term_t raft_get_last_log_term(raft_server_t* me_); raft_term_t raft_get_last_log_term(raft_server_t* me_);
/** Turn a node into a voting node. /** Turn a node into a voting node.
* Voting nodes can take part in elections and in-regards to commiting entries, * Voting nodes can take part in elections and in-regards to committing entries,
* are counted in majorities. */ * are counted in majorities. */
void raft_node_set_voting(raft_node_t* node, int voting); void raft_node_set_voting(raft_node_t* node, int voting);
...@@ -829,6 +856,9 @@ raft_index_t raft_get_first_entry_idx(raft_server_t* me_); ...@@ -829,6 +856,9 @@ raft_index_t raft_get_first_entry_idx(raft_server_t* me_);
* This is usually the result of a snapshot being loaded. * This is usually the result of a snapshot being loaded.
* We need to send an appendentries response. * We need to send an appendentries response.
* *
* This will remove all other nodes (not ourself). The user MUST use the
* snapshot to load the new membership information.
*
* @param[in] last_included_term Term of the last log of the snapshot * @param[in] last_included_term Term of the last log of the snapshot
* @param[in] last_included_index Index of the last log of the snapshot * @param[in] last_included_index Index of the last log of the snapshot
* *
...@@ -889,4 +919,19 @@ void raft_set_heap_functions(void *(*_malloc)(size_t), ...@@ -889,4 +919,19 @@ void raft_set_heap_functions(void *(*_malloc)(size_t),
void *(*_realloc)(void *, size_t), void *(*_realloc)(void *, size_t),
void (*_free)(void *)); void (*_free)(void *));
/** Confirm that a node's voting status is final
* @param[in] node The node
* @param[in] voting Whether this node's voting status is committed or not */
void raft_node_set_voting_committed(raft_node_t* me_, int voting);
/** Confirm that a node's voting status is final
* @param[in] node The node
* @param[in] committed Whether this node's membership is committed or not */
void raft_node_set_addition_committed(raft_node_t* me_, int committed);
/** Check if a voting change is in progress
* @param[in] raft The Raft server
* @return 1 if a voting change is in progress */
int raft_voting_change_is_in_progress(raft_server_t* me_);
#endif /* RAFT_H_ */ #endif /* RAFT_H_ */
...@@ -19,7 +19,7 @@ void log_clear(log_t* me_); ...@@ -19,7 +19,7 @@ void log_clear(log_t* me_);
* Add entry to log. * Add entry to log.
* Don't add entry if we've already added this entry (based off ID) * Don't add entry if we've already added this entry (based off ID)
* Don't add entries with ID=0 * Don't add entries with ID=0
* @return 0 if unsucessful; 1 otherwise */ * @return 0 if unsuccessful; 1 otherwise */
int log_append_entry(log_t* me_, raft_entry_t* c); int log_append_entry(log_t* me_, raft_entry_t* c);
/** /**
......
...@@ -80,21 +80,7 @@ int log_load_from_snapshot(log_t *me_, raft_index_t idx, raft_term_t term) ...@@ -80,21 +80,7 @@ int log_load_from_snapshot(log_t *me_, raft_index_t idx, raft_term_t term)
log_private_t* me = (log_private_t*)me_; log_private_t* me = (log_private_t*)me_;
log_clear(me_); log_clear(me_);
me->base = idx;
raft_entry_t ety;
ety.data.len = 0;
ety.id = 1;
ety.term = term;
ety.type = RAFT_LOGTYPE_SNAPSHOT;
int e = log_append_entry(me_, &ety);
if (e != 0)
{
assert(0);
return e;
}
me->base = idx - 1;
return 0; return 0;
} }
...@@ -202,10 +188,7 @@ raft_entry_t* log_get_at_idx(log_t* me_, raft_index_t idx) ...@@ -202,10 +188,7 @@ raft_entry_t* log_get_at_idx(log_t* me_, raft_index_t idx)
if (idx == 0) if (idx == 0)
return NULL; return NULL;
if (idx <= me->base) if (me->base + me->count < idx || idx <= me->base)
return NULL;
if (me->base + me->count < idx)
return NULL; return NULL;
/* idx starts at 1 */ /* idx starts at 1 */
......
...@@ -431,15 +431,25 @@ int raft_recv_appendentries( ...@@ -431,15 +431,25 @@ int raft_recv_appendentries(
{ {
raft_entry_t* ety = raft_get_entry_from_idx(me_, ae->prev_log_idx); raft_entry_t* ety = raft_get_entry_from_idx(me_, ae->prev_log_idx);
/* Is a snapshot */
if (ae->prev_log_idx == me->snapshot_last_idx)
{
if (me->snapshot_last_term != ae->prev_log_term)
{
/* Should never happen; something is seriously wrong! */
__log(me_, node, "Snapshot AE prev conflicts with committed entry");
e = RAFT_ERR_SHUTDOWN;
goto out;
}
}
/* 2. Reply false if log doesn't contain an entry at prevLogIndex /* 2. Reply false if log doesn't contain an entry at prevLogIndex
whose term matches prevLogTerm (§5.3) */ whose term matches prevLogTerm (§5.3) */
if (!ety) else if (!ety)
{ {
__log(me_, node, "AE no log at prev_idx %d", ae->prev_log_idx); __log(me_, node, "AE no log at prev_idx %d", ae->prev_log_idx);
goto out; goto out;
} }
else if (ety->term != ae->prev_log_term)
if (ety->term != ae->prev_log_term)
{ {
__log(me_, node, "AE term doesn't match prev_term (ie. %d vs %d) ci:%d comi:%d lcomi:%d pli:%d", __log(me_, node, "AE term doesn't match prev_term (ie. %d vs %d) ci:%d comi:%d lcomi:%d pli:%d",
ety->term, ae->prev_log_term, raft_get_current_idx(me_), ety->term, ae->prev_log_term, raft_get_current_idx(me_),
...@@ -528,7 +538,7 @@ static int __should_grant_vote(raft_server_private_t* me, msg_requestvote_t* vr) ...@@ -528,7 +538,7 @@ static int __should_grant_vote(raft_server_private_t* me, msg_requestvote_t* vr)
if (vr->term < raft_get_current_term((void*)me)) if (vr->term < raft_get_current_term((void*)me))
return 0; return 0;
/* TODO: if voted for is candiate return 1 (if below checks pass) */ /* TODO: if voted for is candidate return 1 (if below checks pass) */
if (raft_already_voted((void*)me)) if (raft_already_voted((void*)me))
return 0; return 0;
...@@ -541,10 +551,20 @@ static int __should_grant_vote(raft_server_private_t* me, msg_requestvote_t* vr) ...@@ -541,10 +551,20 @@ static int __should_grant_vote(raft_server_private_t* me, msg_requestvote_t* vr)
return 1; return 1;
raft_entry_t* ety = raft_get_entry_from_idx((void*)me, current_idx); raft_entry_t* ety = raft_get_entry_from_idx((void*)me, current_idx);
if (ety->term < vr->last_log_term) int ety_term;
// TODO: add test
if (ety)
ety_term = ety->term;
else if (!ety && me->snapshot_last_idx == current_idx)
ety_term = me->snapshot_last_term;
else
return 0;
if (ety_term < vr->last_log_term)
return 1; return 1;
if (vr->last_log_term == ety->term && current_idx <= vr->last_log_idx) if (vr->last_log_term == ety_term && current_idx <= vr->last_log_idx)
return 1; return 1;
return 0; return 0;
...@@ -591,7 +611,7 @@ int raft_recv_requestvote(raft_server_t* me_, ...@@ -591,7 +611,7 @@ int raft_recv_requestvote(raft_server_t* me_,
else else
r->vote_granted = 0; r->vote_granted = 0;
/* there must be in an election. */ /* must be in an election. */
me->current_leader = NULL; me->current_leader = NULL;
me->timeout_elapsed = 0; me->timeout_elapsed = 0;
...@@ -722,6 +742,7 @@ int raft_recv_entry(raft_server_t* me_, ...@@ -722,6 +742,7 @@ int raft_recv_entry(raft_server_t* me_,
int e = raft_append_entry(me_, ety); int e = raft_append_entry(me_, ety);
if (0 != e) if (0 != e)
return e; return e;
for (i = 0; i < me->num_nodes; i++) for (i = 0; i < me->num_nodes; i++)
{ {
raft_node_t* node = me->nodes[i]; raft_node_t* node = me->nodes[i];
...@@ -748,6 +769,7 @@ int raft_recv_entry(raft_server_t* me_, ...@@ -748,6 +769,7 @@ int raft_recv_entry(raft_server_t* me_,
r->idx = raft_get_current_idx(me_); r->idx = raft_get_current_idx(me_);
r->term = me->current_term; r->term = me->current_term;
/* FIXME: is this required if raft_append_entry does this too? */
if (raft_entry_is_voting_cfg_change(ety)) if (raft_entry_is_voting_cfg_change(ety))
me->voting_cfg_change_log_idx = raft_get_current_idx(me_); me->voting_cfg_change_log_idx = raft_get_current_idx(me_);
...@@ -892,11 +914,16 @@ int raft_send_appendentries(raft_server_t* me_, raft_node_t* node) ...@@ -892,11 +914,16 @@ int raft_send_appendentries(raft_server_t* me_, raft_node_t* node)
if (1 < next_idx) if (1 < next_idx)
{ {
raft_entry_t* prev_ety = raft_get_entry_from_idx(me_, next_idx - 1); raft_entry_t* prev_ety = raft_get_entry_from_idx(me_, next_idx - 1);
if (!prev_ety)
{
ae.prev_log_idx = me->snapshot_last_idx;
ae.prev_log_term = me->snapshot_last_term;
}
else
{
ae.prev_log_idx = next_idx - 1; ae.prev_log_idx = next_idx - 1;
if (prev_ety)
ae.prev_log_term = prev_ety->term; ae.prev_log_term = prev_ety->term;
else }
ae.prev_log_term = me->snapshot_last_term;
} }
__log(me_, node, "sending appendentries node: ci:%d comi:%d t:%d lc:%d pli:%d plt:%d", __log(me_, node, "sending appendentries node: ci:%d comi:%d t:%d lc:%d pli:%d plt:%d",
...@@ -961,7 +988,12 @@ raft_node_t* raft_add_node(raft_server_t* me_, void* udata, raft_node_id_t id, i ...@@ -961,7 +988,12 @@ raft_node_t* raft_add_node(raft_server_t* me_, void* udata, raft_node_id_t id, i
if (is_self) if (is_self)
me->node = me->nodes[me->num_nodes - 1]; me->node = me->nodes[me->num_nodes - 1];
return me->nodes[me->num_nodes - 1]; node = me->nodes[me->num_nodes - 1];
if (me->cb.notify_membership_event)
me->cb.notify_membership_event(me_, raft_get_udata(me_), node, RAFT_MEMBERSHIP_ADD);
return node;
} }
raft_node_t* raft_add_non_voting_node(raft_server_t* me_, void* udata, raft_node_id_t id, int is_self) raft_node_t* raft_add_non_voting_node(raft_server_t* me_, void* udata, raft_node_id_t id, int is_self)
...@@ -981,6 +1013,9 @@ void raft_remove_node(raft_server_t* me_, raft_node_t* node) ...@@ -981,6 +1013,9 @@ void raft_remove_node(raft_server_t* me_, raft_node_t* node)
{ {
raft_server_private_t* me = (raft_server_private_t*)me_; raft_server_private_t* me = (raft_server_private_t*)me_;
if (me->cb.notify_membership_event)
me->cb.notify_membership_event(me_, raft_get_udata(me_), node, RAFT_MEMBERSHIP_REMOVE);
assert(node); assert(node);
int i, found = 0; int i, found = 0;
...@@ -1104,16 +1139,14 @@ void raft_offer_log(raft_server_t* me_, raft_entry_t* ety, const raft_index_t id ...@@ -1104,16 +1139,14 @@ void raft_offer_log(raft_server_t* me_, raft_entry_t* ety, const raft_index_t id
} }
else if (!node) else if (!node)
{ {
raft_node_t* node = raft_add_non_voting_node(me_, NULL, node_id, is_self); node = raft_add_non_voting_node(me_, NULL, node_id, is_self);
assert(node); assert(node);
} }
} }
break; break;
case RAFT_LOGTYPE_ADD_NODE: case RAFT_LOGTYPE_ADD_NODE:
if (!node) {
node = raft_add_node(me_, NULL, node_id, is_self); node = raft_add_node(me_, NULL, node_id, is_self);
}
assert(node); assert(node);
assert(raft_node_is_voting(node)); assert(raft_node_is_voting(node));
break; break;
...@@ -1261,6 +1294,9 @@ int raft_end_snapshot(raft_server_t *me_) ...@@ -1261,6 +1294,9 @@ int raft_end_snapshot(raft_server_t *me_)
if (!me->snapshot_in_progress || me->snapshot_last_idx == 0) if (!me->snapshot_in_progress || me->snapshot_last_idx == 0)
return -1; return -1;
assert(raft_get_num_snapshottable_logs(me_) != 0);
assert(me->snapshot_last_idx == raft_get_commit_idx(me_));
/* If needed, remove compacted logs */ /* If needed, remove compacted logs */
raft_index_t i = log_get_base(me->log) + 1, end = raft_get_commit_idx(me_); raft_index_t i = log_get_base(me->log) + 1, end = raft_get_commit_idx(me_);
for (; i <= end; i++) for (; i <= end; i++)
...@@ -1299,6 +1335,8 @@ int raft_end_snapshot(raft_server_t *me_) ...@@ -1299,6 +1335,8 @@ int raft_end_snapshot(raft_server_t *me_)
} }
} }
assert(raft_get_log_count(me_) == 1);
return 0; return 0;
} }
......
import cffi
import subprocess
ffi = cffi.FFI()
ffi.set_source(
"tests",
"""
""",
sources="""
src/raft_log.c
src/raft_server.c
src/raft_server_properties.c
src/raft_node.c
""".split(),
include_dirs=["include"],
)
library = ffi.compile()
ffi = cffi.FFI()
lib = ffi.dlopen(library)
def load(fname):
return '\n'.join(
[line for line in subprocess.check_output(
["gcc", "-E", fname]).decode('utf-8').split('\n')])
ffi.cdef('void *malloc(size_t __size);')
ffi.cdef(load('include/raft.h'))
ffi.cdef(load('include/raft_log.h'))
...@@ -399,10 +399,7 @@ void TestLog_load_from_snapshot(CuTest * tc) ...@@ -399,10 +399,7 @@ void TestLog_load_from_snapshot(CuTest * tc)
CuAssertIntEquals(tc, 0, log_get_current_idx(l)); CuAssertIntEquals(tc, 0, log_get_current_idx(l));
CuAssertIntEquals(tc, 0, log_load_from_snapshot(l, 10, 5)); CuAssertIntEquals(tc, 0, log_load_from_snapshot(l, 10, 5));
CuAssertIntEquals(tc, 10, log_get_current_idx(l)); CuAssertIntEquals(tc, 10, log_get_current_idx(l));
CuAssertIntEquals(tc, 0, log_count(l));
/* this is just a marker
* it should never be sent to any nodes because it is part of a snapshot */
CuAssertIntEquals(tc, 1, log_count(l));
} }
void TestLog_load_from_snapshot_clears_log(CuTest * tc) void TestLog_load_from_snapshot_clears_log(CuTest * tc)
...@@ -422,7 +419,7 @@ void TestLog_load_from_snapshot_clears_log(CuTest * tc) ...@@ -422,7 +419,7 @@ void TestLog_load_from_snapshot_clears_log(CuTest * tc)
CuAssertIntEquals(tc, 2, log_get_current_idx(l)); CuAssertIntEquals(tc, 2, log_get_current_idx(l));
CuAssertIntEquals(tc, 0, log_load_from_snapshot(l, 10, 5)); CuAssertIntEquals(tc, 0, log_load_from_snapshot(l, 10, 5));
CuAssertIntEquals(tc, 1, log_count(l)); CuAssertIntEquals(tc, 0, log_count(l));
CuAssertIntEquals(tc, 10, log_get_current_idx(l)); CuAssertIntEquals(tc, 10, log_get_current_idx(l));
} }
......
...@@ -82,14 +82,6 @@ static int __raft_log_offer(raft_server_t* raft, ...@@ -82,14 +82,6 @@ static int __raft_log_offer(raft_server_t* raft,
raft_entry_t *entry, raft_entry_t *entry,
raft_index_t entry_idx) raft_index_t entry_idx)
{ {
switch (entry->type) {
case RAFT_LOGTYPE_ADD_NONVOTING_NODE:
raft_add_non_voting_node(raft, NULL, atoi(entry->data.buf), 0);
break;
case RAFT_LOGTYPE_ADD_NODE:
raft_add_node(raft, NULL, atoi(entry->data.buf), 0);
break;
}
return 0; return 0;
} }
...@@ -3963,10 +3955,10 @@ void TestRaft_leader_recv_appendentries_response_set_has_sufficient_logs_after_v ...@@ -3963,10 +3955,10 @@ void TestRaft_leader_recv_appendentries_response_set_has_sufficient_logs_after_v
.type = RAFT_LOGTYPE_ADD_NONVOTING_NODE .type = RAFT_LOGTYPE_ADD_NONVOTING_NODE
}; };
msg_entry_response_t etyr; msg_entry_response_t etyr;
raft_recv_entry(r, &ety, &etyr); CuAssertIntEquals(tc, 0, raft_recv_entry(r, &ety, &etyr));
ety.id++; ety.id++;
ety.data.buf = "3"; ety.data.buf = "3";
raft_recv_entry(r, &ety, &etyr); CuAssertIntEquals(tc, 0, raft_recv_entry(r, &ety, &etyr));
msg_appendentries_response_t aer = { msg_appendentries_response_t aer = {
.term = 1, .success = 1, .current_idx = 2, .first_idx = 0 .term = 1, .success = 1, .current_idx = 2, .first_idx = 0
...@@ -3999,4 +3991,3 @@ void TestRaft_leader_recv_appendentries_response_set_has_sufficient_logs_after_v ...@@ -3999,4 +3991,3 @@ void TestRaft_leader_recv_appendentries_response_set_has_sufficient_logs_after_v
raft_recv_appendentries_response(r, raft_get_node(r, 2), &aer); raft_recv_appendentries_response(r, raft_get_node(r, 2), &aer);
CuAssertIntEquals(tc, 2, has_sufficient_logs_flag); CuAssertIntEquals(tc, 2, has_sufficient_logs_flag);
} }
...@@ -352,7 +352,7 @@ void TestRaft_follower_load_from_snapshot(CuTest * tc) ...@@ -352,7 +352,7 @@ void TestRaft_follower_load_from_snapshot(CuTest * tc)
CuAssertIntEquals(tc, 0, raft_get_log_count(r)); CuAssertIntEquals(tc, 0, raft_get_log_count(r));
CuAssertIntEquals(tc, 0, raft_begin_load_snapshot(r, 5, 5)); CuAssertIntEquals(tc, 0, raft_begin_load_snapshot(r, 5, 5));
CuAssertIntEquals(tc, 0, raft_end_load_snapshot(r)); CuAssertIntEquals(tc, 0, raft_end_load_snapshot(r));
CuAssertIntEquals(tc, 1, raft_get_log_count(r)); CuAssertIntEquals(tc, 0, raft_get_log_count(r));
CuAssertIntEquals(tc, 0, raft_get_num_snapshottable_logs(r)); CuAssertIntEquals(tc, 0, raft_get_num_snapshottable_logs(r));
CuAssertIntEquals(tc, 5, raft_get_commit_idx(r)); CuAssertIntEquals(tc, 5, raft_get_commit_idx(r));
CuAssertIntEquals(tc, 5, raft_get_last_applied_idx(r)); CuAssertIntEquals(tc, 5, raft_get_last_applied_idx(r));
...@@ -408,7 +408,7 @@ void TestRaft_follower_load_from_snapshot_fails_if_already_loaded(CuTest * tc) ...@@ -408,7 +408,7 @@ void TestRaft_follower_load_from_snapshot_fails_if_already_loaded(CuTest * tc)
CuAssertIntEquals(tc, 0, raft_get_log_count(r)); CuAssertIntEquals(tc, 0, raft_get_log_count(r));
CuAssertIntEquals(tc, 0, raft_begin_load_snapshot(r, 5, 5)); CuAssertIntEquals(tc, 0, raft_begin_load_snapshot(r, 5, 5));
CuAssertIntEquals(tc, 0, raft_end_load_snapshot(r)); CuAssertIntEquals(tc, 0, raft_end_load_snapshot(r));
CuAssertIntEquals(tc, 1, raft_get_log_count(r)); CuAssertIntEquals(tc, 0, raft_get_log_count(r));
CuAssertIntEquals(tc, 0, raft_get_num_snapshottable_logs(r)); CuAssertIntEquals(tc, 0, raft_get_num_snapshottable_logs(r));
CuAssertIntEquals(tc, 5, raft_get_commit_idx(r)); CuAssertIntEquals(tc, 5, raft_get_commit_idx(r));
CuAssertIntEquals(tc, 5, raft_get_last_applied_idx(r)); CuAssertIntEquals(tc, 5, raft_get_last_applied_idx(r));
...@@ -527,7 +527,7 @@ void TestRaft_leader_sends_appendentries_when_node_next_index_was_compacted(CuTe ...@@ -527,7 +527,7 @@ void TestRaft_leader_sends_appendentries_when_node_next_index_was_compacted(CuTe
raft_set_current_term(r, 2); raft_set_current_term(r, 2);
CuAssertIntEquals(tc, 0, raft_send_appendentries(r, node)); CuAssertIntEquals(tc, 0, raft_send_appendentries(r, node));
CuAssertIntEquals(tc, 2, ae.term); CuAssertIntEquals(tc, 2, ae.term);
CuAssertIntEquals(tc, 2, ae.prev_log_idx); CuAssertIntEquals(tc, 3, ae.prev_log_idx);
CuAssertIntEquals(tc, 2, ae.prev_log_term); CuAssertIntEquals(tc, 2, ae.prev_log_term);
} }
...@@ -568,3 +568,69 @@ void TestRaft_recv_entry_fails_if_snapshot_in_progress(CuTest* tc) ...@@ -568,3 +568,69 @@ void TestRaft_recv_entry_fails_if_snapshot_in_progress(CuTest* tc)
ety.type = RAFT_LOGTYPE_ADD_NODE; ety.type = RAFT_LOGTYPE_ADD_NODE;
CuAssertIntEquals(tc, RAFT_ERR_SNAPSHOT_IN_PROGRESS, raft_recv_entry(r, &ety, &cr)); CuAssertIntEquals(tc, RAFT_ERR_SNAPSHOT_IN_PROGRESS, raft_recv_entry(r, &ety, &cr));
} }
void TestRaft_follower_recv_appendentries_is_successful_when_previous_log_idx_equals_snapshot_last_idx(
CuTest * tc)
{
raft_cbs_t funcs = {
.persist_term = __raft_persist_term,
};
void *r = raft_new();
raft_set_callbacks(r, &funcs, NULL);
raft_add_node(r, NULL, 1, 1);
raft_add_node(r, NULL, 2, 0);
CuAssertIntEquals(tc, 0, raft_begin_load_snapshot(r, 2, 2));
CuAssertIntEquals(tc, 0, raft_end_load_snapshot(r));
msg_appendentries_t ae;
msg_appendentries_response_t aer;
memset(&ae, 0, sizeof(msg_appendentries_t));
ae.term = 3;
ae.prev_log_idx = 2;
ae.prev_log_term = 2;
/* include entries */
msg_entry_t e[5];
memset(&e, 0, sizeof(msg_entry_t) * 4);
e[0].term = 3;
e[0].id = 3;
ae.entries = e;
ae.n_entries = 1;
CuAssertIntEquals(tc, 0, raft_recv_appendentries(r, raft_get_node(r, 2), &ae, &aer));
CuAssertIntEquals(tc, 1, aer.success);
}
void TestRaft_leader_sends_appendentries_with_correct_prev_log_idx_when_snapshotted(
CuTest * tc)
{
raft_cbs_t funcs = {
.send_appendentries = sender_appendentries,
.log = NULL
};
void *sender = sender_new(NULL);
void *r = raft_new();
raft_set_callbacks(r, &funcs, sender);
CuAssertTrue(tc, NULL != raft_add_node(r, NULL, 1, 1));
CuAssertTrue(tc, NULL != raft_add_node(r, NULL, 2, 0));
CuAssertIntEquals(tc, 0, raft_begin_load_snapshot(r, 2, 4));
CuAssertIntEquals(tc, 0, raft_end_load_snapshot(r));
/* i'm leader */
raft_set_state(r, RAFT_STATE_LEADER);
raft_node_t* p = raft_get_node_from_idx(r, 1);
CuAssertTrue(tc, NULL != p);
raft_node_set_next_idx(p, 4);
/* receive appendentries messages */
raft_send_appendentries(r, p);
msg_appendentries_t* ae = sender_poll_msg_data(sender);
CuAssertTrue(tc, NULL != ae);
CuAssertIntEquals(tc, 2, ae->prev_log_term);
CuAssertIntEquals(tc, 4, ae->prev_log_idx);
}
This diff is collapsed.
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment