Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
ruanhaishen
RedisLabs Raft
Commits
3c134468
Commit
3c134468
authored
Jul 11, 2018
by
Yossi Gottlieb
Browse files
Merge remote-tracking branch 'upstream/master'
parents
876f8d7c
9b983183
Changes
14
Hide whitespace changes
Inline
Side-by-side
Makefile
View file @
3c134468
...
@@ -72,9 +72,7 @@ tests_full:
...
@@ -72,9 +72,7 @@ tests_full:
.PHONY
:
test_virtraft
.PHONY
:
test_virtraft
test_virtraft
:
test_virtraft
:
cp
src/
*
.c virtraft/deps/raft/
python3 tests/virtraft2.py
--servers
7
-i
20000
--compaction_rate
50
--drop_rate
5
-P
10
--seed
1
-m
3
cp
include/
*
.h virtraft/deps/raft/
cd
virtraft
;
make clean
;
make
;
make tests
.PHONY
:
amalgamation
.PHONY
:
amalgamation
amalgamation
:
amalgamation
:
...
...
README.rst
View file @
3c134468
...
@@ -27,12 +27,43 @@ Quality Assurance
...
@@ -27,12 +27,43 @@ Quality Assurance
We use the following methods to ensure that the library is safe:
We use the following methods to ensure that the library is safe:
* A
`
simulator
<https://github.com/willemt/
virtraft
>`_
is used to test the Raft invariants on unreliable networks
* A simulator
(
virtraft
2)
is used to test the Raft invariants on unreliable networks
* `Fuzzing/property-based-testing <https://github.com/willemt/virtraft/blob/master/tests/test_fuzzer.py>`_ via `Hypothesis <https://github.com/DRMacIver/hypothesis/>`_
* `Fuzzing/property-based-testing <https://github.com/willemt/virtraft/blob/master/tests/test_fuzzer.py>`_ via `Hypothesis <https://github.com/DRMacIver/hypothesis/>`_
* All bugs have regression tests
* All bugs have regression tests
* Many unit tests
* Many unit tests
* `Usage <https://github.com/willemt/ticketd>`_
* `Usage <https://github.com/willemt/ticketd>`_
virtraft2
---------
This cluster simulator checks the following:
* Log Matching (servers must have matching logs)
* State Machine Safety (applied entries have the same ID)
* Election Safety (only one valid leader per term)
* Current Index Validity (does the current index have an existing entry?)
* Entry ID Monotonicity (entries aren't appended out of order)
* Committed entry popping (committed entries are not popped from the log)
* Log Accuracy (does the server's log match mirror an independent log?)
* Deadlock detection (does the cluster continuously make progress?)
Chaos generated by virtraft2:
* Random bi-directional partitions between nodes
* Message dropping
* Message duplication
* Membership change injection
* Random compactions
Run the simulator using:
.. code-block:: bash
:class: ignore
make test_virtraft
virtraft2 succeeds `virtraft <https://github.com/willemt/virtraft>`_
Single file amalgamation
Single file amalgamation
========================
========================
...
@@ -440,7 +471,7 @@ The process works like this:
...
@@ -440,7 +471,7 @@ The process works like this:
2. Save the current membership details to the snapshot.
2. Save the current membership details to the snapshot.
3. Save the finite state machine to the snapshot.
3. Save the finite state machine to the snapshot.
4. End snapshotting with ``raft_end_snapshot``.
4. End snapshotting with ``raft_end_snapshot``.
5. When the ``send_snapshot`` callback fires, the user must propogate the snapshot to the
other node
.
5. When the ``send_snapshot`` callback fires, the user must propogate the snapshot to the
peer
.
6. Once the peer has the snapshot, they call ``raft_begin_load_snapshot``.
6. Once the peer has the snapshot, they call ``raft_begin_load_snapshot``.
7. Peer calls ``raft_add_node`` to add nodes as per the snapshot's membership info.
7. Peer calls ``raft_add_node`` to add nodes as per the snapshot's membership info.
8. Peer calls ``raft_node_set_voting`` to nodes as per the snapshot's membership info.
8. Peer calls ``raft_node_set_voting`` to nodes as per the snapshot's membership info.
...
...
include/raft.h
View file @
3c134468
...
@@ -12,14 +12,21 @@
...
@@ -12,14 +12,21 @@
#include "raft_types.h"
#include "raft_types.h"
#define RAFT_ERR_NOT_LEADER -2
typedef
enum
{
#define RAFT_ERR_ONE_VOTING_CHANGE_ONLY -3
RAFT_ERR_NOT_LEADER
=-
2
,
#define RAFT_ERR_SHUTDOWN -4
RAFT_ERR_ONE_VOTING_CHANGE_ONLY
=-
3
,
#define RAFT_ERR_NOMEM -5
RAFT_ERR_SHUTDOWN
=-
4
,
#define RAFT_ERR_NEEDS_SNAPSHOT -6
RAFT_ERR_NOMEM
=-
5
,
#define RAFT_ERR_SNAPSHOT_IN_PROGRESS -7
RAFT_ERR_NEEDS_SNAPSHOT
=-
6
,
#define RAFT_ERR_SNAPSHOT_ALREADY_LOADED -8
RAFT_ERR_SNAPSHOT_IN_PROGRESS
=-
7
,
#define RAFT_ERR_LAST -100
RAFT_ERR_SNAPSHOT_ALREADY_LOADED
=-
8
,
RAFT_ERR_LAST
=-
100
,
}
raft_error_e
;
typedef
enum
{
RAFT_MEMBERSHIP_ADD
,
RAFT_MEMBERSHIP_REMOVE
,
}
raft_membership_e
;
#define RAFT_REQUESTVOTE_ERR_GRANTED 1
#define RAFT_REQUESTVOTE_ERR_GRANTED 1
#define RAFT_REQUESTVOTE_ERR_NOT_GRANTED 0
#define RAFT_REQUESTVOTE_ERR_NOT_GRANTED 0
...
@@ -64,7 +71,6 @@ typedef enum {
...
@@ -64,7 +71,6 @@ typedef enum {
* Removing nodes is a 2 step process: first demote, then remove.
* Removing nodes is a 2 step process: first demote, then remove.
*/
*/
RAFT_LOGTYPE_REMOVE_NODE
,
RAFT_LOGTYPE_REMOVE_NODE
,
RAFT_LOGTYPE_SNAPSHOT
,
/**
/**
* Users can piggyback the entry mechanism by specifying log types that
* Users can piggyback the entry mechanism by specifying log types that
* are higher than RAFT_LOGTYPE_NUM.
* are higher than RAFT_LOGTYPE_NUM.
...
@@ -229,7 +235,7 @@ typedef int (
...
@@ -229,7 +235,7 @@ typedef int (
msg_appendentries_t
*
msg
msg_appendentries_t
*
msg
);
);
/**
/**
* Log compaction
* Log compaction
* Callback for telling the user to send a snapshot.
* Callback for telling the user to send a snapshot.
*
*
...
@@ -297,7 +303,7 @@ typedef int (
...
@@ -297,7 +303,7 @@ typedef int (
* @param[in] raft The Raft server making this callback
* @param[in] raft The Raft server making this callback
* @param[in] user_data User data that is passed from Raft server
* @param[in] user_data User data that is passed from Raft server
* @param[in] term Current term
* @param[in] term Current term
* @param[in] vote The node value dicating we haven't voted for anybody
* @param[in] vote The node value dic
t
ating we haven't voted for anybody
* @return 0 on success */
* @return 0 on success */
typedef
int
(
typedef
int
(
*
func_persist_term_f
*
func_persist_term_f
...
@@ -337,6 +343,25 @@ typedef int (
...
@@ -337,6 +343,25 @@ typedef int (
raft_index_t
entry_idx
raft_index_t
entry_idx
);
);
/** Callback for being notified of membership changes.
*
* Implementing this callback is optional.
*
* Remove notification happens before the node is about to be removed.
*
* @param[in] raft The Raft server making this callback
* @param[in] user_data User data that is passed from Raft server
* @param[in] node The node that is the subject of this log. Could be NULL.
* @param[in] type The type of membership change */
typedef
void
(
*
func_membership_event_f
)
(
raft_server_t
*
raft
,
void
*
user_data
,
raft_node_t
*
node
,
raft_membership_e
type
);
typedef
struct
typedef
struct
{
{
/** Callback for sending request vote messages */
/** Callback for sending request vote messages */
...
@@ -388,6 +413,8 @@ typedef struct
...
@@ -388,6 +413,8 @@ typedef struct
/** Callback for detecting when a non-voting node has sufficient logs. */
/** Callback for detecting when a non-voting node has sufficient logs. */
func_node_has_sufficient_logs_f
node_has_sufficient_logs
;
func_node_has_sufficient_logs_f
node_has_sufficient_logs
;
func_membership_event_f
notify_membership_event
;
/** Callback for catching debugging log messages
/** Callback for catching debugging log messages
* This callback is optional */
* This callback is optional */
func_log_f
log
;
func_log_f
log
;
...
@@ -735,12 +762,12 @@ raft_node_id_t raft_node_get_id(raft_node_t* me_);
...
@@ -735,12 +762,12 @@ raft_node_id_t raft_node_get_id(raft_node_t* me_);
* @return get state of type raft_state_e. */
* @return get state of type raft_state_e. */
int
raft_get_state
(
raft_server_t
*
me_
);
int
raft_get_state
(
raft_server_t
*
me_
);
/**
The
the most recent log's term
/**
Get
the most recent log's term
* @return the last log term */
* @return the last log term */
raft_term_t
raft_get_last_log_term
(
raft_server_t
*
me_
);
raft_term_t
raft_get_last_log_term
(
raft_server_t
*
me_
);
/** Turn a node into a voting node.
/** Turn a node into a voting node.
* Voting nodes can take part in elections and in-regards to commiting entries,
* Voting nodes can take part in elections and in-regards to commit
t
ing entries,
* are counted in majorities. */
* are counted in majorities. */
void
raft_node_set_voting
(
raft_node_t
*
node
,
int
voting
);
void
raft_node_set_voting
(
raft_node_t
*
node
,
int
voting
);
...
@@ -829,8 +856,11 @@ raft_index_t raft_get_first_entry_idx(raft_server_t* me_);
...
@@ -829,8 +856,11 @@ raft_index_t raft_get_first_entry_idx(raft_server_t* me_);
* This is usually the result of a snapshot being loaded.
* This is usually the result of a snapshot being loaded.
* We need to send an appendentries response.
* We need to send an appendentries response.
*
*
* This will remove all other nodes (not ourself). The user MUST use the
* snapshot to load the new membership information.
*
* @param[in] last_included_term Term of the last log of the snapshot
* @param[in] last_included_term Term of the last log of the snapshot
* @param[in] last_included_index Index of the last log of the snapshot
* @param[in] last_included_index Index of the last log of the snapshot
*
*
* @return
* @return
* 0 on success
* 0 on success
...
@@ -889,4 +919,19 @@ void raft_set_heap_functions(void *(*_malloc)(size_t),
...
@@ -889,4 +919,19 @@ void raft_set_heap_functions(void *(*_malloc)(size_t),
void
*
(
*
_realloc
)(
void
*
,
size_t
),
void
*
(
*
_realloc
)(
void
*
,
size_t
),
void
(
*
_free
)(
void
*
));
void
(
*
_free
)(
void
*
));
/** Confirm that a node's voting status is final
* @param[in] node The node
* @param[in] voting Whether this node's voting status is committed or not */
void
raft_node_set_voting_committed
(
raft_node_t
*
me_
,
int
voting
);
/** Confirm that a node's voting status is final
* @param[in] node The node
* @param[in] committed Whether this node's membership is committed or not */
void
raft_node_set_addition_committed
(
raft_node_t
*
me_
,
int
committed
);
/** Check if a voting change is in progress
* @param[in] raft The Raft server
* @return 1 if a voting change is in progress */
int
raft_voting_change_is_in_progress
(
raft_server_t
*
me_
);
#endif
/* RAFT_H_ */
#endif
/* RAFT_H_ */
include/raft_log.h
View file @
3c134468
...
@@ -18,8 +18,8 @@ void log_clear(log_t* me_);
...
@@ -18,8 +18,8 @@ void log_clear(log_t* me_);
/**
/**
* Add entry to log.
* Add entry to log.
* Don't add entry if we've already added this entry (based off ID)
* Don't add entry if we've already added this entry (based off ID)
* Don't add entries with ID=0
* Don't add entries with ID=0
* @return 0 if unsucessful; 1 otherwise */
* @return 0 if unsuc
c
essful; 1 otherwise */
int
log_append_entry
(
log_t
*
me_
,
raft_entry_t
*
c
);
int
log_append_entry
(
log_t
*
me_
,
raft_entry_t
*
c
);
/**
/**
...
...
include/raft_private.h
View file @
3c134468
/**
/**
* Copyright (c) 2013, Willem-Hendrik Thiart
* Copyright (c) 2013, Willem-Hendrik Thiart
* Use of this source code is governed by a BSD-style license that can be
* Use of this source code is governed by a BSD-style license that can be
* found in the LICENSE file.
* found in the LICENSE file.
*
*
* @file
* @file
* @author Willem Thiart himself@willemthiart.com
* @author Willem Thiart himself@willemthiart.com
...
@@ -46,7 +46,7 @@ typedef struct {
...
@@ -46,7 +46,7 @@ typedef struct {
/* amount of time left till timeout */
/* amount of time left till timeout */
int
timeout_elapsed
;
int
timeout_elapsed
;
raft_node_t
*
nodes
;
raft_node_t
*
nodes
;
int
num_nodes
;
int
num_nodes
;
...
...
src/raft_log.c
View file @
3c134468
...
@@ -80,21 +80,7 @@ int log_load_from_snapshot(log_t *me_, raft_index_t idx, raft_term_t term)
...
@@ -80,21 +80,7 @@ int log_load_from_snapshot(log_t *me_, raft_index_t idx, raft_term_t term)
log_private_t
*
me
=
(
log_private_t
*
)
me_
;
log_private_t
*
me
=
(
log_private_t
*
)
me_
;
log_clear
(
me_
);
log_clear
(
me_
);
me
->
base
=
idx
;
raft_entry_t
ety
;
ety
.
data
.
len
=
0
;
ety
.
id
=
1
;
ety
.
term
=
term
;
ety
.
type
=
RAFT_LOGTYPE_SNAPSHOT
;
int
e
=
log_append_entry
(
me_
,
&
ety
);
if
(
e
!=
0
)
{
assert
(
0
);
return
e
;
}
me
->
base
=
idx
-
1
;
return
0
;
return
0
;
}
}
...
@@ -202,10 +188,7 @@ raft_entry_t* log_get_at_idx(log_t* me_, raft_index_t idx)
...
@@ -202,10 +188,7 @@ raft_entry_t* log_get_at_idx(log_t* me_, raft_index_t idx)
if
(
idx
==
0
)
if
(
idx
==
0
)
return
NULL
;
return
NULL
;
if
(
idx
<=
me
->
base
)
if
(
me
->
base
+
me
->
count
<
idx
||
idx
<=
me
->
base
)
return
NULL
;
if
(
me
->
base
+
me
->
count
<
idx
)
return
NULL
;
return
NULL
;
/* idx starts at 1 */
/* idx starts at 1 */
...
...
src/raft_server.c
View file @
3c134468
...
@@ -233,7 +233,7 @@ int raft_periodic(raft_server_t* me_, int msec_since_last_period)
...
@@ -233,7 +233,7 @@ int raft_periodic(raft_server_t* me_, int msec_since_last_period)
if
(
me
->
request_timeout
<=
me
->
timeout_elapsed
)
if
(
me
->
request_timeout
<=
me
->
timeout_elapsed
)
raft_send_appendentries_all
(
me_
);
raft_send_appendentries_all
(
me_
);
}
}
else
if
(
me
->
election_timeout_rand
<=
me
->
timeout_elapsed
&&
else
if
(
me
->
election_timeout_rand
<=
me
->
timeout_elapsed
&&
/* Don't become the leader when building snapshots or bad things will
/* Don't become the leader when building snapshots or bad things will
* happen when we get a client request */
* happen when we get a client request */
!
raft_snapshot_is_in_progress
(
me_
))
!
raft_snapshot_is_in_progress
(
me_
))
...
@@ -431,15 +431,25 @@ int raft_recv_appendentries(
...
@@ -431,15 +431,25 @@ int raft_recv_appendentries(
{
{
raft_entry_t
*
ety
=
raft_get_entry_from_idx
(
me_
,
ae
->
prev_log_idx
);
raft_entry_t
*
ety
=
raft_get_entry_from_idx
(
me_
,
ae
->
prev_log_idx
);
/* Is a snapshot */
if
(
ae
->
prev_log_idx
==
me
->
snapshot_last_idx
)
{
if
(
me
->
snapshot_last_term
!=
ae
->
prev_log_term
)
{
/* Should never happen; something is seriously wrong! */
__log
(
me_
,
node
,
"Snapshot AE prev conflicts with committed entry"
);
e
=
RAFT_ERR_SHUTDOWN
;
goto
out
;
}
}
/* 2. Reply false if log doesn't contain an entry at prevLogIndex
/* 2. Reply false if log doesn't contain an entry at prevLogIndex
whose term matches prevLogTerm (§5.3) */
whose term matches prevLogTerm (§5.3) */
if
(
!
ety
)
else
if
(
!
ety
)
{
{
__log
(
me_
,
node
,
"AE no log at prev_idx %d"
,
ae
->
prev_log_idx
);
__log
(
me_
,
node
,
"AE no log at prev_idx %d"
,
ae
->
prev_log_idx
);
goto
out
;
goto
out
;
}
}
else
if
(
ety
->
term
!=
ae
->
prev_log_term
)
if
(
ety
->
term
!=
ae
->
prev_log_term
)
{
{
__log
(
me_
,
node
,
"AE term doesn't match prev_term (ie. %d vs %d) ci:%d comi:%d lcomi:%d pli:%d"
,
__log
(
me_
,
node
,
"AE term doesn't match prev_term (ie. %d vs %d) ci:%d comi:%d lcomi:%d pli:%d"
,
ety
->
term
,
ae
->
prev_log_term
,
raft_get_current_idx
(
me_
),
ety
->
term
,
ae
->
prev_log_term
,
raft_get_current_idx
(
me_
),
...
@@ -528,7 +538,7 @@ static int __should_grant_vote(raft_server_private_t* me, msg_requestvote_t* vr)
...
@@ -528,7 +538,7 @@ static int __should_grant_vote(raft_server_private_t* me, msg_requestvote_t* vr)
if
(
vr
->
term
<
raft_get_current_term
((
void
*
)
me
))
if
(
vr
->
term
<
raft_get_current_term
((
void
*
)
me
))
return
0
;
return
0
;
/* TODO: if voted for is candiate return 1 (if below checks pass) */
/* TODO: if voted for is candi
d
ate return 1 (if below checks pass) */
if
(
raft_already_voted
((
void
*
)
me
))
if
(
raft_already_voted
((
void
*
)
me
))
return
0
;
return
0
;
...
@@ -541,10 +551,20 @@ static int __should_grant_vote(raft_server_private_t* me, msg_requestvote_t* vr)
...
@@ -541,10 +551,20 @@ static int __should_grant_vote(raft_server_private_t* me, msg_requestvote_t* vr)
return
1
;
return
1
;
raft_entry_t
*
ety
=
raft_get_entry_from_idx
((
void
*
)
me
,
current_idx
);
raft_entry_t
*
ety
=
raft_get_entry_from_idx
((
void
*
)
me
,
current_idx
);
if
(
ety
->
term
<
vr
->
last_log_term
)
int
ety_term
;
// TODO: add test
if
(
ety
)
ety_term
=
ety
->
term
;
else
if
(
!
ety
&&
me
->
snapshot_last_idx
==
current_idx
)
ety_term
=
me
->
snapshot_last_term
;
else
return
0
;
if
(
ety_term
<
vr
->
last_log_term
)
return
1
;
return
1
;
if
(
vr
->
last_log_term
==
ety
->
term
&&
current_idx
<=
vr
->
last_log_idx
)
if
(
vr
->
last_log_term
==
ety
_
term
&&
current_idx
<=
vr
->
last_log_idx
)
return
1
;
return
1
;
return
0
;
return
0
;
...
@@ -591,7 +611,7 @@ int raft_recv_requestvote(raft_server_t* me_,
...
@@ -591,7 +611,7 @@ int raft_recv_requestvote(raft_server_t* me_,
else
else
r
->
vote_granted
=
0
;
r
->
vote_granted
=
0
;
/*
there
must be in an election. */
/* must be in an election. */
me
->
current_leader
=
NULL
;
me
->
current_leader
=
NULL
;
me
->
timeout_elapsed
=
0
;
me
->
timeout_elapsed
=
0
;
...
@@ -722,6 +742,7 @@ int raft_recv_entry(raft_server_t* me_,
...
@@ -722,6 +742,7 @@ int raft_recv_entry(raft_server_t* me_,
int
e
=
raft_append_entry
(
me_
,
ety
);
int
e
=
raft_append_entry
(
me_
,
ety
);
if
(
0
!=
e
)
if
(
0
!=
e
)
return
e
;
return
e
;
for
(
i
=
0
;
i
<
me
->
num_nodes
;
i
++
)
for
(
i
=
0
;
i
<
me
->
num_nodes
;
i
++
)
{
{
raft_node_t
*
node
=
me
->
nodes
[
i
];
raft_node_t
*
node
=
me
->
nodes
[
i
];
...
@@ -748,6 +769,7 @@ int raft_recv_entry(raft_server_t* me_,
...
@@ -748,6 +769,7 @@ int raft_recv_entry(raft_server_t* me_,
r
->
idx
=
raft_get_current_idx
(
me_
);
r
->
idx
=
raft_get_current_idx
(
me_
);
r
->
term
=
me
->
current_term
;
r
->
term
=
me
->
current_term
;
/* FIXME: is this required if raft_append_entry does this too? */
if
(
raft_entry_is_voting_cfg_change
(
ety
))
if
(
raft_entry_is_voting_cfg_change
(
ety
))
me
->
voting_cfg_change_log_idx
=
raft_get_current_idx
(
me_
);
me
->
voting_cfg_change_log_idx
=
raft_get_current_idx
(
me_
);
...
@@ -892,11 +914,16 @@ int raft_send_appendentries(raft_server_t* me_, raft_node_t* node)
...
@@ -892,11 +914,16 @@ int raft_send_appendentries(raft_server_t* me_, raft_node_t* node)
if
(
1
<
next_idx
)
if
(
1
<
next_idx
)
{
{
raft_entry_t
*
prev_ety
=
raft_get_entry_from_idx
(
me_
,
next_idx
-
1
);
raft_entry_t
*
prev_ety
=
raft_get_entry_from_idx
(
me_
,
next_idx
-
1
);
ae
.
prev_log_idx
=
next_idx
-
1
;
if
(
!
prev_ety
)
if
(
prev_ety
)
{
ae
.
prev_log_term
=
prev_ety
->
term
;
ae
.
prev_log_idx
=
me
->
snapshot_last_idx
;
else
ae
.
prev_log_term
=
me
->
snapshot_last_term
;
ae
.
prev_log_term
=
me
->
snapshot_last_term
;
}
else
{
ae
.
prev_log_idx
=
next_idx
-
1
;
ae
.
prev_log_term
=
prev_ety
->
term
;
}
}
}
__log
(
me_
,
node
,
"sending appendentries node: ci:%d comi:%d t:%d lc:%d pli:%d plt:%d"
,
__log
(
me_
,
node
,
"sending appendentries node: ci:%d comi:%d t:%d lc:%d pli:%d plt:%d"
,
...
@@ -961,7 +988,12 @@ raft_node_t* raft_add_node(raft_server_t* me_, void* udata, raft_node_id_t id, i
...
@@ -961,7 +988,12 @@ raft_node_t* raft_add_node(raft_server_t* me_, void* udata, raft_node_id_t id, i
if
(
is_self
)
if
(
is_self
)
me
->
node
=
me
->
nodes
[
me
->
num_nodes
-
1
];
me
->
node
=
me
->
nodes
[
me
->
num_nodes
-
1
];
return
me
->
nodes
[
me
->
num_nodes
-
1
];
node
=
me
->
nodes
[
me
->
num_nodes
-
1
];
if
(
me
->
cb
.
notify_membership_event
)
me
->
cb
.
notify_membership_event
(
me_
,
raft_get_udata
(
me_
),
node
,
RAFT_MEMBERSHIP_ADD
);
return
node
;
}
}
raft_node_t
*
raft_add_non_voting_node
(
raft_server_t
*
me_
,
void
*
udata
,
raft_node_id_t
id
,
int
is_self
)
raft_node_t
*
raft_add_non_voting_node
(
raft_server_t
*
me_
,
void
*
udata
,
raft_node_id_t
id
,
int
is_self
)
...
@@ -981,6 +1013,9 @@ void raft_remove_node(raft_server_t* me_, raft_node_t* node)
...
@@ -981,6 +1013,9 @@ void raft_remove_node(raft_server_t* me_, raft_node_t* node)
{
{
raft_server_private_t
*
me
=
(
raft_server_private_t
*
)
me_
;
raft_server_private_t
*
me
=
(
raft_server_private_t
*
)
me_
;
if
(
me
->
cb
.
notify_membership_event
)
me
->
cb
.
notify_membership_event
(
me_
,
raft_get_udata
(
me_
),
node
,
RAFT_MEMBERSHIP_REMOVE
);
assert
(
node
);
assert
(
node
);
int
i
,
found
=
0
;
int
i
,
found
=
0
;
...
@@ -1104,16 +1139,14 @@ void raft_offer_log(raft_server_t* me_, raft_entry_t* ety, const raft_index_t id
...
@@ -1104,16 +1139,14 @@ void raft_offer_log(raft_server_t* me_, raft_entry_t* ety, const raft_index_t id
}
}
else
if
(
!
node
)
else
if
(
!
node
)
{
{
raft_node_t
*
node
=
raft_add_non_voting_node
(
me_
,
NULL
,
node_id
,
is_self
);
node
=
raft_add_non_voting_node
(
me_
,
NULL
,
node_id
,
is_self
);
assert
(
node
);
assert
(
node
);
}
}
}
}
break
;
break
;
case
RAFT_LOGTYPE_ADD_NODE
:
case
RAFT_LOGTYPE_ADD_NODE
:
if
(
!
node
)
{
node
=
raft_add_node
(
me_
,
NULL
,
node_id
,
is_self
);
node
=
raft_add_node
(
me_
,
NULL
,
node_id
,
is_self
);
}
assert
(
node
);
assert
(
node
);
assert
(
raft_node_is_voting
(
node
));
assert
(
raft_node_is_voting
(
node
));
break
;
break
;
...
@@ -1246,7 +1279,7 @@ int raft_begin_snapshot(raft_server_t *me_)
...
@@ -1246,7 +1279,7 @@ int raft_begin_snapshot(raft_server_t *me_)
me
->
snapshot_in_progress
=
1
;
me
->
snapshot_in_progress
=
1
;
__log
(
me_
,
NULL
,
__log
(
me_
,
NULL
,
"begin snapshot sli:%d slt:%d slogs:%d
\n
"
,
"begin snapshot sli:%d slt:%d slogs:%d
\n
"
,
me
->
snapshot_last_idx
,
me
->
snapshot_last_idx
,
me
->
snapshot_last_term
,
me
->
snapshot_last_term
,
raft_get_num_snapshottable_logs
(
me_
));
raft_get_num_snapshottable_logs
(
me_
));
...
@@ -1261,6 +1294,9 @@ int raft_end_snapshot(raft_server_t *me_)
...
@@ -1261,6 +1294,9 @@ int raft_end_snapshot(raft_server_t *me_)
if
(
!
me
->
snapshot_in_progress
||
me
->
snapshot_last_idx
==
0
)
if
(
!
me
->
snapshot_in_progress
||
me
->
snapshot_last_idx
==
0
)
return
-
1
;
return
-
1
;
assert
(
raft_get_num_snapshottable_logs
(
me_
)
!=
0
);
assert
(
me
->
snapshot_last_idx
==
raft_get_commit_idx
(
me_
));
/* If needed, remove compacted logs */
/* If needed, remove compacted logs */
raft_index_t
i
=
log_get_base
(
me
->
log
)
+
1
,
end
=
raft_get_commit_idx
(
me_
);
raft_index_t
i
=
log_get_base
(
me
->
log
)
+
1
,
end
=
raft_get_commit_idx
(
me_
);
for
(;
i
<=
end
;
i
++
)
for
(;
i
<=
end
;
i
++
)
...
@@ -1299,6 +1335,8 @@ int raft_end_snapshot(raft_server_t *me_)
...
@@ -1299,6 +1335,8 @@ int raft_end_snapshot(raft_server_t *me_)
}
}
}
}
assert
(
raft_get_log_count
(
me_
)
==
1
);
return
0
;
return
0
;
}
}
...
@@ -1354,7 +1392,7 @@ int raft_begin_load_snapshot(
...
@@ -1354,7 +1392,7 @@ int raft_begin_load_snapshot(
me
->
num_nodes
=
1
;
me
->
num_nodes
=
1
;
__log
(
me_
,
NULL
,
__log
(
me_
,
NULL
,
"loaded snapshot sli:%d slt:%d slogs:%d
\n
"
,
"loaded snapshot sli:%d slt:%d slogs:%d
\n
"
,
me
->
snapshot_last_idx
,
me
->
snapshot_last_idx
,
me
->
snapshot_last_term
,
me
->
snapshot_last_term
,
raft_get_num_snapshottable_logs
(
me_
));
raft_get_num_snapshottable_logs
(
me_
));
...
...
tests/CuTest.c
View file @
3c134468
...
@@ -151,7 +151,7 @@ void CuFail_Line(CuTest* tc, const char* file, int line, const char* message2, c
...
@@ -151,7 +151,7 @@ void CuFail_Line(CuTest* tc, const char* file, int line, const char* message2, c
CuString
string
;
CuString
string
;
CuStringInit
(
&
string
);
CuStringInit
(
&
string
);
if
(
message2
!=
NULL
)
if
(
message2
!=
NULL
)
{
{
CuStringAppend
(
&
string
,
message2
);
CuStringAppend
(
&
string
,
message2
);
CuStringAppend
(
&
string
,
": "
);
CuStringAppend
(
&
string
,
": "
);
...
@@ -166,7 +166,7 @@ void CuAssert_Line(CuTest* tc, const char* file, int line, const char* message,
...
@@ -166,7 +166,7 @@ void CuAssert_Line(CuTest* tc, const char* file, int line, const char* message,
CuFail_Line
(
tc
,
file
,
line
,
NULL
,
message
);
CuFail_Line
(
tc
,
file
,
line
,
NULL
,
message
);
}
}
void
CuAssertStrEquals_LineMsg
(
CuTest
*
tc
,
const
char
*
file
,
int
line
,
const
char
*
message
,
void
CuAssertStrEquals_LineMsg
(
CuTest
*
tc
,
const
char
*
file
,
int
line
,
const
char
*
message
,
const
char
*
expected
,
const
char
*
actual
)
const
char
*
expected
,
const
char
*
actual
)
{
{
CuString
string
;
CuString
string
;
...
@@ -178,7 +178,7 @@ void CuAssertStrEquals_LineMsg(CuTest* tc, const char* file, int line, const cha
...
@@ -178,7 +178,7 @@ void CuAssertStrEquals_LineMsg(CuTest* tc, const char* file, int line, const cha
}
}
CuStringInit
(
&
string
);
CuStringInit
(
&
string
);
if
(
message
!=
NULL
)
if
(
message
!=
NULL
)
{
{
CuStringAppend
(
&
string
,
message
);
CuStringAppend
(
&
string
,
message
);
CuStringAppend
(
&
string
,
": "
);
CuStringAppend
(
&
string
,
": "
);
...
@@ -191,7 +191,7 @@ void CuAssertStrEquals_LineMsg(CuTest* tc, const char* file, int line, const cha
...
@@ -191,7 +191,7 @@ void CuAssertStrEquals_LineMsg(CuTest* tc, const char* file, int line, const cha
CuFailInternal
(
tc
,
file
,
line
,
&
string
);
CuFailInternal
(
tc
,
file
,
line
,
&
string
);
}
}
void
CuAssertIntEquals_LineMsg
(
CuTest
*
tc
,
const
char
*
file
,
int
line
,
const
char
*
message
,
void
CuAssertIntEquals_LineMsg
(
CuTest
*
tc
,
const
char
*
file
,
int
line
,
const
char
*
message
,
int
expected
,
int
actual
)
int
expected
,
int
actual
)
{
{
char
buf
[
STRING_MAX
];
char
buf
[
STRING_MAX
];
...
@@ -200,7 +200,7 @@ void CuAssertIntEquals_LineMsg(CuTest* tc, const char* file, int line, const cha
...
@@ -200,7 +200,7 @@ void CuAssertIntEquals_LineMsg(CuTest* tc, const char* file, int line, const cha
CuFail_Line
(
tc
,
file
,
line
,
message
,
buf
);
CuFail_Line
(
tc
,
file
,
line
,
message
,
buf
);
}
}
void
CuAssertDblEquals_LineMsg
(
CuTest
*
tc
,
const
char
*
file
,
int
line
,
const
char
*
message
,
void
CuAssertDblEquals_LineMsg
(
CuTest
*
tc
,
const
char
*
file
,
int
line
,
const
char
*
message
,
double
expected
,
double
actual
,
double
delta
)
double
expected
,
double
actual
,
double
delta
)
{
{
char
buf
[
STRING_MAX
];
char
buf
[
STRING_MAX
];
...
@@ -209,7 +209,7 @@ void CuAssertDblEquals_LineMsg(CuTest* tc, const char* file, int line, const cha
...
@@ -209,7 +209,7 @@ void CuAssertDblEquals_LineMsg(CuTest* tc, const char* file, int line, const cha
CuFail_Line
(
tc
,
file
,
line
,
message
,
buf
);
CuFail_Line
(
tc
,
file
,
line
,
message
,
buf
);
}
}
void
CuAssertPtrEquals_LineMsg
(
CuTest
*
tc
,
const
char
*
file
,
int
line
,
const
char
*
message
,
void
CuAssertPtrEquals_LineMsg
(
CuTest
*
tc
,
const
char
*
file
,
int
line
,
const
char
*
message
,
void
*
expected
,
void
*
actual
)
void
*
expected
,
void
*
actual
)
{
{
char
buf
[
STRING_MAX
];
char
buf
[
STRING_MAX
];
...
...
tests/CuTest.h
View file @
3c134468
...
@@ -54,17 +54,17 @@ void CuTestRun(CuTest* tc);
...
@@ -54,17 +54,17 @@ void CuTestRun(CuTest* tc);
/* Internal versions of assert functions -- use the public versions */
/* Internal versions of assert functions -- use the public versions */
void
CuFail_Line
(
CuTest
*
tc
,
const
char
*
file
,
int
line
,
const
char
*
message2
,
const
char
*
message
);
void
CuFail_Line
(
CuTest
*
tc
,
const
char
*
file
,
int
line
,
const
char
*
message2
,
const
char
*
message
);
void
CuAssert_Line
(
CuTest
*
tc
,
const
char
*
file
,
int
line
,
const
char
*
message
,
int
condition
);
void
CuAssert_Line
(
CuTest
*
tc
,
const
char
*
file
,
int
line
,
const
char
*
message
,
int
condition
);
void
CuAssertStrEquals_LineMsg
(
CuTest
*
tc
,
void
CuAssertStrEquals_LineMsg
(
CuTest
*
tc
,
const
char
*
file
,
int
line
,
const
char
*
message
,
const
char
*
file
,
int
line
,
const
char
*
message
,
const
char
*
expected
,
const
char
*
actual
);
const
char
*
expected
,
const
char
*
actual
);
void
CuAssertIntEquals_LineMsg
(
CuTest
*
tc
,
void
CuAssertIntEquals_LineMsg
(
CuTest
*
tc
,
const
char
*
file
,
int
line
,
const
char
*
message
,
const
char
*
file
,
int
line
,
const
char
*
message
,
int
expected
,
int
actual
);
int
expected
,
int
actual
);
void
CuAssertDblEquals_LineMsg
(
CuTest
*
tc
,
void
CuAssertDblEquals_LineMsg
(
CuTest
*
tc
,
const
char
*
file
,
int
line
,
const
char
*
message
,
const
char
*
file
,
int
line
,
const
char
*
message
,
double
expected
,
double
actual
,
double
delta
);
double
expected
,
double
actual
,
double
delta
);
void
CuAssertPtrEquals_LineMsg
(
CuTest
*
tc
,
void
CuAssertPtrEquals_LineMsg
(
CuTest
*
tc
,
const
char
*
file
,
int
line
,
const
char
*
message
,
const
char
*
file
,
int
line
,
const
char
*
message
,
void
*
expected
,
void
*
actual
);
void
*
expected
,
void
*
actual
);
/* public assert functions */
/* public assert functions */
...
...
tests/raft_cffi.py
0 → 100644
View file @
3c134468
import
cffi
import
subprocess
ffi
=
cffi
.
FFI
()
ffi
.
set_source
(
"tests"
,
"""
"""
,
sources
=
"""
src/raft_log.c
src/raft_server.c
src/raft_server_properties.c
src/raft_node.c
"""
.
split
(),
include_dirs
=
[
"include"
],
)
library
=
ffi
.
compile
()
ffi
=
cffi
.
FFI
()
lib
=
ffi
.
dlopen
(
library
)
def
load
(
fname
):
return
'
\n
'
.
join
(
[
line
for
line
in
subprocess
.
check_output
(
[
"gcc"
,
"-E"
,
fname
]).
decode
(
'utf-8'
).
split
(
'
\n
'
)])
ffi
.
cdef
(
'void *malloc(size_t __size);'
)
ffi
.
cdef
(
load
(
'include/raft.h'
))
ffi
.
cdef
(
load
(
'include/raft_log.h'
))
tests/test_log.c
View file @
3c134468
...
@@ -399,10 +399,7 @@ void TestLog_load_from_snapshot(CuTest * tc)
...
@@ -399,10 +399,7 @@ void TestLog_load_from_snapshot(CuTest * tc)
CuAssertIntEquals
(
tc
,
0
,
log_get_current_idx
(
l
));
CuAssertIntEquals
(
tc
,
0
,
log_get_current_idx
(
l
));
CuAssertIntEquals
(
tc
,
0
,
log_load_from_snapshot
(
l
,
10
,
5
));
CuAssertIntEquals
(
tc
,
0
,
log_load_from_snapshot
(
l
,
10
,
5
));
CuAssertIntEquals
(
tc
,
10
,
log_get_current_idx
(
l
));
CuAssertIntEquals
(
tc
,
10
,
log_get_current_idx
(
l
));
CuAssertIntEquals
(
tc
,
0
,
log_count
(
l
));
/* this is just a marker
* it should never be sent to any nodes because it is part of a snapshot */
CuAssertIntEquals
(
tc
,
1
,
log_count
(
l
));
}
}
void
TestLog_load_from_snapshot_clears_log
(
CuTest
*
tc
)
void
TestLog_load_from_snapshot_clears_log
(
CuTest
*
tc
)
...
@@ -422,7 +419,7 @@ void TestLog_load_from_snapshot_clears_log(CuTest * tc)
...
@@ -422,7 +419,7 @@ void TestLog_load_from_snapshot_clears_log(CuTest * tc)
CuAssertIntEquals
(
tc
,
2
,
log_get_current_idx
(
l
));
CuAssertIntEquals
(
tc
,
2
,
log_get_current_idx
(
l
));
CuAssertIntEquals
(
tc
,
0
,
log_load_from_snapshot
(
l
,
10
,
5
));
CuAssertIntEquals
(
tc
,
0
,
log_load_from_snapshot
(
l
,
10
,
5
));
CuAssertIntEquals
(
tc
,
1
,
log_count
(
l
));
CuAssertIntEquals
(
tc
,
0
,
log_count
(
l
));
CuAssertIntEquals
(
tc
,
10
,
log_get_current_idx
(
l
));
CuAssertIntEquals
(
tc
,
10
,
log_get_current_idx
(
l
));
}
}
...
...
tests/test_server.c
View file @
3c134468
...
@@ -82,14 +82,6 @@ static int __raft_log_offer(raft_server_t* raft,
...
@@ -82,14 +82,6 @@ static int __raft_log_offer(raft_server_t* raft,
raft_entry_t
*
entry
,
raft_entry_t
*
entry
,
raft_index_t
entry_idx
)
raft_index_t
entry_idx
)
{
{
switch
(
entry
->
type
)
{
case
RAFT_LOGTYPE_ADD_NONVOTING_NODE
:
raft_add_non_voting_node
(
raft
,
NULL
,
atoi
(
entry
->
data
.
buf
),
0
);
break
;
case
RAFT_LOGTYPE_ADD_NODE
:
raft_add_node
(
raft
,
NULL
,
atoi
(
entry
->
data
.
buf
),
0
);
break
;
}
return
0
;
return
0
;
}
}
...
@@ -1471,7 +1463,7 @@ void TestRaft_follower_recv_appendentries_delete_entries_if_conflict_with_new_en
...
@@ -1471,7 +1463,7 @@ void TestRaft_follower_recv_appendentries_delete_entries_if_conflict_with_new_en
char
*
strs
[]
=
{
"111"
,
"222"
,
"333"
};
char
*
strs
[]
=
{
"111"
,
"222"
,
"333"
};
raft_entry_t
*
ety_appended
;
raft_entry_t
*
ety_appended
;
__create_mock_entries_for_conflict_tests
(
tc
,
r
,
strs
);
__create_mock_entries_for_conflict_tests
(
tc
,
r
,
strs
);
CuAssertIntEquals
(
tc
,
3
,
raft_get_log_count
(
r
));
CuAssertIntEquals
(
tc
,
3
,
raft_get_log_count
(
r
));
...
@@ -1967,7 +1959,7 @@ void TestRaft_follower_recv_appendentries_heartbeat_does_not_overwrite_logs(
...
@@ -1967,7 +1959,7 @@ void TestRaft_follower_recv_appendentries_heartbeat_does_not_overwrite_logs(
raft_recv_appendentries
(
r
,
raft_get_node
(
r
,
2
),
&
ae
,
&
aer
);
raft_recv_appendentries
(
r
,
raft_get_node
(
r
,
2
),
&
ae
,
&
aer
);
/* receive a heartbeat
/* receive a heartbeat
* NOTE: the leader hasn't received the response to the last AE so it can
* NOTE: the leader hasn't received the response to the last AE so it can
* only assume prev_Log_idx is still 1 */
* only assume prev_Log_idx is still 1 */
memset
(
&
ae
,
0
,
sizeof
(
msg_appendentries_t
));
memset
(
&
ae
,
0
,
sizeof
(
msg_appendentries_t
));
ae
.
term
=
1
;
ae
.
term
=
1
;
...
@@ -3963,10 +3955,10 @@ void TestRaft_leader_recv_appendentries_response_set_has_sufficient_logs_after_v
...
@@ -3963,10 +3955,10 @@ void TestRaft_leader_recv_appendentries_response_set_has_sufficient_logs_after_v
.
type
=
RAFT_LOGTYPE_ADD_NONVOTING_NODE
.
type
=
RAFT_LOGTYPE_ADD_NONVOTING_NODE
};
};
msg_entry_response_t
etyr
;
msg_entry_response_t
etyr
;
raft_recv_entry
(
r
,
&
ety
,
&
etyr
);
CuAssertIntEquals
(
tc
,
0
,
raft_recv_entry
(
r
,
&
ety
,
&
etyr
)
)
;
ety
.
id
++
;
ety
.
id
++
;
ety
.
data
.
buf
=
"3"
;
ety
.
data
.
buf
=
"3"
;
raft_recv_entry
(
r
,
&
ety
,
&
etyr
);
CuAssertIntEquals
(
tc
,
0
,
raft_recv_entry
(
r
,
&
ety
,
&
etyr
)
)
;
msg_appendentries_response_t
aer
=
{
msg_appendentries_response_t
aer
=
{
.
term
=
1
,
.
success
=
1
,
.
current_idx
=
2
,
.
first_idx
=
0
.
term
=
1
,
.
success
=
1
,
.
current_idx
=
2
,
.
first_idx
=
0
...
@@ -3999,4 +3991,3 @@ void TestRaft_leader_recv_appendentries_response_set_has_sufficient_logs_after_v
...
@@ -3999,4 +3991,3 @@ void TestRaft_leader_recv_appendentries_response_set_has_sufficient_logs_after_v
raft_recv_appendentries_response
(
r
,
raft_get_node
(
r
,
2
),
&
aer
);
raft_recv_appendentries_response
(
r
,
raft_get_node
(
r
,
2
),
&
aer
);
CuAssertIntEquals
(
tc
,
2
,
has_sufficient_logs_flag
);
CuAssertIntEquals
(
tc
,
2
,
has_sufficient_logs_flag
);
}
}
tests/test_snapshotting.c
View file @
3c134468
...
@@ -352,7 +352,7 @@ void TestRaft_follower_load_from_snapshot(CuTest * tc)
...
@@ -352,7 +352,7 @@ void TestRaft_follower_load_from_snapshot(CuTest * tc)
CuAssertIntEquals
(
tc
,
0
,
raft_get_log_count
(
r
));
CuAssertIntEquals
(
tc
,
0
,
raft_get_log_count
(
r
));
CuAssertIntEquals
(
tc
,
0
,
raft_begin_load_snapshot
(
r
,
5
,
5
));
CuAssertIntEquals
(
tc
,
0
,
raft_begin_load_snapshot
(
r
,
5
,
5
));
CuAssertIntEquals
(
tc
,
0
,
raft_end_load_snapshot
(
r
));
CuAssertIntEquals
(
tc
,
0
,
raft_end_load_snapshot
(
r
));
CuAssertIntEquals
(
tc
,
1
,
raft_get_log_count
(
r
));
CuAssertIntEquals
(
tc
,
0
,
raft_get_log_count
(
r
));
CuAssertIntEquals
(
tc
,
0
,
raft_get_num_snapshottable_logs
(
r
));
CuAssertIntEquals
(
tc
,
0
,
raft_get_num_snapshottable_logs
(
r
));
CuAssertIntEquals
(
tc
,
5
,
raft_get_commit_idx
(
r
));
CuAssertIntEquals
(
tc
,
5
,
raft_get_commit_idx
(
r
));
CuAssertIntEquals
(
tc
,
5
,
raft_get_last_applied_idx
(
r
));
CuAssertIntEquals
(
tc
,
5
,
raft_get_last_applied_idx
(
r
));
...
@@ -408,7 +408,7 @@ void TestRaft_follower_load_from_snapshot_fails_if_already_loaded(CuTest * tc)
...
@@ -408,7 +408,7 @@ void TestRaft_follower_load_from_snapshot_fails_if_already_loaded(CuTest * tc)
CuAssertIntEquals
(
tc
,
0
,
raft_get_log_count
(
r
));
CuAssertIntEquals
(
tc
,
0
,
raft_get_log_count
(
r
));
CuAssertIntEquals
(
tc
,
0
,
raft_begin_load_snapshot
(
r
,
5
,
5
));
CuAssertIntEquals
(
tc
,
0
,
raft_begin_load_snapshot
(
r
,
5
,
5
));
CuAssertIntEquals
(
tc
,
0
,
raft_end_load_snapshot
(
r
));
CuAssertIntEquals
(
tc
,
0
,
raft_end_load_snapshot
(
r
));
CuAssertIntEquals
(
tc
,
1
,
raft_get_log_count
(
r
));
CuAssertIntEquals
(
tc
,
0
,
raft_get_log_count
(
r
));
CuAssertIntEquals
(
tc
,
0
,
raft_get_num_snapshottable_logs
(
r
));
CuAssertIntEquals
(
tc
,
0
,
raft_get_num_snapshottable_logs
(
r
));
CuAssertIntEquals
(
tc
,
5
,
raft_get_commit_idx
(
r
));
CuAssertIntEquals
(
tc
,
5
,
raft_get_commit_idx
(
r
));
CuAssertIntEquals
(
tc
,
5
,
raft_get_last_applied_idx
(
r
));
CuAssertIntEquals
(
tc
,
5
,
raft_get_last_applied_idx
(
r
));
...
@@ -527,7 +527,7 @@ void TestRaft_leader_sends_appendentries_when_node_next_index_was_compacted(CuTe
...
@@ -527,7 +527,7 @@ void TestRaft_leader_sends_appendentries_when_node_next_index_was_compacted(CuTe
raft_set_current_term
(
r
,
2
);
raft_set_current_term
(
r
,
2
);
CuAssertIntEquals
(
tc
,
0
,
raft_send_appendentries
(
r
,
node
));
CuAssertIntEquals
(
tc
,
0
,
raft_send_appendentries
(
r
,
node
));
CuAssertIntEquals
(
tc
,
2
,
ae
.
term
);
CuAssertIntEquals
(
tc
,
2
,
ae
.
term
);
CuAssertIntEquals
(
tc
,
2
,
ae
.
prev_log_idx
);
CuAssertIntEquals
(
tc
,
3
,
ae
.
prev_log_idx
);
CuAssertIntEquals
(
tc
,
2
,
ae
.
prev_log_term
);
CuAssertIntEquals
(
tc
,
2
,
ae
.
prev_log_term
);
}
}
...
@@ -568,3 +568,69 @@ void TestRaft_recv_entry_fails_if_snapshot_in_progress(CuTest* tc)
...
@@ -568,3 +568,69 @@ void TestRaft_recv_entry_fails_if_snapshot_in_progress(CuTest* tc)
ety
.
type
=
RAFT_LOGTYPE_ADD_NODE
;
ety
.
type
=
RAFT_LOGTYPE_ADD_NODE
;
CuAssertIntEquals
(
tc
,
RAFT_ERR_SNAPSHOT_IN_PROGRESS
,
raft_recv_entry
(
r
,
&
ety
,
&
cr
));
CuAssertIntEquals
(
tc
,
RAFT_ERR_SNAPSHOT_IN_PROGRESS
,
raft_recv_entry
(
r
,
&
ety
,
&
cr
));
}
}
void
TestRaft_follower_recv_appendentries_is_successful_when_previous_log_idx_equals_snapshot_last_idx
(
CuTest
*
tc
)
{
raft_cbs_t
funcs
=
{
.
persist_term
=
__raft_persist_term
,
};
void
*
r
=
raft_new
();
raft_set_callbacks
(
r
,
&
funcs
,
NULL
);
raft_add_node
(
r
,
NULL
,
1
,
1
);
raft_add_node
(
r
,
NULL
,
2
,
0
);
CuAssertIntEquals
(
tc
,
0
,
raft_begin_load_snapshot
(
r
,
2
,
2
));
CuAssertIntEquals
(
tc
,
0
,
raft_end_load_snapshot
(
r
));
msg_appendentries_t
ae
;
msg_appendentries_response_t
aer
;
memset
(
&
ae
,
0
,
sizeof
(
msg_appendentries_t
));
ae
.
term
=
3
;
ae
.
prev_log_idx
=
2
;
ae
.
prev_log_term
=
2
;
/* include entries */
msg_entry_t
e
[
5
];
memset
(
&
e
,
0
,
sizeof
(
msg_entry_t
)
*
4
);
e
[
0
].
term
=
3
;
e
[
0
].
id
=
3
;
ae
.
entries
=
e
;
ae
.
n_entries
=
1
;
CuAssertIntEquals
(
tc
,
0
,
raft_recv_appendentries
(
r
,
raft_get_node
(
r
,
2
),
&
ae
,
&
aer
));
CuAssertIntEquals
(
tc
,
1
,
aer
.
success
);
}
void
TestRaft_leader_sends_appendentries_with_correct_prev_log_idx_when_snapshotted
(
CuTest
*
tc
)
{
raft_cbs_t
funcs
=
{
.
send_appendentries
=
sender_appendentries
,
.
log
=
NULL
};
void
*
sender
=
sender_new
(
NULL
);
void
*
r
=
raft_new
();
raft_set_callbacks
(
r
,
&
funcs
,
sender
);
CuAssertTrue
(
tc
,
NULL
!=
raft_add_node
(
r
,
NULL
,
1
,
1
));
CuAssertTrue
(
tc
,
NULL
!=
raft_add_node
(
r
,
NULL
,
2
,
0
));
CuAssertIntEquals
(
tc
,
0
,
raft_begin_load_snapshot
(
r
,
2
,
4
));
CuAssertIntEquals
(
tc
,
0
,
raft_end_load_snapshot
(
r
));
/* i'm leader */
raft_set_state
(
r
,
RAFT_STATE_LEADER
);
raft_node_t
*
p
=
raft_get_node_from_idx
(
r
,
1
);
CuAssertTrue
(
tc
,
NULL
!=
p
);
raft_node_set_next_idx
(
p
,
4
);
/* receive appendentries messages */
raft_send_appendentries
(
r
,
p
);
msg_appendentries_t
*
ae
=
sender_poll_msg_data
(
sender
);
CuAssertTrue
(
tc
,
NULL
!=
ae
);
CuAssertIntEquals
(
tc
,
2
,
ae
->
prev_log_term
);
CuAssertIntEquals
(
tc
,
4
,
ae
->
prev_log_idx
);
}
tests/virtraft2.py
0 → 100644
View file @
3c134468
#!/usr/bin/env python
"""
virtraft2 - Simulate a raft network
Some quality checks we do:
- Log Matching (servers must have matching logs)
- State Machine Safety (applied entries have the same ID)
- Election Safety (only one valid leader per term)
- Current Index Validity (does current index have an existing entry?)
- Entry ID Monotonicity (entries aren't appended out of order)
- Committed entry popping (committed entries are not popped from the log)
- Log Accuracy (does the server's log match mirror an independent log?)
- Deadlock detection (does the cluster continuously make progress?)
Some chaos we generate:
- Random bi-directional partitions between nodes
- Message dropping
- Message duplication
- Message re-ordering
Usage:
virtraft --servers SERVERS [-d RATE] [-D RATE] [-c RATE] [-C RATE] [-m RATE]
[-P RATE] [-s SEED] [-i ITERS] [-p] [--tsv]
[-q] [-v] [-l LEVEL]
virtraft --version
virtraft --help
Options:
-n --servers SERVERS Number of servers
-d --drop_rate RATE Message drop rate 0-100 [default: 0]
-D --dupe_rate RATE Message duplication rate 0-100 [default: 0]
-c --client_rate RATE Rate entries are received from the client 0-100
[default: 100]
-C --compaction_rate RATE Rate that log compactions occur 0-100 [default: 0]
-m --member_rate RATE Membership change rate 0-100000 [default: 0]
-P --partition_rate RATE Rate that partitions occur or are healed 0-100
[default: 0]
-p --no_random_period Don't use a random period
-s --seed SEED The simulation's seed [default: 0]
-q --quiet No output at end of run
-i --iterations ITERS Number of iterations before the simulation ends
[default: 1000]
--tsv Output node status tab separated values at exit
-v --verbose Show debug logs
-l --log_level LEVEL Set log level
-V --version Display version.
-h --help Prints a short usage summary.
Examples:
Output a node status table:
virtraft --servers 3 --iterations 1000 --tsv | column -t
"""
import
collections
import
colorama
import
coloredlogs
import
docopt
import
logging
import
random
import
re
import
sys
import
terminaltables
from
raft_cffi
import
ffi
,
lib
NODE_DISCONNECTED
=
0
NODE_CONNECTING
=
1
NODE_CONNECTED
=
2
NODE_DISCONNECTING
=
3
class
ServerDoesNotExist
(
Exception
):
pass
def
logtype2str
(
log_type
):
if
log_type
==
lib
.
RAFT_LOGTYPE_NORMAL
:
return
'normal'
elif
log_type
==
lib
.
RAFT_LOGTYPE_DEMOTE_NODE
:
return
'demote'
elif
log_type
==
lib
.
RAFT_LOGTYPE_REMOVE_NODE
:
return
'remove'
elif
log_type
==
lib
.
RAFT_LOGTYPE_ADD_NONVOTING_NODE
:
return
'add_nonvoting'
elif
log_type
==
lib
.
RAFT_LOGTYPE_ADD_NODE
:
return
'add'
else
:
return
'unknown'
def
state2str
(
state
):
if
state
==
lib
.
RAFT_STATE_LEADER
:
return
colorama
.
Fore
.
GREEN
+
'leader'
+
colorama
.
Style
.
RESET_ALL
elif
state
==
lib
.
RAFT_STATE_CANDIDATE
:
return
'candidate'
elif
state
==
lib
.
RAFT_STATE_FOLLOWER
:
return
'follower'
else
:
return
'unknown'
def
connectstatus2str
(
connectstatus
):
return
{
NODE_DISCONNECTED
:
colorama
.
Fore
.
RED
+
'DISCONNECTED'
+
colorama
.
Style
.
RESET_ALL
,
NODE_CONNECTING
:
'CONNECTING'
,
NODE_CONNECTED
:
colorama
.
Fore
.
GREEN
+
'CONNECTED'
+
colorama
.
Style
.
RESET_ALL
,
NODE_DISCONNECTING
:
colorama
.
Fore
.
YELLOW
+
'DISCONNECTING'
+
colorama
.
Style
.
RESET_ALL
,
}[
connectstatus
]
def
err2str
(
err
):
return
{
lib
.
RAFT_ERR_NOT_LEADER
:
'RAFT_ERR_NOT_LEADER'
,
lib
.
RAFT_ERR_ONE_VOTING_CHANGE_ONLY
:
'RAFT_ERR_ONE_VOTING_CHANGE_ONLY'
,
lib
.
RAFT_ERR_SHUTDOWN
:
'RAFT_ERR_SHUTDOWN'
,
lib
.
RAFT_ERR_NOMEM
:
'RAFT_ERR_NOMEM'
,
lib
.
RAFT_ERR_NEEDS_SNAPSHOT
:
'RAFT_ERR_NEEDS_SNAPSHOT'
,
lib
.
RAFT_ERR_SNAPSHOT_IN_PROGRESS
:
'RAFT_ERR_SNAPSHOT_IN_PROGRESS'
,
lib
.
RAFT_ERR_SNAPSHOT_ALREADY_LOADED
:
'RAFT_ERR_SNAPSHOT_ALREADY_LOADED'
,
lib
.
RAFT_ERR_LAST
:
'RAFT_ERR_LAST'
,
}[
err
]
class
ChangeRaftEntry
(
object
):
def
__init__
(
self
,
node_id
):
self
.
node_id
=
node_id
class
SetRaftEntry
(
object
):
def
__init__
(
self
,
key
,
val
):
self
.
key
=
key
self
.
val
=
val
class
RaftEntry
(
object
):
def
__init__
(
self
,
entry
):
self
.
term
=
entry
.
term
self
.
id
=
entry
.
id
class
Snapshot
(
object
):
def
__init__
(
self
):
self
.
members
=
[]
SnapshotMember
=
collections
.
namedtuple
(
'SnapshotMember'
,
[
'id'
,
'voting'
],
verbose
=
False
)
def
raft_send_requestvote
(
raft
,
udata
,
node
,
msg
):
server
=
ffi
.
from_handle
(
udata
)
dst_server
=
ffi
.
from_handle
(
lib
.
raft_node_get_udata
(
node
))
server
.
network
.
enqueue_msg
(
msg
,
server
,
dst_server
)
return
0
def
raft_send_appendentries
(
raft
,
udata
,
node
,
msg
):
server
=
ffi
.
from_handle
(
udata
)
assert
node
dst_server
=
ffi
.
from_handle
(
lib
.
raft_node_get_udata
(
node
))
server
.
network
.
enqueue_msg
(
msg
,
server
,
dst_server
)
# Collect statistics
if
server
.
network
.
max_entries_in_ae
<
msg
.
n_entries
:
server
.
network
.
max_entries_in_ae
=
msg
.
n_entries
return
0
def
raft_send_snapshot
(
raft
,
udata
,
node
):
return
ffi
.
from_handle
(
udata
).
send_snapshot
(
node
)
def
raft_applylog
(
raft
,
udata
,
ety
,
idx
):
try
:
return
ffi
.
from_handle
(
udata
).
entry_apply
(
ety
,
idx
)
except
:
return
lib
.
RAFT_ERR_SHUTDOWN
def
raft_persist_vote
(
raft
,
udata
,
voted_for
):
return
ffi
.
from_handle
(
udata
).
persist_vote
(
voted_for
)
def
raft_persist_term
(
raft
,
udata
,
term
,
vote
):
return
ffi
.
from_handle
(
udata
).
persist_term
(
term
,
vote
)
def
raft_logentry_offer
(
raft
,
udata
,
ety
,
ety_idx
):
return
ffi
.
from_handle
(
udata
).
entry_append
(
ety
,
ety_idx
)
def
raft_logentry_poll
(
raft
,
udata
,
ety
,
ety_idx
):
return
ffi
.
from_handle
(
udata
).
entry_poll
(
ety
,
ety_idx
)
def
raft_logentry_pop
(
raft
,
udata
,
ety
,
ety_idx
):
return
ffi
.
from_handle
(
udata
).
entry_pop
(
ety
,
ety_idx
)
def
raft_logentry_get_node_id
(
raft
,
udata
,
ety
,
ety_idx
):
change_entry
=
ffi
.
from_handle
(
ety
.
data
.
buf
)
assert
isinstance
(
change_entry
,
ChangeRaftEntry
)
return
change_entry
.
node_id
def
raft_node_has_sufficient_logs
(
raft
,
udata
,
node
):
return
ffi
.
from_handle
(
udata
).
node_has_sufficient_entries
(
node
)
def
raft_notify_membership_event
(
raft
,
udata
,
node
,
event_type
):
return
ffi
.
from_handle
(
udata
).
notify_membership_event
(
node
,
event_type
)
def
raft_log
(
raft
,
node
,
udata
,
buf
):
server
=
ffi
.
from_handle
(
lib
.
raft_get_udata
(
raft
))
# print(server.id, ffi.string(buf).decode('utf8'))
if
node
!=
ffi
.
NULL
:
node
=
ffi
.
from_handle
(
lib
.
raft_node_get_udata
(
node
))
# if server.id in [1] or (node and node.id in [1]):
logging
.
info
(
'{0}> {1}:{2}: {3}'
.
format
(
server
.
network
.
iteration
,
server
.
id
,
node
.
id
if
node
else
''
,
ffi
.
string
(
buf
).
decode
(
'utf8'
),
))
class
Message
(
object
):
def
__init__
(
self
,
msg
,
sendor
,
sendee
):
self
.
data
=
msg
self
.
sendor
=
sendor
self
.
sendee
=
sendee
class
Network
(
object
):
def
__init__
(
self
,
seed
=
0
):
self
.
servers
=
[]
self
.
messages
=
[]
self
.
drop_rate
=
0
self
.
dupe_rate
=
0
self
.
partition_rate
=
0
self
.
iteration
=
0
self
.
leader
=
None
self
.
ety_id
=
0
self
.
entries
=
[]
self
.
random
=
random
.
Random
(
seed
)
self
.
partitions
=
[]
self
.
no_random_period
=
False
self
.
server_id
=
0
# Information
self
.
max_entries_in_ae
=
0
self
.
leadership_changes
=
0
self
.
log_pops
=
0
self
.
num_unique_nodes
=
0
self
.
num_membership_changes
=
0
self
.
num_compactions
=
0
self
.
latest_applied_log_idx
=
0
def
add_server
(
self
,
server
):
self
.
server_id
+=
1
server
.
id
=
self
.
server_id
assert
server
.
id
not
in
set
([
s
.
id
for
s
in
self
.
servers
])
server
.
set_network
(
self
)
self
.
servers
.
append
(
server
)
def
push_set_entry
(
self
,
k
,
v
):
for
sv
in
self
.
active_servers
:
if
lib
.
raft_is_leader
(
sv
.
raft
):
ety
=
ffi
.
new
(
'msg_entry_t*'
)
ety
.
term
=
0
ety
.
id
=
self
.
new_entry_id
()
ety
.
type
=
lib
.
RAFT_LOGTYPE_NORMAL
change
=
ffi
.
new_handle
(
SetRaftEntry
(
k
,
v
))
ety
.
data
.
buf
=
change
ety
.
data
.
len
=
ffi
.
sizeof
(
ety
.
data
.
buf
)
self
.
entries
.
append
((
ety
,
change
))
e
=
sv
.
recv_entry
(
ety
)
assert
e
==
0
break
def
id2server
(
self
,
id
):
for
server
in
self
.
servers
:
if
server
.
id
==
id
:
return
server
# if lib.raft_get_nodeid(server.raft) == id:
# return server
raise
ServerDoesNotExist
(
'Could not find server: {}'
.
format
(
id
))
def
add_partition
(
self
):
if
len
(
self
.
active_servers
)
<=
1
:
return
nodes
=
list
(
self
.
active_servers
)
self
.
random
.
shuffle
(
nodes
)
n1
=
nodes
.
pop
()
n2
=
nodes
.
pop
()
self
.
partitions
.
append
((
n1
,
n2
))
def
remove_partition
(
self
):
self
.
random
.
shuffle
(
self
.
partitions
)
self
.
partitions
.
pop
()
def
periodic
(
self
):
if
self
.
random
.
randint
(
1
,
100
)
<
self
.
member_rate
:
if
20
<
self
.
random
.
randint
(
1
,
100
):
self
.
add_member
()
else
:
self
.
remove_member
()
if
self
.
random
.
randint
(
1
,
100
)
<
self
.
partition_rate
:
self
.
add_partition
()
if
self
.
partitions
and
self
.
random
.
randint
(
1
,
100
)
<
self
.
partition_rate
:
self
.
remove_partition
()
if
self
.
random
.
randint
(
1
,
100
)
<
self
.
client_rate
:
self
.
push_set_entry
(
self
.
random
.
randint
(
1
,
10
),
self
.
random
.
randint
(
1
,
10
))
for
server
in
self
.
active_servers
:
if
self
.
no_random_period
:
server
.
periodic
(
100
)
else
:
server
.
periodic
(
self
.
random
.
randint
(
1
,
100
))
# Deadlock detection
if
self
.
latest_applied_log_idx
!=
0
and
self
.
latest_applied_log_iteration
+
5000
<
self
.
iteration
:
logging
.
error
(
"deadlock detected iteration:{0} appliedidx:{1}
\n
"
.
format
(
self
.
latest_applied_log_iteration
,
self
.
latest_applied_log_idx
,
))
self
.
diagnotistic_info
()
sys
.
exit
(
1
)
# Count leadership changes
leader_node
=
lib
.
raft_get_current_leader_node
(
self
.
active_servers
[
0
].
raft
)
if
leader_node
:
leader
=
ffi
.
from_handle
(
lib
.
raft_node_get_udata
(
leader_node
))
if
self
.
leader
is
not
leader
:
self
.
leadership_changes
+=
1
self
.
leader
=
leader
def
enqueue_msg
(
self
,
msg
,
sendor
,
sendee
):
# Drop message if this edge is partitioned
for
partition
in
self
.
partitions
:
# Partitions are in one direction
if
partition
[
0
]
is
sendor
and
partition
[
1
]
is
sendee
:
return
if
self
.
random
.
randint
(
1
,
100
)
<
self
.
drop_rate
:
return
while
self
.
random
.
randint
(
1
,
100
)
<
self
.
dupe_rate
:
self
.
_enqueue_msg
(
msg
,
sendor
,
sendee
)
self
.
_enqueue_msg
(
msg
,
sendor
,
sendee
)
def
_enqueue_msg
(
self
,
msg
,
sendor
,
sendee
):
msg_type
=
ffi
.
getctype
(
ffi
.
typeof
(
msg
))
msg_size
=
ffi
.
sizeof
(
msg
[
0
])
new_msg
=
ffi
.
cast
(
ffi
.
typeof
(
msg
),
lib
.
malloc
(
msg_size
))
ffi
.
memmove
(
new_msg
,
msg
,
msg_size
)
if
msg_type
==
'msg_appendentries_t *'
:
size_of_entries
=
ffi
.
sizeof
(
ffi
.
getctype
(
'msg_entry_t'
))
*
new_msg
.
n_entries
new_msg
.
entries
=
ffi
.
cast
(
'msg_entry_t*'
,
lib
.
malloc
(
size_of_entries
))
ffi
.
memmove
(
new_msg
.
entries
,
msg
.
entries
,
size_of_entries
)
self
.
messages
.
append
(
Message
(
new_msg
,
sendor
,
sendee
))
def
poll_message
(
self
,
msg
):
msg_type
=
ffi
.
getctype
(
ffi
.
typeof
(
msg
.
data
))
if
msg_type
==
'msg_appendentries_t *'
:
node
=
lib
.
raft_get_node
(
msg
.
sendee
.
raft
,
msg
.
sendor
.
id
)
response
=
ffi
.
new
(
'msg_appendentries_response_t*'
)
e
=
lib
.
raft_recv_appendentries
(
msg
.
sendee
.
raft
,
node
,
msg
.
data
,
response
)
if
lib
.
RAFT_ERR_SHUTDOWN
==
e
:
logging
.
error
(
'Catastrophic'
)
print
(
msg
.
sendee
.
debug_log
())
print
(
msg
.
sendor
.
debug_log
())
sys
.
exit
(
1
)
elif
lib
.
RAFT_ERR_NEEDS_SNAPSHOT
==
e
:
pass
# TODO: pretend as if snapshot works
else
:
self
.
enqueue_msg
(
response
,
msg
.
sendee
,
msg
.
sendor
)
elif
msg_type
==
'msg_appendentries_response_t *'
:
node
=
lib
.
raft_get_node
(
msg
.
sendee
.
raft
,
msg
.
sendor
.
id
)
lib
.
raft_recv_appendentries_response
(
msg
.
sendee
.
raft
,
node
,
msg
.
data
)
elif
msg_type
==
'msg_requestvote_t *'
:
response
=
ffi
.
new
(
'msg_requestvote_response_t*'
)
node
=
lib
.
raft_get_node
(
msg
.
sendee
.
raft
,
msg
.
sendor
.
id
)
lib
.
raft_recv_requestvote
(
msg
.
sendee
.
raft
,
node
,
msg
.
data
,
response
)
self
.
enqueue_msg
(
response
,
msg
.
sendee
,
msg
.
sendor
)
elif
msg_type
==
'msg_requestvote_response_t *'
:
node
=
lib
.
raft_get_node
(
msg
.
sendee
.
raft
,
msg
.
sendor
.
id
)
e
=
lib
.
raft_recv_requestvote_response
(
msg
.
sendee
.
raft
,
node
,
msg
.
data
)
if
lib
.
RAFT_ERR_SHUTDOWN
==
e
:
msg
.
sendor
.
shutdown
()
else
:
assert
False
def
poll_messages
(
self
):
msgs
=
self
.
messages
# Chaos: re-ordered messages
# self.random.shuffle(msgs)
self
.
messages
=
[]
for
msg
in
msgs
:
self
.
poll_message
(
msg
)
for
server
in
self
.
active_servers
:
if
hasattr
(
server
,
'abort_exception'
):
raise
server
.
abort_exception
self
.
_check_current_idx_validity
(
server
)
self
.
_check_election_safety
()
def
_check_current_idx_validity
(
self
,
server
):
"""
Check that current idx is valid, ie. it exists
"""
ci
=
lib
.
raft_get_current_idx
(
server
.
raft
)
if
0
<
ci
and
not
lib
.
raft_get_snapshot_last_idx
(
server
.
raft
)
==
ci
:
ety
=
lib
.
raft_get_entry_from_idx
(
server
.
raft
,
ci
)
try
:
assert
ety
except
Exception
:
print
(
'current idx '
,
ci
)
print
(
'count'
,
lib
.
raft_get_log_count
(
server
.
raft
))
print
(
'last snapshot'
,
lib
.
raft_get_snapshot_last_idx
(
server
.
raft
))
print
(
server
.
debug_log
())
raise
def
_check_election_safety
(
self
):
"""
FIXME: this is O(n^2)
Election Safety
At most one leader can be elected in a given term.
"""
for
i
,
sv1
in
enumerate
(
net
.
active_servers
):
if
not
lib
.
raft_is_leader
(
sv1
.
raft
):
continue
for
sv2
in
net
.
active_servers
[
i
+
1
:]:
term1
=
lib
.
raft_get_current_term
(
sv1
.
raft
)
term2
=
lib
.
raft_get_current_term
(
sv2
.
raft
)
if
lib
.
raft_is_leader
(
sv2
.
raft
)
and
term1
==
term2
:
logging
.
error
(
"election safety invalidated"
)
print
(
sv1
,
sv2
,
term1
)
print
(
'partitions:'
,
self
.
partitions
)
sys
.
exit
(
1
)
def
commit_static_configuration
(
self
):
for
server
in
net
.
active_servers
:
server
.
connection_status
=
NODE_CONNECTED
for
sv
in
net
.
active_servers
:
is_self
=
1
if
sv
.
id
==
server
.
id
else
0
node
=
lib
.
raft_add_node
(
server
.
raft
,
sv
.
udata
,
sv
.
id
,
is_self
)
# FIXME: it's a bit much to expect to set these too
lib
.
raft_node_set_voting_committed
(
node
,
1
)
lib
.
raft_node_set_addition_committed
(
node
,
1
)
lib
.
raft_node_set_active
(
node
,
1
)
def
prep_dynamic_configuration
(
self
):
"""
Add configuration change for leader's node
"""
server
=
self
.
active_servers
[
0
]
self
.
leader
=
server
server
.
set_connection_status
(
NODE_CONNECTED
)
lib
.
raft_add_non_voting_node
(
server
.
raft
,
server
.
udata
,
server
.
id
,
1
)
lib
.
raft_become_leader
(
server
.
raft
)
# Configuration change entry to bootstrap other nodes
ety
=
ffi
.
new
(
'msg_entry_t*'
)
ety
.
term
=
0
ety
.
type
=
lib
.
RAFT_LOGTYPE_ADD_NODE
ety
.
id
=
self
.
new_entry_id
()
change
=
ffi
.
new_handle
(
ChangeRaftEntry
(
server
.
id
))
ety
.
data
.
buf
=
change
ety
.
data
.
len
=
ffi
.
sizeof
(
ety
.
data
.
buf
)
self
.
entries
.
append
((
ety
,
change
))
e
=
server
.
recv_entry
(
ety
)
assert
e
==
0
lib
.
raft_set_commit_idx
(
server
.
raft
,
1
)
e
=
lib
.
raft_apply_all
(
server
.
raft
)
assert
e
==
0
def
new_entry_id
(
self
):
self
.
ety_id
+=
1
return
self
.
ety_id
def
remove_server
(
self
,
server
):
server
.
removed
=
True
# self.servers = [s for s in self.servers if s is not server]
@
property
def
active_servers
(
self
):
return
[
s
for
s
in
self
.
servers
if
not
getattr
(
s
,
'removed'
,
False
)]
def
add_member
(
self
):
if
net
.
num_of_servers
<=
len
(
self
.
active_servers
):
return
if
not
self
.
leader
:
logging
.
error
(
'no leader'
)
return
leader
=
self
.
leader
if
not
lib
.
raft_is_leader
(
leader
.
raft
):
return
if
lib
.
raft_voting_change_is_in_progress
(
leader
.
raft
):
# logging.error('{} voting change in progress'.format(server))
return
server
=
RaftServer
(
self
)
# Create a new configuration entry to be processed by the leader
ety
=
ffi
.
new
(
'msg_entry_t*'
)
ety
.
term
=
0
ety
.
id
=
self
.
new_entry_id
()
ety
.
type
=
lib
.
RAFT_LOGTYPE_ADD_NONVOTING_NODE
change
=
ffi
.
new_handle
(
ChangeRaftEntry
(
server
.
id
))
ety
.
data
.
buf
=
change
ety
.
data
.
len
=
ffi
.
sizeof
(
ety
.
data
.
buf
)
assert
(
lib
.
raft_entry_is_cfg_change
(
ety
))
self
.
entries
.
append
((
ety
,
change
))
e
=
leader
.
recv_entry
(
ety
)
if
0
!=
e
:
logging
.
error
(
err2str
(
e
))
return
else
:
self
.
num_membership_changes
+=
1
# Wake up new node
server
.
set_connection_status
(
NODE_CONNECTING
)
assert
server
.
udata
added_node
=
lib
.
raft_add_non_voting_node
(
server
.
raft
,
server
.
udata
,
server
.
id
,
1
)
assert
added_node
def
remove_member
(
self
):
if
not
self
.
leader
:
logging
.
error
(
'no leader'
)
return
leader
=
self
.
leader
server
=
self
.
random
.
choice
(
self
.
active_servers
)
if
not
lib
.
raft_is_leader
(
leader
.
raft
):
return
if
lib
.
raft_voting_change_is_in_progress
(
leader
.
raft
):
# logging.error('{} voting change in progress'.format(server))
return
if
leader
==
server
:
# logging.error('can not remove leader')
return
if
server
.
connection_status
in
[
NODE_CONNECTING
,
NODE_DISCONNECTING
]:
# logging.error('can not remove server that is changing connection status')
return
if
NODE_DISCONNECTED
==
server
.
connection_status
:
self
.
remove_server
(
server
)
return
# Create a new configuration entry to be processed by the leader
ety
=
ffi
.
new
(
'msg_entry_t*'
)
ety
.
term
=
0
ety
.
id
=
self
.
new_entry_id
()
assert
server
.
connection_status
==
NODE_CONNECTED
ety
.
type
=
lib
.
RAFT_LOGTYPE_DEMOTE_NODE
change
=
ffi
.
new_handle
(
ChangeRaftEntry
(
server
.
id
))
ety
.
data
.
buf
=
change
ety
.
data
.
len
=
ffi
.
sizeof
(
ety
.
data
.
buf
)
assert
(
lib
.
raft_entry_is_cfg_change
(
ety
))
self
.
entries
.
append
((
ety
,
change
))
e
=
leader
.
recv_entry
(
ety
)
if
0
!=
e
:
logging
.
error
(
err2str
(
e
))
return
else
:
self
.
num_membership_changes
+=
1
# Wake up new node
assert
NODE_CONNECTED
==
server
.
connection_status
server
.
set_connection_status
(
NODE_DISCONNECTING
)
def
diagnotistic_info
(
self
):
print
()
info
=
{
"Maximum appendentries size"
:
self
.
max_entries_in_ae
,
"Leadership changes"
:
self
.
leadership_changes
,
"Log pops"
:
self
.
log_pops
,
"Unique nodes"
:
self
.
num_unique_nodes
,
"Membership changes"
:
self
.
num_membership_changes
,
"Compactions"
:
self
.
num_compactions
,
}
for
k
,
v
in
info
.
items
():
print
(
k
,
v
)
print
()
def
abbreviate
(
k
):
return
re
.
sub
(
r
'([a-z])[a-z]*_'
,
r
'\1'
,
k
)
# Servers
keys
=
sorted
(
net
.
servers
[
0
].
debug_statistics
().
keys
())
print
(
keys
)
data
=
[
list
(
map
(
abbreviate
,
keys
))]
+
[
[
s
.
debug_statistics
()[
key
]
for
key
in
keys
]
for
s
in
net
.
servers
]
table
=
terminaltables
.
AsciiTable
(
data
)
print
(
table
.
table
)
class
RaftServer
(
object
):
def
__init__
(
self
,
network
):
self
.
connection_status
=
NODE_DISCONNECTED
self
.
raft
=
lib
.
raft_new
()
self
.
udata
=
ffi
.
new_handle
(
self
)
network
.
add_server
(
self
)
self
.
load_callbacks
()
cbs
=
ffi
.
new
(
'raft_cbs_t*'
)
cbs
.
send_requestvote
=
self
.
raft_send_requestvote
cbs
.
send_appendentries
=
self
.
raft_send_appendentries
cbs
.
send_snapshot
=
self
.
raft_send_snapshot
cbs
.
applylog
=
self
.
raft_applylog
cbs
.
persist_vote
=
self
.
raft_persist_vote
cbs
.
persist_term
=
self
.
raft_persist_term
cbs
.
log_offer
=
self
.
raft_logentry_offer
cbs
.
log_poll
=
self
.
raft_logentry_poll
cbs
.
log_pop
=
self
.
raft_logentry_pop
cbs
.
log_get_node_id
=
self
.
raft_logentry_get_node_id
cbs
.
node_has_sufficient_logs
=
self
.
raft_node_has_sufficient_logs
cbs
.
notify_membership_event
=
self
.
raft_notify_membership_event
cbs
.
log
=
self
.
raft_log
lib
.
raft_set_callbacks
(
self
.
raft
,
cbs
,
self
.
udata
)
lib
.
raft_set_election_timeout
(
self
.
raft
,
500
)
self
.
fsm_dict
=
{}
self
.
fsm_log
=
[]
def
__str__
(
self
):
return
'<Server: {0}>'
.
format
(
self
.
id
)
def
__repr__
(
self
):
return
'sv:{0}'
.
format
(
self
.
id
)
def
set_connection_status
(
self
,
new_status
):
assert
(
not
(
self
.
connection_status
==
NODE_CONNECTED
and
new_status
==
NODE_CONNECTING
))
# logging.warning('{}: {} -> {}'.format(
# self,
# connectstatus2str(self.connection_status),
# connectstatus2str(new_status)))
self
.
connection_status
=
new_status
def
debug_log
(
self
):
first_idx
=
lib
.
raft_get_snapshot_last_idx
(
self
.
raft
)
return
[(
i
+
first_idx
,
l
.
term
,
l
.
id
)
for
i
,
l
in
enumerate
(
self
.
fsm_log
)]
def
do_compaction
(
self
):
# logging.warning('{} snapshotting'.format(self))
# entries_before = lib.raft_get_log_count(self.raft)
e
=
lib
.
raft_begin_snapshot
(
self
.
raft
)
if
e
!=
0
:
return
assert
(
lib
.
raft_snapshot_is_in_progress
(
self
.
raft
))
e
=
lib
.
raft_end_snapshot
(
self
.
raft
)
assert
(
e
==
0
)
if
e
!=
0
:
return
self
.
do_membership_snapshot
()
self
.
snapshot
.
image
=
dict
(
self
.
fsm_dict
)
self
.
snapshot
.
last_term
=
lib
.
raft_get_snapshot_last_term
(
self
.
raft
)
self
.
snapshot
.
last_idx
=
lib
.
raft_get_snapshot_last_idx
(
self
.
raft
)
self
.
network
.
num_compactions
+=
1
# logging.warning('{} entries compacted {}'.format(
# self,
# entries_before - lib.raft_get_log_count(self.raft)
# ))
def
periodic
(
self
,
msec
):
if
self
.
network
.
random
.
randint
(
1
,
100000
)
<
self
.
network
.
compaction_rate
:
self
.
do_compaction
()
e
=
lib
.
raft_periodic
(
self
.
raft
,
msec
)
if
lib
.
RAFT_ERR_SHUTDOWN
==
e
:
self
.
shutdown
()
# e = lib.raft_apply_all(self.raft)
# if lib.RAFT_ERR_SHUTDOWN == e:
# self.shutdown()
# return
if
hasattr
(
self
,
'abort_exception'
):
raise
self
.
abort_exception
def
shutdown
(
self
):
# logging.error('{} shutting down'.format(self))
self
.
set_connection_status
(
NODE_DISCONNECTED
)
self
.
network
.
remove_server
(
self
)
def
set_network
(
self
,
network
):
self
.
network
=
network
def
load_callbacks
(
self
):
self
.
raft_send_requestvote
=
ffi
.
callback
(
"int(raft_server_t*, void*, raft_node_t*, msg_requestvote_t*)"
,
raft_send_requestvote
)
self
.
raft_send_appendentries
=
ffi
.
callback
(
"int(raft_server_t*, void*, raft_node_t*, msg_appendentries_t*)"
,
raft_send_appendentries
)
self
.
raft_send_snapshot
=
ffi
.
callback
(
"int(raft_server_t*, void* , raft_node_t*)"
,
raft_send_snapshot
)
self
.
raft_applylog
=
ffi
.
callback
(
"int(raft_server_t*, void*, raft_entry_t*, raft_index_t)"
,
raft_applylog
)
self
.
raft_persist_vote
=
ffi
.
callback
(
"int(raft_server_t*, void*, raft_node_id_t)"
,
raft_persist_vote
)
self
.
raft_persist_term
=
ffi
.
callback
(
"int(raft_server_t*, void*, raft_term_t, raft_node_id_t)"
,
raft_persist_term
)
self
.
raft_logentry_offer
=
ffi
.
callback
(
"int(raft_server_t*, void*, raft_entry_t*, raft_index_t)"
,
raft_logentry_offer
)
self
.
raft_logentry_poll
=
ffi
.
callback
(
"int(raft_server_t*, void*, raft_entry_t*, raft_index_t)"
,
raft_logentry_poll
)
self
.
raft_logentry_pop
=
ffi
.
callback
(
"int(raft_server_t*, void*, raft_entry_t*, raft_index_t)"
,
raft_logentry_pop
)
self
.
raft_logentry_get_node_id
=
ffi
.
callback
(
"int(raft_server_t*, void*, raft_entry_t*, raft_index_t)"
,
raft_logentry_get_node_id
)
self
.
raft_node_has_sufficient_logs
=
ffi
.
callback
(
"int(raft_server_t* raft, void *user_data, raft_node_t* node)"
,
raft_node_has_sufficient_logs
)
self
.
raft_notify_membership_event
=
ffi
.
callback
(
"void(raft_server_t* raft, void *user_data, raft_node_t* node, raft_membership_e)"
,
raft_notify_membership_event
)
self
.
raft_log
=
ffi
.
callback
(
"void(raft_server_t*, raft_node_t*, void*, const char* buf)"
,
raft_log
)
def
recv_entry
(
self
,
ety
):
# FIXME: leak
response
=
ffi
.
new
(
'msg_entry_response_t*'
)
return
lib
.
raft_recv_entry
(
self
.
raft
,
ety
,
response
)
def
get_entry
(
self
,
idx
):
idx
=
idx
-
lib
.
raft_get_snapshot_last_idx
(
self
.
raft
)
if
idx
<
0
:
raise
IndexError
try
:
return
self
.
fsm_log
[
idx
]
except
:
# self.abort_exception = e
raise
def
_check_log_matching
(
self
,
our_log
,
idx
):
"""
Quality:
Log Matching: if two logs contain an entry with the same index and
term, then the logs are identical in all entries up through the given
index. §5.3
State Machine Safety: if a server has applied a log entry at a given
index to its state machine, no other server will ever apply a
different log entry for the same index. §5.4.3
"""
for
server
in
self
.
network
.
active_servers
:
if
server
is
self
:
continue
their_commit_idx
=
lib
.
raft_get_commit_idx
(
server
.
raft
)
if
lib
.
raft_get_commit_idx
(
self
.
raft
)
<=
their_commit_idx
and
idx
<=
their_commit_idx
:
their_log
=
lib
.
raft_get_entry_from_idx
(
server
.
raft
,
idx
)
if
their_log
==
ffi
.
NULL
:
assert
idx
<
lib
.
raft_get_snapshot_last_idx
(
self
.
raft
)
if
their_log
.
type
in
[
lib
.
RAFT_LOGTYPE_NORMAL
]:
try
:
assert
their_log
.
term
==
our_log
.
term
assert
their_log
.
id
==
our_log
.
id
except
Exception
as
e
:
ety1
=
lib
.
raft_get_entry_from_idx
(
self
.
raft
,
idx
)
ety2
=
lib
.
raft_get_entry_from_idx
(
server
.
raft
,
idx
)
logging
.
error
(
'ids'
,
ety1
.
id
,
ety2
.
id
)
logging
.
error
(
'{0}vs{1} idx:{2} terms:{3} {4} ids:{5} {6}'
.
format
(
self
,
server
,
idx
,
our_log
.
term
,
their_log
.
term
,
our_log
.
id
,
their_log
.
id
))
self
.
abort_exception
=
e
logging
.
error
(
self
.
debug_log
())
logging
.
error
(
server
.
debug_log
())
return
lib
.
RAFT_ERR_SHUTDOWN
def
entry_apply
(
self
,
ety
,
idx
):
# collect stats
if
self
.
network
.
latest_applied_log_idx
<
idx
:
self
.
network
.
latest_applied_log_idx
=
idx
self
.
network
.
latest_applied_log_iteration
=
self
.
network
.
iteration
e
=
self
.
_check_log_matching
(
ety
,
idx
)
if
e
is
not
None
:
return
e
change
=
ffi
.
from_handle
(
ety
.
data
.
buf
)
if
ety
.
type
==
lib
.
RAFT_LOGTYPE_NORMAL
:
self
.
fsm_dict
[
change
.
key
]
=
change
.
val
elif
ety
.
type
==
lib
.
RAFT_LOGTYPE_DEMOTE_NODE
:
if
change
.
node_id
==
lib
.
raft_get_nodeid
(
self
.
raft
):
# logging.warning("{} shutting down because of demotion".format(self))
return
lib
.
RAFT_ERR_SHUTDOWN
# Follow up by removing the node by receiving new entry
elif
lib
.
raft_is_leader
(
self
.
raft
):
new_ety
=
ffi
.
new
(
'msg_entry_t*'
)
new_ety
.
term
=
0
new_ety
.
id
=
self
.
network
.
new_entry_id
()
new_ety
.
type
=
lib
.
RAFT_LOGTYPE_REMOVE_NODE
new_ety
.
data
.
buf
=
ety
.
data
.
buf
new_ety
.
data
.
len
=
ffi
.
sizeof
(
ety
.
data
.
buf
)
assert
(
lib
.
raft_entry_is_cfg_change
(
new_ety
))
e
=
self
.
recv_entry
(
new_ety
)
assert
e
==
0
elif
ety
.
type
==
lib
.
RAFT_LOGTYPE_REMOVE_NODE
:
if
change
.
node_id
==
lib
.
raft_get_nodeid
(
self
.
raft
):
# logging.warning("{} shutting down because of removal".format(self))
return
lib
.
RAFT_ERR_SHUTDOWN
elif
ety
.
type
==
lib
.
RAFT_LOGTYPE_ADD_NODE
:
if
change
.
node_id
==
self
.
id
:
self
.
set_connection_status
(
NODE_CONNECTED
)
elif
ety
.
type
==
lib
.
RAFT_LOGTYPE_ADD_NONVOTING_NODE
:
pass
return
0
def
do_membership_snapshot
(
self
):
self
.
snapshot
=
Snapshot
()
for
i
in
range
(
0
,
lib
.
raft_get_num_nodes
(
self
.
raft
)):
n
=
lib
.
raft_get_node_from_idx
(
self
.
raft
,
i
)
id
=
lib
.
raft_node_get_id
(
n
)
if
0
==
lib
.
raft_node_is_addition_committed
(
n
):
id
=
-
1
self
.
snapshot
.
members
.
append
(
SnapshotMember
(
id
,
lib
.
raft_node_is_voting_committed
(
n
)))
def
load_snapshot
(
self
,
snapshot
,
other
):
# logging.warning('{} loading snapshot'.format(self))
e
=
lib
.
raft_begin_load_snapshot
(
self
.
raft
,
snapshot
.
last_term
,
snapshot
.
last_idx
,
)
if
e
==
-
1
:
return
0
elif
e
==
lib
.
RAFT_ERR_SNAPSHOT_ALREADY_LOADED
:
return
0
elif
e
==
0
:
pass
else
:
assert
False
# Send appendentries response for this snapshot
response
=
ffi
.
new
(
'msg_appendentries_response_t*'
)
response
.
success
=
1
response
.
current_idx
=
snapshot
.
last_idx
response
.
term
=
lib
.
raft_get_current_term
(
self
.
raft
)
response
.
first_idx
=
response
.
current_idx
self
.
network
.
enqueue_msg
(
response
,
self
,
other
)
node_id
=
lib
.
raft_get_nodeid
(
self
.
raft
)
# set membership configuration according to snapshot
for
member
in
snapshot
.
members
:
if
-
1
==
member
.
id
:
continue
node
=
lib
.
raft_get_node
(
self
.
raft
,
member
.
id
)
if
not
node
:
udata
=
ffi
.
NULL
try
:
node_sv
=
self
.
network
.
id2server
(
member
.
id
)
udata
=
node_sv
.
udata
except
ServerDoesNotExist
:
pass
node
=
lib
.
raft_add_node
(
self
.
raft
,
udata
,
member
.
id
,
member
.
id
==
node_id
)
lib
.
raft_node_set_active
(
node
,
1
)
if
member
.
voting
and
not
lib
.
raft_node_is_voting
(
node
):
lib
.
raft_node_set_voting
(
node
,
1
)
elif
not
member
.
voting
and
lib
.
raft_node_is_voting
(
node
):
lib
.
raft_node_set_voting
(
node
,
0
)
if
node_id
!=
member
.
id
:
assert
node
# TODO: this is quite ugly
# we should have a function that removes all nodes by ourself
# if (!raft_get_my_node(self->raft)) */
# raft_add_non_voting_node(self->raft, NULL, node_id, 1); */
e
=
lib
.
raft_end_load_snapshot
(
self
.
raft
)
assert
(
0
==
e
)
assert
(
lib
.
raft_get_log_count
(
self
.
raft
)
==
0
)
self
.
do_membership_snapshot
()
self
.
snapshot
.
image
=
dict
(
snapshot
.
image
)
self
.
snapshot
.
last_term
=
snapshot
.
last_term
self
.
snapshot
.
last_idx
=
snapshot
.
last_idx
assert
(
lib
.
raft_get_my_node
(
self
.
raft
))
# assert(sv->snapshot_fsm);
self
.
fsm_dict
=
dict
(
snapshot
.
image
)
logging
.
warning
(
'{} loaded snapshot t:{} idx:{}'
.
format
(
self
,
snapshot
.
last_term
,
snapshot
.
last_idx
))
def
send_snapshot
(
self
,
node
):
assert
not
lib
.
raft_snapshot_is_in_progress
(
self
.
raft
)
# FIXME: Why would this happen?
if
not
hasattr
(
self
,
'snapshot'
):
return
0
node_sv
=
ffi
.
from_handle
(
lib
.
raft_node_get_udata
(
node
))
# TODO: Why would this happen?
# seems odd that we would send something to a node that didn't exist
if
not
node_sv
:
return
0
# NOTE:
# In a real server we would have to send the snapshot file to the
# other node. Here we have the convenience of the transfer being
# "immediate".
node_sv
.
load_snapshot
(
self
.
snapshot
,
self
)
return
0
def
persist_vote
(
self
,
voted_for
):
# TODO: add disk simulation
return
0
def
persist_term
(
self
,
term
,
vote
):
# TODO: add disk simulation
return
0
def
_check_id_monoticity
(
self
,
ety
):
"""
Check last entry has smaller ID than new entry.
This is a virtraft specific check to make sure entry passing is
working correctly.
"""
ci
=
lib
.
raft_get_current_idx
(
self
.
raft
)
if
0
<
ci
and
not
lib
.
raft_get_snapshot_last_idx
(
self
.
raft
)
==
ci
:
try
:
prev_ety
=
lib
.
raft_get_entry_from_idx
(
self
.
raft
,
ci
)
assert
prev_ety
other_id
=
prev_ety
.
id
assert
other_id
<
ety
.
id
except
Exception
as
e
:
logging
.
error
(
other_id
,
ety
.
id
)
self
.
abort_exception
=
e
raise
def
entry_append
(
self
,
ety
,
ety_idx
):
try
:
assert
not
self
.
fsm_log
or
self
.
fsm_log
[
-
1
].
term
<=
ety
.
term
except
Exception
as
e
:
self
.
abort_exception
=
e
# FIXME: consider returning RAFT_ERR_SHUTDOWN
raise
self
.
_check_id_monoticity
(
ety
)
self
.
fsm_log
.
append
(
RaftEntry
(
ety
))
return
0
def
entry_poll
(
self
,
ety
,
ety_idx
):
self
.
fsm_log
.
pop
(
0
)
return
0
def
_check_committed_entry_popping
(
self
,
ety_idx
):
"""
Check we aren't popping a committed entry
"""
try
:
assert
lib
.
raft_get_commit_idx
(
self
.
raft
)
<
ety_idx
except
Exception
as
e
:
self
.
abort_exception
=
e
return
lib
.
RAFT_ERR_SHUTDOWN
return
0
def
entry_pop
(
self
,
ety
,
ety_idx
):
# logging.warning("POP {} {}".format(self, ety_idx))
e
=
self
.
_check_committed_entry_popping
(
ety_idx
)
if
e
!=
0
:
return
e
self
.
fsm_log
.
pop
()
self
.
network
.
log_pops
+=
1
change
=
ffi
.
from_handle
(
ety
.
data
.
buf
)
if
ety
.
type
==
lib
.
RAFT_LOGTYPE_DEMOTE_NODE
:
pass
elif
ety
.
type
==
lib
.
RAFT_LOGTYPE_REMOVE_NODE
:
if
change
.
node_id
==
lib
.
raft_get_nodeid
(
self
.
raft
):
self
.
set_connection_status
(
NODE_CONNECTED
)
elif
ety
.
type
==
lib
.
RAFT_LOGTYPE_ADD_NONVOTING_NODE
:
if
change
.
node_id
==
lib
.
raft_get_nodeid
(
self
.
raft
):
logging
.
error
(
"POP disconnect {} {}"
.
format
(
self
,
ety_idx
))
self
.
set_connection_status
(
NODE_DISCONNECTED
)
elif
ety
.
type
==
lib
.
RAFT_LOGTYPE_ADD_NODE
:
if
change
.
node_id
==
lib
.
raft_get_nodeid
(
self
.
raft
):
self
.
set_connection_status
(
NODE_CONNECTING
)
return
0
def
node_has_sufficient_entries
(
self
,
node
):
assert
(
not
lib
.
raft_node_is_voting
(
node
))
ety
=
ffi
.
new
(
'msg_entry_t*'
)
ety
.
term
=
0
ety
.
id
=
self
.
network
.
new_entry_id
()
ety
.
type
=
lib
.
RAFT_LOGTYPE_ADD_NODE
change
=
ffi
.
new_handle
(
ChangeRaftEntry
(
lib
.
raft_node_get_id
(
node
)))
ety
.
data
.
buf
=
change
ety
.
data
.
len
=
ffi
.
sizeof
(
ety
.
data
.
buf
)
self
.
network
.
entries
.
append
((
ety
,
change
))
assert
(
lib
.
raft_entry_is_cfg_change
(
ety
))
# FIXME: leak
e
=
self
.
recv_entry
(
ety
)
# print(err2str(e))
assert
e
==
0
return
0
def
notify_membership_event
(
self
,
node
,
event_type
):
# Convenience: Ensure that added node has udata set
if
event_type
==
lib
.
RAFT_MEMBERSHIP_ADD
:
node_id
=
lib
.
raft_node_get_id
(
node
)
try
:
server
=
self
.
network
.
id2server
(
node_id
)
except
ServerDoesNotExist
:
pass
else
:
node
=
lib
.
raft_get_node
(
self
.
raft
,
node_id
)
lib
.
raft_node_set_udata
(
node
,
server
.
udata
)
def
debug_statistics
(
self
):
return
{
"node"
:
lib
.
raft_get_nodeid
(
self
.
raft
),
"state"
:
state2str
(
lib
.
raft_get_state
(
self
.
raft
)),
"current_idx"
:
lib
.
raft_get_current_idx
(
self
.
raft
),
"last_log_term"
:
lib
.
raft_get_last_log_term
(
self
.
raft
),
"current_term"
:
lib
.
raft_get_current_term
(
self
.
raft
),
"commit_idx"
:
lib
.
raft_get_commit_idx
(
self
.
raft
),
"last_applied_idx"
:
lib
.
raft_get_last_applied_idx
(
self
.
raft
),
"log_count"
:
lib
.
raft_get_log_count
(
self
.
raft
),
"peers"
:
lib
.
raft_get_num_nodes
(
self
.
raft
),
"voting_peers"
:
lib
.
raft_get_num_voting_nodes
(
self
.
raft
),
"connection_status"
:
connectstatus2str
(
self
.
connection_status
),
"voting_change_in_progress"
:
lib
.
raft_voting_change_is_in_progress
(
self
.
raft
),
"removed"
:
getattr
(
self
,
'removed'
,
False
),
}
if
__name__
==
'__main__'
:
try
:
args
=
docopt
.
docopt
(
__doc__
,
version
=
'virtraft 0.1'
)
except
docopt
.
DocoptExit
as
e
:
print
(
e
)
sys
.
exit
()
if
args
[
'--verbose'
]
or
args
[
'--log_level'
]:
coloredlogs
.
install
(
fmt
=
'%(asctime)s %(levelname)s %(message)s'
)
level
=
logging
.
DEBUG
if
args
[
'--log_level'
]:
level
=
int
(
args
[
'--log_level'
])
logging
.
basicConfig
(
level
=
level
,
format
=
'%(asctime)s %(message)s'
)
net
=
Network
(
int
(
args
[
'--seed'
]))
net
.
dupe_rate
=
int
(
args
[
'--dupe_rate'
])
net
.
drop_rate
=
int
(
args
[
'--drop_rate'
])
net
.
client_rate
=
int
(
args
[
'--client_rate'
])
net
.
member_rate
=
int
(
args
[
'--member_rate'
])
net
.
compaction_rate
=
int
(
args
[
'--compaction_rate'
])
net
.
partition_rate
=
int
(
args
[
'--partition_rate'
])
net
.
no_random_period
=
1
==
int
(
args
[
'--no_random_period'
])
net
.
num_of_servers
=
int
(
args
[
'--servers'
])
if
net
.
member_rate
==
0
:
for
i
in
range
(
0
,
int
(
args
[
'--servers'
])):
RaftServer
(
net
)
net
.
commit_static_configuration
()
else
:
RaftServer
(
net
)
net
.
prep_dynamic_configuration
()
for
i
in
range
(
0
,
int
(
args
[
'--iterations'
])):
net
.
iteration
+=
1
try
:
net
.
periodic
()
net
.
poll_messages
()
except
:
# for server in net.servers:
# print(server, [l.term for l in server.fsm_log])
raise
net
.
diagnotistic_info
()
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment