Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
ruanhaishen
RedisLabs Raft
Commits
c7b78dbb
Commit
c7b78dbb
authored
Sep 02, 2018
by
Yossi Gottlieb
Browse files
Merge remote-tracking branch 'upstream/master'
parents
9cad8a34
0e755635
Changes
5
Hide whitespace changes
Inline
Side-by-side
.travis.yml
View file @
c7b78dbb
language
:
c
language
:
c
addons
:
apt
:
packages
:
-
"
python3"
-
"
python3-pip"
before_install
:
before_install
:
-
sudo pip install cpp-coveralls
-
sudo pip install cpp-coveralls
-
sudo pip3 install cffi
-
sudo pip3 install colorama
-
sudo pip3 install coloredlogs
-
sudo pip3 install docopt
-
sudo pip3 install terminaltables
script
:
script
:
-
make tests
-
make tests
-
make test_virtraft
after_success
:
after_success
:
-
coveralls --verbose --exclude include --exclude tests --exclude CLinkedListQueue
-
coveralls --verbose --exclude include --exclude tests --exclude CLinkedListQueue
Makefile
View file @
c7b78dbb
...
@@ -72,7 +72,13 @@ tests_full:
...
@@ -72,7 +72,13 @@ tests_full:
.PHONY
:
test_virtraft
.PHONY
:
test_virtraft
test_virtraft
:
test_virtraft
:
python3 tests/virtraft2.py
--servers
5
-i
20000
--compaction_rate
50
--drop_rate
5
-P
10
--seed
1
-m
3
python3 tests/virtraft2.py
--servers
7
-i
20000
--compaction_rate
50
--drop_rate
5
-P
10
--seed
1
-m
3
python3 tests/virtraft2.py
--servers
7
-i
20000
--compaction_rate
50
--drop_rate
5
-P
10
--seed
1
-m
3
python3 tests/virtraft2.py
--servers
5
-i
20000
--compaction_rate
50
--drop_rate
5
-P
10
--seed
2
-m
3
python3 tests/virtraft2.py
--servers
5
-i
20000
--compaction_rate
50
--drop_rate
5
-P
10
--seed
3
-m
3
python3 tests/virtraft2.py
--servers
5
-i
20000
--compaction_rate
50
--drop_rate
5
-P
10
--seed
4
-m
3
python3 tests/virtraft2.py
--servers
5
-i
20000
--compaction_rate
50
--drop_rate
5
-P
10
--seed
5
-m
3
python3 tests/virtraft2.py
--servers
5
-i
20000
--compaction_rate
50
--drop_rate
5
-P
10
--seed
6
-m
3
.PHONY
:
amalgamation
.PHONY
:
amalgamation
amalgamation
:
amalgamation
:
...
...
src/raft_server.c
View file @
c7b78dbb
...
@@ -1308,7 +1308,7 @@ int raft_cancel_snapshot(raft_server_t *me_)
...
@@ -1308,7 +1308,7 @@ int raft_cancel_snapshot(raft_server_t *me_)
me
->
snapshot_last_idx
=
me
->
saved_snapshot_last_idx
;
me
->
snapshot_last_idx
=
me
->
saved_snapshot_last_idx
;
me
->
snapshot_last_term
=
me
->
saved_snapshot_last_term
;
me
->
snapshot_last_term
=
me
->
saved_snapshot_last_term
;
me
->
snapshot_in_progress
=
1
;
me
->
snapshot_in_progress
=
0
;
return
0
;
return
0
;
}
}
...
...
tests/test_snapshotting.c
View file @
c7b78dbb
...
@@ -237,6 +237,13 @@ void TestRaft_leader_snapshot_end_succeeds_if_log_compacted(CuTest * tc)
...
@@ -237,6 +237,13 @@ void TestRaft_leader_snapshot_end_succeeds_if_log_compacted(CuTest * tc)
CuAssertIntEquals
(
tc
,
3
,
raft_get_log_count
(
r
));
CuAssertIntEquals
(
tc
,
3
,
raft_get_log_count
(
r
));
CuAssertIntEquals
(
tc
,
2
,
raft_get_num_snapshottable_logs
(
r
));
CuAssertIntEquals
(
tc
,
2
,
raft_get_num_snapshottable_logs
(
r
));
CuAssertIntEquals
(
tc
,
0
,
raft_begin_snapshot
(
r
,
0
));
raft_entry_t
*
_ety
;
int
i
=
raft_get_first_entry_idx
(
r
);
for
(;
i
<
raft_get_commit_idx
(
r
);
i
++
)
CuAssertIntEquals
(
tc
,
0
,
raft_poll_entry
(
r
,
&
_ety
));
CuAssertIntEquals
(
tc
,
0
,
raft_begin_snapshot
(
r
,
0
));
CuAssertIntEquals
(
tc
,
0
,
raft_begin_snapshot
(
r
,
0
));
CuAssertIntEquals
(
tc
,
0
,
raft_end_snapshot
(
r
));
CuAssertIntEquals
(
tc
,
0
,
raft_end_snapshot
(
r
));
CuAssertIntEquals
(
tc
,
0
,
raft_get_num_snapshottable_logs
(
r
));
CuAssertIntEquals
(
tc
,
0
,
raft_get_num_snapshottable_logs
(
r
));
...
@@ -283,6 +290,12 @@ void TestRaft_leader_snapshot_end_succeeds_if_log_compacted2(CuTest * tc)
...
@@ -283,6 +290,12 @@ void TestRaft_leader_snapshot_end_succeeds_if_log_compacted2(CuTest * tc)
CuAssertIntEquals
(
tc
,
2
,
raft_get_num_snapshottable_logs
(
r
));
CuAssertIntEquals
(
tc
,
2
,
raft_get_num_snapshottable_logs
(
r
));
CuAssertIntEquals
(
tc
,
0
,
raft_begin_snapshot
(
r
,
0
));
CuAssertIntEquals
(
tc
,
0
,
raft_begin_snapshot
(
r
,
0
));
raft_entry_t
*
_ety
;
int
i
=
raft_get_first_entry_idx
(
r
);
for
(;
i
<=
raft_get_commit_idx
(
r
);
i
++
)
CuAssertIntEquals
(
tc
,
0
,
raft_poll_entry
(
r
,
&
_ety
));
CuAssertIntEquals
(
tc
,
0
,
raft_end_snapshot
(
r
));
CuAssertIntEquals
(
tc
,
0
,
raft_end_snapshot
(
r
));
CuAssertIntEquals
(
tc
,
0
,
raft_get_num_snapshottable_logs
(
r
));
CuAssertIntEquals
(
tc
,
0
,
raft_get_num_snapshottable_logs
(
r
));
CuAssertIntEquals
(
tc
,
1
,
raft_get_log_count
(
r
));
CuAssertIntEquals
(
tc
,
1
,
raft_get_log_count
(
r
));
...
@@ -673,3 +686,55 @@ void TestRaft_leader_sends_appendentries_with_correct_prev_log_idx_when_snapshot
...
@@ -673,3 +686,55 @@ void TestRaft_leader_sends_appendentries_with_correct_prev_log_idx_when_snapshot
CuAssertIntEquals
(
tc
,
2
,
ae
->
prev_log_term
);
CuAssertIntEquals
(
tc
,
2
,
ae
->
prev_log_term
);
CuAssertIntEquals
(
tc
,
4
,
ae
->
prev_log_idx
);
CuAssertIntEquals
(
tc
,
4
,
ae
->
prev_log_idx
);
}
}
void
TestRaft_cancel_snapshot_restores_state
(
CuTest
*
tc
)
{
raft_cbs_t
funcs
=
{
.
send_appendentries
=
__raft_send_appendentries
,
};
void
*
r
=
raft_new
();
raft_set_callbacks
(
r
,
&
funcs
,
NULL
);
msg_entry_response_t
cr
;
raft_add_node
(
r
,
NULL
,
1
,
1
);
raft_add_node
(
r
,
NULL
,
2
,
0
);
/* I am the leader */
raft_set_state
(
r
,
RAFT_STATE_LEADER
);
CuAssertIntEquals
(
tc
,
0
,
raft_get_log_count
(
r
));
/* single entry */
msg_entry_t
ety
=
{};
ety
.
id
=
1
;
ety
.
data
.
buf
=
"entry"
;
ety
.
data
.
len
=
strlen
(
"entry"
);
raft_recv_entry
(
r
,
&
ety
,
&
cr
);
ety
.
id
=
2
;
raft_recv_entry
(
r
,
&
ety
,
&
cr
);
raft_set_commit_idx
(
r
,
2
);
CuAssertIntEquals
(
tc
,
0
,
raft_begin_snapshot
(
r
,
0
));
CuAssertIntEquals
(
tc
,
0
,
raft_end_snapshot
(
r
));
/* more entries */
ety
.
id
=
3
;
raft_recv_entry
(
r
,
&
ety
,
&
cr
);
ety
.
id
=
4
;
raft_recv_entry
(
r
,
&
ety
,
&
cr
);
CuAssertIntEquals
(
tc
,
2
,
raft_get_log_count
(
r
));
CuAssertIntEquals
(
tc
,
2
,
raft_get_snapshot_last_idx
(
r
));
/* begin and cancel another snapshot */
raft_set_commit_idx
(
r
,
4
);
CuAssertIntEquals
(
tc
,
0
,
raft_begin_snapshot
(
r
,
0
));
CuAssertIntEquals
(
tc
,
1
,
raft_snapshot_is_in_progress
(
r
));
CuAssertIntEquals
(
tc
,
0
,
raft_cancel_snapshot
(
r
));
/* snapshot no longer in progress, index must not have changed */
CuAssertIntEquals
(
tc
,
0
,
raft_snapshot_is_in_progress
(
r
));
CuAssertIntEquals
(
tc
,
2
,
raft_get_snapshot_last_idx
(
r
));
}
tests/virtraft2.py
View file @
c7b78dbb
...
@@ -21,7 +21,7 @@ Some chaos we generate:
...
@@ -21,7 +21,7 @@ Some chaos we generate:
Usage:
Usage:
virtraft --servers SERVERS [-d RATE] [-D RATE] [-c RATE] [-C RATE] [-m RATE]
virtraft --servers SERVERS [-d RATE] [-D RATE] [-c RATE] [-C RATE] [-m RATE]
[-P RATE] [-s SEED] [-i ITERS] [-p] [--tsv]
[-P RATE] [-s SEED] [-i ITERS] [-p] [--tsv]
[-q] [-v] [-l LEVEL]
[-q] [-v] [-l LEVEL]
[-j] [-L LOGFILE]
virtraft --version
virtraft --version
virtraft --help
virtraft --help
...
@@ -42,7 +42,9 @@ Options:
...
@@ -42,7 +42,9 @@ Options:
[default: 1000]
[default: 1000]
--tsv Output node status tab separated values at exit
--tsv Output node status tab separated values at exit
-v --verbose Show debug logs
-v --verbose Show debug logs
-j --json_output JSON output for diagnostics
-l --log_level LEVEL Set log level
-l --log_level LEVEL Set log level
-L --log_file LOGFILE Set log file
-V --version Display version.
-V --version Display version.
-h --help Prints a short usage summary.
-h --help Prints a short usage summary.
...
@@ -63,6 +65,10 @@ import terminaltables
...
@@ -63,6 +65,10 @@ import terminaltables
from
raft_cffi
import
ffi
,
lib
from
raft_cffi
import
ffi
,
lib
logger
=
logging
.
getLogger
(
'raft'
)
NODE_DISCONNECTED
=
0
NODE_DISCONNECTED
=
0
NODE_CONNECTING
=
1
NODE_CONNECTING
=
1
NODE_CONNECTED
=
2
NODE_CONNECTED
=
2
...
@@ -207,8 +213,8 @@ def raft_node_has_sufficient_logs(raft, udata, node):
...
@@ -207,8 +213,8 @@ def raft_node_has_sufficient_logs(raft, udata, node):
return
ffi
.
from_handle
(
udata
).
node_has_sufficient_entries
(
node
)
return
ffi
.
from_handle
(
udata
).
node_has_sufficient_entries
(
node
)
def
raft_notify_membership_event
(
raft
,
udata
,
node
,
event_type
):
def
raft_notify_membership_event
(
raft
,
udata
,
node
,
ety
,
event_type
):
return
ffi
.
from_handle
(
udata
).
notify_membership_event
(
node
,
event_type
)
return
ffi
.
from_handle
(
udata
).
notify_membership_event
(
node
,
ety
,
event_type
)
def
raft_log
(
raft
,
node
,
udata
,
buf
):
def
raft_log
(
raft
,
node
,
udata
,
buf
):
...
@@ -217,7 +223,7 @@ def raft_log(raft, node, udata, buf):
...
@@ -217,7 +223,7 @@ def raft_log(raft, node, udata, buf):
if
node
!=
ffi
.
NULL
:
if
node
!=
ffi
.
NULL
:
node
=
ffi
.
from_handle
(
lib
.
raft_node_get_udata
(
node
))
node
=
ffi
.
from_handle
(
lib
.
raft_node_get_udata
(
node
))
# if server.id in [1] or (node and node.id in [1]):
# if server.id in [1] or (node and node.id in [1]):
logg
ing
.
info
(
'{0}> {1}:{2}: {3}'
.
format
(
logg
er
.
info
(
'{0}> {1}:{2}: {3}'
.
format
(
server
.
network
.
iteration
,
server
.
network
.
iteration
,
server
.
id
,
server
.
id
,
node
.
id
if
node
else
''
,
node
.
id
if
node
else
''
,
...
@@ -244,7 +250,7 @@ class Network(object):
...
@@ -244,7 +250,7 @@ class Network(object):
self
.
ety_id
=
0
self
.
ety_id
=
0
self
.
entries
=
[]
self
.
entries
=
[]
self
.
random
=
random
.
Random
(
seed
)
self
.
random
=
random
.
Random
(
seed
)
self
.
partitions
=
[]
self
.
partitions
=
set
()
self
.
no_random_period
=
False
self
.
no_random_period
=
False
self
.
server_id
=
0
self
.
server_id
=
0
...
@@ -296,11 +302,12 @@ class Network(object):
...
@@ -296,11 +302,12 @@ class Network(object):
self
.
random
.
shuffle
(
nodes
)
self
.
random
.
shuffle
(
nodes
)
n1
=
nodes
.
pop
()
n1
=
nodes
.
pop
()
n2
=
nodes
.
pop
()
n2
=
nodes
.
pop
()
self
.
partitions
.
a
ppen
d
((
n1
,
n2
))
self
.
partitions
.
a
d
d
((
n1
,
n2
))
def
remove_partition
(
self
):
def
remove_partition
(
self
):
self
.
random
.
shuffle
(
self
.
partitions
)
parts
=
sorted
(
list
(
self
.
partitions
))
self
.
partitions
.
pop
()
self
.
random
.
shuffle
(
parts
)
self
.
partitions
.
remove
(
parts
.
pop
())
def
periodic
(
self
):
def
periodic
(
self
):
if
self
.
random
.
randint
(
1
,
100
)
<
self
.
member_rate
:
if
self
.
random
.
randint
(
1
,
100
)
<
self
.
member_rate
:
...
@@ -326,7 +333,7 @@ class Network(object):
...
@@ -326,7 +333,7 @@ class Network(object):
# Deadlock detection
# Deadlock detection
if
self
.
latest_applied_log_idx
!=
0
and
self
.
latest_applied_log_iteration
+
5000
<
self
.
iteration
:
if
self
.
latest_applied_log_idx
!=
0
and
self
.
latest_applied_log_iteration
+
5000
<
self
.
iteration
:
logg
ing
.
error
(
"deadlock detected iteration:{0} appliedidx:{1}
\n
"
.
format
(
logg
er
.
error
(
"deadlock detected iteration:{0} appliedidx:{1}
\n
"
.
format
(
self
.
latest_applied_log_iteration
,
self
.
latest_applied_log_iteration
,
self
.
latest_applied_log_idx
,
self
.
latest_applied_log_idx
,
))
))
...
@@ -378,7 +385,7 @@ class Network(object):
...
@@ -378,7 +385,7 @@ class Network(object):
response
=
ffi
.
new
(
'msg_appendentries_response_t*'
)
response
=
ffi
.
new
(
'msg_appendentries_response_t*'
)
e
=
lib
.
raft_recv_appendentries
(
msg
.
sendee
.
raft
,
node
,
msg
.
data
,
response
)
e
=
lib
.
raft_recv_appendentries
(
msg
.
sendee
.
raft
,
node
,
msg
.
data
,
response
)
if
lib
.
RAFT_ERR_SHUTDOWN
==
e
:
if
lib
.
RAFT_ERR_SHUTDOWN
==
e
:
logg
ing
.
error
(
'Catastrophic'
)
logg
er
.
error
(
'Catastrophic'
)
print
(
msg
.
sendee
.
debug_log
())
print
(
msg
.
sendee
.
debug_log
())
print
(
msg
.
sendor
.
debug_log
())
print
(
msg
.
sendor
.
debug_log
())
sys
.
exit
(
1
)
sys
.
exit
(
1
)
...
@@ -451,7 +458,7 @@ class Network(object):
...
@@ -451,7 +458,7 @@ class Network(object):
term1
=
lib
.
raft_get_current_term
(
sv1
.
raft
)
term1
=
lib
.
raft_get_current_term
(
sv1
.
raft
)
term2
=
lib
.
raft_get_current_term
(
sv2
.
raft
)
term2
=
lib
.
raft_get_current_term
(
sv2
.
raft
)
if
lib
.
raft_is_leader
(
sv2
.
raft
)
and
term1
==
term2
:
if
lib
.
raft_is_leader
(
sv2
.
raft
)
and
term1
==
term2
:
logg
ing
.
error
(
"election safety invalidated"
)
logg
er
.
error
(
"election safety invalidated"
)
print
(
sv1
,
sv2
,
term1
)
print
(
sv1
,
sv2
,
term1
)
print
(
'partitions:'
,
self
.
partitions
)
print
(
'partitions:'
,
self
.
partitions
)
sys
.
exit
(
1
)
sys
.
exit
(
1
)
...
@@ -512,7 +519,7 @@ class Network(object):
...
@@ -512,7 +519,7 @@ class Network(object):
return
return
if
not
self
.
leader
:
if
not
self
.
leader
:
logg
ing
.
error
(
'no leader'
)
logg
er
.
error
(
'no leader'
)
return
return
leader
=
self
.
leader
leader
=
self
.
leader
...
@@ -521,7 +528,7 @@ class Network(object):
...
@@ -521,7 +528,7 @@ class Network(object):
return
return
if
lib
.
raft_voting_change_is_in_progress
(
leader
.
raft
):
if
lib
.
raft_voting_change_is_in_progress
(
leader
.
raft
):
# logg
ing
.error('{} voting change in progress'.format(server))
# logg
er
.error('{} voting change in progress'.format(server))
return
return
server
=
RaftServer
(
self
)
server
=
RaftServer
(
self
)
...
@@ -540,7 +547,7 @@ class Network(object):
...
@@ -540,7 +547,7 @@ class Network(object):
e
=
leader
.
recv_entry
(
ety
)
e
=
leader
.
recv_entry
(
ety
)
if
0
!=
e
:
if
0
!=
e
:
logg
ing
.
error
(
err2str
(
e
))
logg
er
.
error
(
err2str
(
e
))
return
return
else
:
else
:
self
.
num_membership_changes
+=
1
self
.
num_membership_changes
+=
1
...
@@ -553,7 +560,7 @@ class Network(object):
...
@@ -553,7 +560,7 @@ class Network(object):
def
remove_member
(
self
):
def
remove_member
(
self
):
if
not
self
.
leader
:
if
not
self
.
leader
:
logg
ing
.
error
(
'no leader'
)
logg
er
.
error
(
'no leader'
)
return
return
leader
=
self
.
leader
leader
=
self
.
leader
...
@@ -563,15 +570,15 @@ class Network(object):
...
@@ -563,15 +570,15 @@ class Network(object):
return
return
if
lib
.
raft_voting_change_is_in_progress
(
leader
.
raft
):
if
lib
.
raft_voting_change_is_in_progress
(
leader
.
raft
):
# logg
ing
.error('{} voting change in progress'.format(server))
# logg
er
.error('{} voting change in progress'.format(server))
return
return
if
leader
==
server
:
if
leader
==
server
:
# logg
ing
.error('can not remove leader')
# logg
er
.error('can not remove leader')
return
return
if
server
.
connection_status
in
[
NODE_CONNECTING
,
NODE_DISCONNECTING
]:
if
server
.
connection_status
in
[
NODE_CONNECTING
,
NODE_DISCONNECTING
]:
# logg
ing
.error('can not remove server that is changing connection status')
# logg
er
.error('can not remove server that is changing connection status')
return
return
if
NODE_DISCONNECTED
==
server
.
connection_status
:
if
NODE_DISCONNECTED
==
server
.
connection_status
:
...
@@ -593,7 +600,7 @@ class Network(object):
...
@@ -593,7 +600,7 @@ class Network(object):
e
=
leader
.
recv_entry
(
ety
)
e
=
leader
.
recv_entry
(
ety
)
if
0
!=
e
:
if
0
!=
e
:
logg
ing
.
error
(
err2str
(
e
))
logg
er
.
error
(
err2str
(
e
))
return
return
else
:
else
:
self
.
num_membership_changes
+=
1
self
.
num_membership_changes
+=
1
...
@@ -614,6 +621,8 @@ class Network(object):
...
@@ -614,6 +621,8 @@ class Network(object):
"Compactions"
:
self
.
num_compactions
,
"Compactions"
:
self
.
num_compactions
,
}
}
print
(
'partitions:'
,
[
'{} -> {}'
.
format
(
*
map
(
str
,
p
))
for
p
in
self
.
partitions
])
for
k
,
v
in
info
.
items
():
for
k
,
v
in
info
.
items
():
print
(
k
,
v
)
print
(
k
,
v
)
...
@@ -624,7 +633,6 @@ class Network(object):
...
@@ -624,7 +633,6 @@ class Network(object):
# Servers
# Servers
keys
=
sorted
(
net
.
servers
[
0
].
debug_statistics
().
keys
())
keys
=
sorted
(
net
.
servers
[
0
].
debug_statistics
().
keys
())
print
(
keys
)
data
=
[
list
(
map
(
abbreviate
,
keys
))]
+
[
data
=
[
list
(
map
(
abbreviate
,
keys
))]
+
[
[
s
.
debug_statistics
()[
key
]
for
key
in
keys
]
[
s
.
debug_statistics
()[
key
]
for
key
in
keys
]
for
s
in
net
.
servers
for
s
in
net
.
servers
...
@@ -672,9 +680,12 @@ class RaftServer(object):
...
@@ -672,9 +680,12 @@ class RaftServer(object):
def
__repr__
(
self
):
def
__repr__
(
self
):
return
'sv:{0}'
.
format
(
self
.
id
)
return
'sv:{0}'
.
format
(
self
.
id
)
def
__lt__
(
self
,
other
):
return
self
.
id
<
other
.
id
def
set_connection_status
(
self
,
new_status
):
def
set_connection_status
(
self
,
new_status
):
assert
(
not
(
self
.
connection_status
==
NODE_CONNECTED
and
new_status
==
NODE_CONNECTING
))
assert
(
not
(
self
.
connection_status
==
NODE_CONNECTED
and
new_status
==
NODE_CONNECTING
))
# logg
ing
.warning('{}: {} -> {}'.format(
# logg
er
.warning('{}: {} -> {}'.format(
# self,
# self,
# connectstatus2str(self.connection_status),
# connectstatus2str(self.connection_status),
# connectstatus2str(new_status)))
# connectstatus2str(new_status)))
...
@@ -685,10 +696,10 @@ class RaftServer(object):
...
@@ -685,10 +696,10 @@ class RaftServer(object):
return
[(
i
+
first_idx
,
l
.
term
,
l
.
id
)
for
i
,
l
in
enumerate
(
self
.
fsm_log
)]
return
[(
i
+
first_idx
,
l
.
term
,
l
.
id
)
for
i
,
l
in
enumerate
(
self
.
fsm_log
)]
def
do_compaction
(
self
):
def
do_compaction
(
self
):
# logg
ing
.warning('{} snapshotting'.format(self))
# logg
er
.warning('{} snapshotting'.format(self))
# entries_before = lib.raft_get_log_count(self.raft)
# entries_before = lib.raft_get_log_count(self.raft)
e
=
lib
.
raft_begin_snapshot
(
self
.
raft
)
e
=
lib
.
raft_begin_snapshot
(
self
.
raft
,
0
)
if
e
!=
0
:
if
e
!=
0
:
return
return
...
@@ -706,7 +717,7 @@ class RaftServer(object):
...
@@ -706,7 +717,7 @@ class RaftServer(object):
self
.
network
.
num_compactions
+=
1
self
.
network
.
num_compactions
+=
1
# logg
ing
.warning('{} entries compacted {}'.format(
# logg
er
.warning('{} entries compacted {}'.format(
# self,
# self,
# entries_before - lib.raft_get_log_count(self.raft)
# entries_before - lib.raft_get_log_count(self.raft)
# ))
# ))
...
@@ -728,7 +739,7 @@ class RaftServer(object):
...
@@ -728,7 +739,7 @@ class RaftServer(object):
raise
self
.
abort_exception
raise
self
.
abort_exception
def
shutdown
(
self
):
def
shutdown
(
self
):
# logg
ing
.error('{} shutting down'.format(self))
# logg
er
.error('{} shutting down'.format(self))
self
.
set_connection_status
(
NODE_DISCONNECTED
)
self
.
set_connection_status
(
NODE_DISCONNECTED
)
self
.
network
.
remove_server
(
self
)
self
.
network
.
remove_server
(
self
)
...
@@ -747,7 +758,7 @@ class RaftServer(object):
...
@@ -747,7 +758,7 @@ class RaftServer(object):
self
.
raft_logentry_pop
=
ffi
.
callback
(
"int(raft_server_t*, void*, raft_entry_t*, raft_index_t)"
,
raft_logentry_pop
)
self
.
raft_logentry_pop
=
ffi
.
callback
(
"int(raft_server_t*, void*, raft_entry_t*, raft_index_t)"
,
raft_logentry_pop
)
self
.
raft_logentry_get_node_id
=
ffi
.
callback
(
"int(raft_server_t*, void*, raft_entry_t*, raft_index_t)"
,
raft_logentry_get_node_id
)
self
.
raft_logentry_get_node_id
=
ffi
.
callback
(
"int(raft_server_t*, void*, raft_entry_t*, raft_index_t)"
,
raft_logentry_get_node_id
)
self
.
raft_node_has_sufficient_logs
=
ffi
.
callback
(
"int(raft_server_t* raft, void *user_data, raft_node_t* node)"
,
raft_node_has_sufficient_logs
)
self
.
raft_node_has_sufficient_logs
=
ffi
.
callback
(
"int(raft_server_t* raft, void *user_data, raft_node_t* node)"
,
raft_node_has_sufficient_logs
)
self
.
raft_notify_membership_event
=
ffi
.
callback
(
"void(raft_server_t* raft, void *user_data, raft_node_t* node, raft_membership_e)"
,
raft_notify_membership_event
)
self
.
raft_notify_membership_event
=
ffi
.
callback
(
"void(raft_server_t* raft, void *user_data, raft_node_t* node,
raft_entry_t* ety,
raft_membership_e)"
,
raft_notify_membership_event
)
self
.
raft_log
=
ffi
.
callback
(
"void(raft_server_t*, raft_node_t*, void*, const char* buf)"
,
raft_log
)
self
.
raft_log
=
ffi
.
callback
(
"void(raft_server_t*, raft_node_t*, void*, const char* buf)"
,
raft_log
)
def
recv_entry
(
self
,
ety
):
def
recv_entry
(
self
,
ety
):
...
@@ -794,16 +805,16 @@ class RaftServer(object):
...
@@ -794,16 +805,16 @@ class RaftServer(object):
except
Exception
as
e
:
except
Exception
as
e
:
ety1
=
lib
.
raft_get_entry_from_idx
(
self
.
raft
,
idx
)
ety1
=
lib
.
raft_get_entry_from_idx
(
self
.
raft
,
idx
)
ety2
=
lib
.
raft_get_entry_from_idx
(
server
.
raft
,
idx
)
ety2
=
lib
.
raft_get_entry_from_idx
(
server
.
raft
,
idx
)
logg
ing
.
error
(
'ids'
,
ety1
.
id
,
ety2
.
id
)
logg
er
.
error
(
'ids'
,
ety1
.
id
,
ety2
.
id
)
logg
ing
.
error
(
'{0}vs{1} idx:{2} terms:{3} {4} ids:{5} {6}'
.
format
(
logg
er
.
error
(
'{0}vs{1} idx:{2} terms:{3} {4} ids:{5} {6}'
.
format
(
self
,
server
,
self
,
server
,
idx
,
idx
,
our_log
.
term
,
their_log
.
term
,
our_log
.
term
,
their_log
.
term
,
our_log
.
id
,
their_log
.
id
))
our_log
.
id
,
their_log
.
id
))
self
.
abort_exception
=
e
self
.
abort_exception
=
e
logg
ing
.
error
(
self
.
debug_log
())
logg
er
.
error
(
self
.
debug_log
())
logg
ing
.
error
(
server
.
debug_log
())
logg
er
.
error
(
server
.
debug_log
())
return
lib
.
RAFT_ERR_SHUTDOWN
return
lib
.
RAFT_ERR_SHUTDOWN
def
entry_apply
(
self
,
ety
,
idx
):
def
entry_apply
(
self
,
ety
,
idx
):
...
@@ -823,7 +834,7 @@ class RaftServer(object):
...
@@ -823,7 +834,7 @@ class RaftServer(object):
elif
ety
.
type
==
lib
.
RAFT_LOGTYPE_DEMOTE_NODE
:
elif
ety
.
type
==
lib
.
RAFT_LOGTYPE_DEMOTE_NODE
:
if
change
.
node_id
==
lib
.
raft_get_nodeid
(
self
.
raft
):
if
change
.
node_id
==
lib
.
raft_get_nodeid
(
self
.
raft
):
# logg
ing
.warning("{} shutting down because of demotion".format(self))
# logg
er
.warning("{} shutting down because of demotion".format(self))
return
lib
.
RAFT_ERR_SHUTDOWN
return
lib
.
RAFT_ERR_SHUTDOWN
# Follow up by removing the node by receiving new entry
# Follow up by removing the node by receiving new entry
...
@@ -840,7 +851,7 @@ class RaftServer(object):
...
@@ -840,7 +851,7 @@ class RaftServer(object):
elif
ety
.
type
==
lib
.
RAFT_LOGTYPE_REMOVE_NODE
:
elif
ety
.
type
==
lib
.
RAFT_LOGTYPE_REMOVE_NODE
:
if
change
.
node_id
==
lib
.
raft_get_nodeid
(
self
.
raft
):
if
change
.
node_id
==
lib
.
raft_get_nodeid
(
self
.
raft
):
# logg
ing
.warning("{} shutting down because of removal".format(self))
# logg
er
.warning("{} shutting down because of removal".format(self))
return
lib
.
RAFT_ERR_SHUTDOWN
return
lib
.
RAFT_ERR_SHUTDOWN
elif
ety
.
type
==
lib
.
RAFT_LOGTYPE_ADD_NODE
:
elif
ety
.
type
==
lib
.
RAFT_LOGTYPE_ADD_NODE
:
...
@@ -864,7 +875,7 @@ class RaftServer(object):
...
@@ -864,7 +875,7 @@ class RaftServer(object):
SnapshotMember
(
id
,
lib
.
raft_node_is_voting_committed
(
n
)))
SnapshotMember
(
id
,
lib
.
raft_node_is_voting_committed
(
n
)))
def
load_snapshot
(
self
,
snapshot
,
other
):
def
load_snapshot
(
self
,
snapshot
,
other
):
# logg
ing
.warning('{} loading snapshot'.format(self))
# logg
er
.warning('{} loading snapshot'.format(self))
e
=
lib
.
raft_begin_load_snapshot
(
e
=
lib
.
raft_begin_load_snapshot
(
self
.
raft
,
self
.
raft
,
snapshot
.
last_term
,
snapshot
.
last_term
,
...
@@ -935,8 +946,8 @@ class RaftServer(object):
...
@@ -935,8 +946,8 @@ class RaftServer(object):
self
.
fsm_dict
=
dict
(
snapshot
.
image
)
self
.
fsm_dict
=
dict
(
snapshot
.
image
)
logg
ing
.
warning
(
'{} loaded snapshot t:{} idx:{}'
.
format
(
#
logg
er
.warning('{} loaded snapshot t:{} idx:{}'.format(
self
,
snapshot
.
last_term
,
snapshot
.
last_idx
))
#
self, snapshot.last_term, snapshot.last_idx))
def
send_snapshot
(
self
,
node
):
def
send_snapshot
(
self
,
node
):
assert
not
lib
.
raft_snapshot_is_in_progress
(
self
.
raft
)
assert
not
lib
.
raft_snapshot_is_in_progress
(
self
.
raft
)
...
@@ -981,7 +992,7 @@ class RaftServer(object):
...
@@ -981,7 +992,7 @@ class RaftServer(object):
other_id
=
prev_ety
.
id
other_id
=
prev_ety
.
id
assert
other_id
<
ety
.
id
assert
other_id
<
ety
.
id
except
Exception
as
e
:
except
Exception
as
e
:
logg
ing
.
error
(
other_id
,
ety
.
id
)
logg
er
.
error
(
other_id
,
ety
.
id
)
self
.
abort_exception
=
e
self
.
abort_exception
=
e
raise
raise
...
@@ -1015,7 +1026,7 @@ class RaftServer(object):
...
@@ -1015,7 +1026,7 @@ class RaftServer(object):
return
0
return
0
def
entry_pop
(
self
,
ety
,
ety_idx
):
def
entry_pop
(
self
,
ety
,
ety_idx
):
# logg
ing
.warning("POP {} {}".format(self, ety_idx))
# logg
er
.warning("POP {} {}".format(self, ety_idx))
e
=
self
.
_check_committed_entry_popping
(
ety_idx
)
e
=
self
.
_check_committed_entry_popping
(
ety_idx
)
if
e
!=
0
:
if
e
!=
0
:
...
@@ -1034,7 +1045,7 @@ class RaftServer(object):
...
@@ -1034,7 +1045,7 @@ class RaftServer(object):
elif
ety
.
type
==
lib
.
RAFT_LOGTYPE_ADD_NONVOTING_NODE
:
elif
ety
.
type
==
lib
.
RAFT_LOGTYPE_ADD_NONVOTING_NODE
:
if
change
.
node_id
==
lib
.
raft_get_nodeid
(
self
.
raft
):
if
change
.
node_id
==
lib
.
raft_get_nodeid
(
self
.
raft
):
logg
ing
.
error
(
"POP disconnect {} {}"
.
format
(
self
,
ety_idx
))
logg
er
.
error
(
"POP disconnect {} {}"
.
format
(
self
,
ety_idx
))
self
.
set_connection_status
(
NODE_DISCONNECTED
)
self
.
set_connection_status
(
NODE_DISCONNECTED
)
elif
ety
.
type
==
lib
.
RAFT_LOGTYPE_ADD_NODE
:
elif
ety
.
type
==
lib
.
RAFT_LOGTYPE_ADD_NODE
:
...
@@ -1061,7 +1072,7 @@ class RaftServer(object):
...
@@ -1061,7 +1072,7 @@ class RaftServer(object):
assert
e
==
0
assert
e
==
0
return
0
return
0
def
notify_membership_event
(
self
,
node
,
event_type
):
def
notify_membership_event
(
self
,
node
,
ety
,
event_type
):
# Convenience: Ensure that added node has udata set
# Convenience: Ensure that added node has udata set
if
event_type
==
lib
.
RAFT_MEMBERSHIP_ADD
:
if
event_type
==
lib
.
RAFT_MEMBERSHIP_ADD
:
node_id
=
lib
.
raft_node_get_id
(
node
)
node_id
=
lib
.
raft_node_get_id
(
node
)
...
@@ -1100,10 +1111,19 @@ if __name__ == '__main__':
...
@@ -1100,10 +1111,19 @@ if __name__ == '__main__':
if
args
[
'--verbose'
]
or
args
[
'--log_level'
]:
if
args
[
'--verbose'
]
or
args
[
'--log_level'
]:
coloredlogs
.
install
(
fmt
=
'%(asctime)s %(levelname)s %(message)s'
)
coloredlogs
.
install
(
fmt
=
'%(asctime)s %(levelname)s %(message)s'
)
level
=
logging
.
DEBUG
if
args
[
'--log_level'
]:
if
args
[
'--log_level'
]:
level
=
int
(
args
[
'--log_level'
])
level
=
int
(
args
[
'--log_level'
])
logging
.
basicConfig
(
level
=
level
,
format
=
'%(asctime)s %(message)s'
)
else
:
level
=
logging
.
DEBUG
logger
.
setLevel
(
level
)
if
args
[
'--log_file'
]:
fh
=
logging
.
FileHandler
(
args
[
'--log_file'
])
fh
.
setLevel
(
level
)
logger
.
addHandler
(
fh
)
logger
.
propagate
=
False
net
=
Network
(
int
(
args
[
'--seed'
]))
net
=
Network
(
int
(
args
[
'--seed'
]))
...
@@ -1135,4 +1155,8 @@ if __name__ == '__main__':
...
@@ -1135,4 +1155,8 @@ if __name__ == '__main__':
# print(server, [l.term for l in server.fsm_log])
# print(server, [l.term for l in server.fsm_log])
raise
raise
net
.
diagnotistic_info
()
if
args
[
'--json_output'
]:
pass
if
not
args
[
'--quiet'
]:
net
.
diagnotistic_info
()
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment