Unverified Commit a6b5d518 authored by Madelyn Olson's avatar Madelyn Olson Committed by GitHub
Browse files

Improved the reliability of cluster replica sync tests (#9628)

Improved the reliability of cluster replica sync tests
parent 075ac345
...@@ -2694,6 +2694,7 @@ standardConfig configs[] = { ...@@ -2694,6 +2694,7 @@ standardConfig configs[] = {
/* Other configs */ /* Other configs */
createTimeTConfig("repl-backlog-ttl", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.repl_backlog_time_limit, 60*60, INTEGER_CONFIG, NULL, NULL), /* Default: 1 hour */ createTimeTConfig("repl-backlog-ttl", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.repl_backlog_time_limit, 60*60, INTEGER_CONFIG, NULL, NULL), /* Default: 1 hour */
createOffTConfig("auto-aof-rewrite-min-size", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.aof_rewrite_min_size, 64*1024*1024, MEMORY_CONFIG, NULL, NULL), createOffTConfig("auto-aof-rewrite-min-size", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.aof_rewrite_min_size, 64*1024*1024, MEMORY_CONFIG, NULL, NULL),
createOffTConfig("loading-process-events-interval-bytes", NULL, MODIFIABLE_CONFIG, 1024, INT_MAX, server.loading_process_events_interval_bytes, 1024*1024*2, INTEGER_CONFIG, NULL, NULL),
#ifdef USE_OPENSSL #ifdef USE_OPENSSL
createIntConfig("tls-port", NULL, MODIFIABLE_CONFIG, 0, 65535, server.tls_port, 0, INTEGER_CONFIG, NULL, updateTLSPort), /* TCP port. */ createIntConfig("tls-port", NULL, MODIFIABLE_CONFIG, 0, 65535, server.tls_port, 0, INTEGER_CONFIG, NULL, updateTLSPort), /* TCP port. */
......
...@@ -3198,7 +3198,6 @@ void initServerConfig(void) { ...@@ -3198,7 +3198,6 @@ void initServerConfig(void) {
server.cluster_module_flags = CLUSTER_MODULE_FLAG_NONE; server.cluster_module_flags = CLUSTER_MODULE_FLAG_NONE;
server.migrate_cached_sockets = dictCreate(&migrateCacheDictType); server.migrate_cached_sockets = dictCreate(&migrateCacheDictType);
server.next_client_id = 1; /* Client IDs, start from 1 .*/ server.next_client_id = 1; /* Client IDs, start from 1 .*/
server.loading_process_events_interval_bytes = (1024*1024*2);
server.page_size = sysconf(_SC_PAGESIZE); server.page_size = sysconf(_SC_PAGESIZE);
server.pause_cron = 0; server.pause_cron = 0;
......
...@@ -27,26 +27,25 @@ proc is_replica_online {info_repl} { ...@@ -27,26 +27,25 @@ proc is_replica_online {info_repl} {
set master_id 0 set master_id 0
test "Fill up" { test "Fill up primary with data" {
R $master_id debug populate 10000000 key 100 # Set 1 MB of data
R $master_id debug populate 1000 key 1000
} }
test "Add new node as replica" { test "Add new node as replica" {
set replica_id [cluster_find_available_slave 1] set replica_id 1
set master_myself [get_myself $master_id] set replica [R $replica_id CLUSTER MYID]
set replica_myself [get_myself $replica_id] R $replica_id cluster replicate [R $master_id CLUSTER MYID]
set replica [dict get $replica_myself id]
R $replica_id cluster replicate [dict get $master_myself id]
} }
test "Check digest and replica state" { test "Check digest and replica state" {
R 1 readonly
wait_for_condition 1000 50 { wait_for_condition 1000 50 {
[is_in_slots $master_id $replica] [is_in_slots $master_id $replica]
} else { } else {
fail "New replica didn't appear in the slots" fail "New replica didn't appear in the slots"
} }
wait_for_condition 1000 50 {
wait_for_condition 100 50 {
[is_replica_online [R $master_id info replication]] [is_replica_online [R $master_id info replication]]
} else { } else {
fail "Replica is down for too long" fail "Replica is down for too long"
...@@ -57,10 +56,20 @@ test "Check digest and replica state" { ...@@ -57,10 +56,20 @@ test "Check digest and replica state" {
test "Replica in loading state is hidden" { test "Replica in loading state is hidden" {
# Kill replica client for master and load new data to the primary # Kill replica client for master and load new data to the primary
R $master_id multi
R $master_id config set repl-backlog-size 100 R $master_id config set repl-backlog-size 100
# Set the key load delay so that it will take at least
# 2 seconds to fully load the data.
R $replica_id config set key-load-delay 4000
# Trigger event loop processing every 1024 bytes, this trigger
# allows us to send and receive cluster messages, so we are setting
# it low so that the cluster messages are sent more frequently.
R $replica_id config set loading-process-events-interval-bytes 1024
R $master_id multi
R $master_id client kill type replica R $master_id client kill type replica
set num 10000 set num 100
set value [string repeat A 1024] set value [string repeat A 1024]
for {set j 0} {$j < $num} {incr j} { for {set j 0} {$j < $num} {incr j} {
set key "{0}" set key "{0}"
...@@ -69,28 +78,34 @@ test "Replica in loading state is hidden" { ...@@ -69,28 +78,34 @@ test "Replica in loading state is hidden" {
} }
R $master_id exec R $master_id exec
# Check that replica started loading # The master will be the last to know the replica
wait_for_condition 1000 50 { # is loading, so we will wait on that and assert
[s $replica_id loading] eq 1 # the replica is loading afterwards.
wait_for_condition 100 50 {
![is_in_slots $master_id $replica]
} else { } else {
fail "Replica didn't enter loading state" fail "Replica was always present in cluster slots"
} }
# Check that replica is not in cluster slots assert_equal 1 [s $replica_id loading]
assert {![is_in_slots $master_id $replica]}
# Wait for sync to finish # Wait for the replica to finish full-sync and become online
wait_for_condition 1000 50 { wait_for_condition 200 50 {
[s $replica_id loading] eq 0 [s $replica_id master_link_status] eq "up"
} else { } else {
fail "Replica is in loading state for too long" fail "Replica didn't finish loading"
} }
# Check replica is back to cluster slots # Return configs to default values
wait_for_condition 1000 50 { R $replica_id config set loading-process-events-interval-bytes 2097152
R $replica_id config set key-load-delay 0
# Check replica is back in cluster slots
wait_for_condition 100 50 {
[is_in_slots $master_id $replica] [is_in_slots $master_id $replica]
} else { } else {
fail "Replica is not back to slots" fail "Replica is not back to slots"
} }
assert_equal 1 [is_in_slots $replica_id $replica]
} }
test "Check disconnected replica not hidden from slots" { test "Check disconnected replica not hidden from slots" {
......
...@@ -39,6 +39,8 @@ test "Cluster nodes hard reset" { ...@@ -39,6 +39,8 @@ test "Cluster nodes hard reset" {
R $id EXEC R $id EXEC
R $id config set cluster-node-timeout $node_timeout R $id config set cluster-node-timeout $node_timeout
R $id config set cluster-slave-validity-factor 10 R $id config set cluster-slave-validity-factor 10
R $id config set loading-process-events-interval-bytes 2097152
R $id config set key-load-delay 0
R $id config rewrite R $id config rewrite
} }
} }
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment