Unverified Commit 997fa41e authored by Oran Agra's avatar Oran Agra Committed by GitHub
Browse files

Attempt to solve MacOS CI issues in GH Actions (#12013)

The MacOS CI in github actions often hangs without any logs. GH argues that
it's due to resource utilization, either running out of disk space, memory, or CPU
starvation, and thus the runner is terminated.

This PR contains multiple attempts to resolve this:
1. introducing pause_process instead of SIGSTOP, which waits for the process
  to stop before resuming the test, possibly resolving race conditions in some tests,
  this was a suspect since there was one test that could result in an infinite loop in that
 case, in practice this didn't help, but still a good idea to keep.
2. disable the `save` config in many tests that don't need it, specifically ones that use
  heavy writes and could create large files.
3. change the `populate` proc to use short pipeline rather than an infinite one.
4. use `--clients 1` in the macos CI so that we don't risk running multiple resource
  demanding tests in parallel.
5. enable `--verbose` to be repeated to elevate verbosity and print more info to stdout
  when a test or a server starts.
parent 45b8eea1
...@@ -698,10 +698,10 @@ jobs: ...@@ -698,10 +698,10 @@ jobs:
run: make REDIS_CFLAGS='-Werror' run: make REDIS_CFLAGS='-Werror'
- name: test - name: test
if: true && !contains(github.event.inputs.skiptests, 'redis') if: true && !contains(github.event.inputs.skiptests, 'redis')
run: ./runtest --accurate --verbose --no-latency --dump-logs ${{github.event.inputs.test_args}} run: ./runtest --accurate --verbose --verbose --clients 1 --no-latency --dump-logs ${{github.event.inputs.test_args}}
- name: module api test - name: module api test
if: true && !contains(github.event.inputs.skiptests, 'modules') if: true && !contains(github.event.inputs.skiptests, 'modules')
run: ./runtest-moduleapi --verbose --no-latency --dump-logs ${{github.event.inputs.test_args}} run: ./runtest-moduleapi --verbose --verbose --clients 1 --no-latency --dump-logs ${{github.event.inputs.test_args}}
test-macos-latest-sentinel: test-macos-latest-sentinel:
runs-on: macos-latest runs-on: macos-latest
......
...@@ -755,7 +755,7 @@ tags {"external:skip"} { ...@@ -755,7 +755,7 @@ tags {"external:skip"} {
# writing pressure, etc. # writing pressure, etc.
start_server {tags {"Multi Part AOF"} overrides {aof-use-rdb-preamble {yes} appendonly {no}}} { start_server {tags {"Multi Part AOF"} overrides {aof-use-rdb-preamble {yes} appendonly {no} save {}}} {
set dir [get_redis_dir] set dir [get_redis_dir]
set aof_basename "appendonly.aof" set aof_basename "appendonly.aof"
set aof_dirname "appendonlydir" set aof_dirname "appendonlydir"
...@@ -1173,7 +1173,7 @@ tags {"external:skip"} { ...@@ -1173,7 +1173,7 @@ tags {"external:skip"} {
assert {$d1 eq $d2} assert {$d1 eq $d2}
} }
start_server {overrides {aof-use-rdb-preamble {yes} appendonly {no}}} { start_server {overrides {aof-use-rdb-preamble {yes} appendonly {no} save {}}} {
set dir [get_redis_dir] set dir [get_redis_dir]
set aof_basename "appendonly.aof" set aof_basename "appendonly.aof"
set aof_dirname "appendonlydir" set aof_dirname "appendonlydir"
......
...@@ -12,7 +12,7 @@ proc stop_bg_block_op {handle} { ...@@ -12,7 +12,7 @@ proc stop_bg_block_op {handle} {
} }
start_server {tags {"repl" "external:skip"}} { start_server {tags {"repl" "external:skip"}} {
start_server {} { start_server {overrides {save {}}} {
set master [srv -1 client] set master [srv -1 client]
set master_host [srv -1 host] set master_host [srv -1 host]
set master_port [srv -1 port] set master_port [srv -1 port]
......
start_server {tags {"failover external:skip"}} { start_server {tags {"failover external:skip"} overrides {save {}}} {
start_server {} { start_server {overrides {save {}}} {
start_server {} { start_server {overrides {save {}}} {
set node_0 [srv 0 client] set node_0 [srv 0 client]
set node_0_host [srv 0 host] set node_0_host [srv 0 host]
set node_0_port [srv 0 port] set node_0_port [srv 0 port]
...@@ -66,13 +66,13 @@ start_server {} { ...@@ -66,13 +66,13 @@ start_server {} {
# Generate a delta between primary and replica # Generate a delta between primary and replica
set load_handler [start_write_load $node_0_host $node_0_port 5] set load_handler [start_write_load $node_0_host $node_0_port 5]
exec kill -SIGSTOP [srv -1 pid] pause_process [srv -1 pid]
wait_for_condition 50 100 { wait_for_condition 50 100 {
[s 0 total_commands_processed] > 100 [s 0 total_commands_processed] > 100
} else { } else {
fail "Node 0 did not accept writes" fail "Node 0 did not accept writes"
} }
exec kill -SIGCONT [srv -1 pid] resume_process [srv -1 pid]
# Execute the failover # Execute the failover
$node_0 failover to $node_1_host $node_1_port $node_0 failover to $node_1_host $node_1_port
...@@ -108,7 +108,7 @@ start_server {} { ...@@ -108,7 +108,7 @@ start_server {} {
wait_for_ofs_sync $node_1 $node_2 wait_for_ofs_sync $node_1 $node_2
# We stop node 0 to and make sure node 2 is selected # We stop node 0 to and make sure node 2 is selected
exec kill -SIGSTOP $node_0_pid pause_process $node_0_pid
$node_1 set CASE 1 $node_1 set CASE 1
$node_1 FAILOVER $node_1 FAILOVER
...@@ -118,7 +118,7 @@ start_server {} { ...@@ -118,7 +118,7 @@ start_server {} {
} else { } else {
fail "Failover from node 1 to node 2 did not finish" fail "Failover from node 1 to node 2 did not finish"
} }
exec kill -SIGCONT $node_0_pid resume_process $node_0_pid
$node_0 replicaof $node_2_host $node_2_port $node_0 replicaof $node_2_host $node_2_port
wait_for_sync $node_0 wait_for_sync $node_0
...@@ -138,7 +138,7 @@ start_server {} { ...@@ -138,7 +138,7 @@ start_server {} {
set initial_psyncs [s 0 sync_partial_ok] set initial_psyncs [s 0 sync_partial_ok]
set initial_syncs [s 0 sync_full] set initial_syncs [s 0 sync_full]
exec kill -SIGSTOP $node_0_pid pause_process $node_0_pid
# node 0 will never acknowledge this write # node 0 will never acknowledge this write
$node_2 set case 2 $node_2 set case 2
$node_2 failover to $node_0_host $node_0_port TIMEOUT 100 FORCE $node_2 failover to $node_0_host $node_0_port TIMEOUT 100 FORCE
...@@ -155,7 +155,7 @@ start_server {} { ...@@ -155,7 +155,7 @@ start_server {} {
assert_match *slave* [$node_1 role] assert_match *slave* [$node_1 role]
assert_match *slave* [$node_2 role] assert_match *slave* [$node_2 role]
exec kill -SIGCONT $node_0_pid resume_process $node_0_pid
# Wait for failover to end # Wait for failover to end
wait_for_condition 50 100 { wait_for_condition 50 100 {
...@@ -186,7 +186,7 @@ start_server {} { ...@@ -186,7 +186,7 @@ start_server {} {
set initial_syncs [s 0 sync_full] set initial_syncs [s 0 sync_full]
# Stop replica so it never catches up # Stop replica so it never catches up
exec kill -SIGSTOP [srv -1 pid] pause_process [srv -1 pid]
$node_0 SET CASE 1 $node_0 SET CASE 1
$node_0 failover to [srv -1 host] [srv -1 port] TIMEOUT 500 $node_0 failover to [srv -1 host] [srv -1 port] TIMEOUT 500
...@@ -197,7 +197,7 @@ start_server {} { ...@@ -197,7 +197,7 @@ start_server {} {
fail "Failover from node_0 to replica did not finish" fail "Failover from node_0 to replica did not finish"
} }
exec kill -SIGCONT [srv -1 pid] resume_process [srv -1 pid]
# We need to make sure the nodes actually sync back up # We need to make sure the nodes actually sync back up
wait_for_ofs_sync $node_0 $node_1 wait_for_ofs_sync $node_0 $node_1
...@@ -218,7 +218,7 @@ start_server {} { ...@@ -218,7 +218,7 @@ start_server {} {
set initial_syncs [s 0 sync_full] set initial_syncs [s 0 sync_full]
# Stop replica so it never catches up # Stop replica so it never catches up
exec kill -SIGSTOP [srv -1 pid] pause_process [srv -1 pid]
$node_0 SET CASE 2 $node_0 SET CASE 2
$node_0 failover to [srv -1 host] [srv -1 port] TIMEOUT 60000 $node_0 failover to [srv -1 host] [srv -1 port] TIMEOUT 60000
...@@ -230,7 +230,7 @@ start_server {} { ...@@ -230,7 +230,7 @@ start_server {} {
$node_0 failover abort $node_0 failover abort
assert_match [s 0 master_failover_state] "no-failover" assert_match [s 0 master_failover_state] "no-failover"
exec kill -SIGCONT [srv -1 pid] resume_process [srv -1 pid]
# Just make sure everything is still synced # Just make sure everything is still synced
wait_for_ofs_sync $node_0 $node_1 wait_for_ofs_sync $node_0 $node_1
...@@ -255,11 +255,11 @@ start_server {} { ...@@ -255,11 +255,11 @@ start_server {} {
# We pause the target long enough to send a write command # We pause the target long enough to send a write command
# during the pause. This write will not be interrupted. # during the pause. This write will not be interrupted.
exec kill -SIGSTOP [srv -1 pid] pause_process [srv -1 pid]
set rd [redis_deferring_client] set rd [redis_deferring_client]
$rd SET FOO BAR $rd SET FOO BAR
$node_0 failover to $node_1_host $node_1_port $node_0 failover to $node_1_host $node_1_port
exec kill -SIGCONT [srv -1 pid] resume_process [srv -1 pid]
# Wait for failover to end # Wait for failover to end
wait_for_condition 50 100 { wait_for_condition 50 100 {
......
...@@ -42,7 +42,7 @@ start_server {tags {"repl external:skip"}} { ...@@ -42,7 +42,7 @@ start_server {tags {"repl external:skip"}} {
test {No write if min-slaves-max-lag is > of the slave lag} { test {No write if min-slaves-max-lag is > of the slave lag} {
r config set min-slaves-to-write 1 r config set min-slaves-to-write 1
r config set min-slaves-max-lag 2 r config set min-slaves-max-lag 2
exec kill -SIGSTOP [srv -1 pid] pause_process [srv -1 pid]
assert {[r set foo 12345] eq {OK}} assert {[r set foo 12345] eq {OK}}
wait_for_condition 100 100 { wait_for_condition 100 100 {
[catch {r set foo 12345}] != 0 [catch {r set foo 12345}] != 0
...@@ -52,7 +52,7 @@ start_server {tags {"repl external:skip"}} { ...@@ -52,7 +52,7 @@ start_server {tags {"repl external:skip"}} {
catch {r set foo 12345} err catch {r set foo 12345} err
assert_match {NOREPLICAS*} $err assert_match {NOREPLICAS*} $err
} }
exec kill -SIGCONT [srv -1 pid] resume_process [srv -1 pid]
test {min-slaves-to-write is ignored by slaves} { test {min-slaves-to-write is ignored by slaves} {
r config set min-slaves-to-write 1 r config set min-slaves-to-write 1
......
start_server {tags {"repl network external:skip singledb:skip"}} { start_server {tags {"repl network external:skip singledb:skip"} overrides {save {}}} {
start_server {} { start_server { overrides {save {}}} {
set master [srv -1 client] set master [srv -1 client]
set master_host [srv -1 host] set master_host [srv -1 host]
...@@ -104,7 +104,7 @@ start_server {tags {"repl external:skip"}} { ...@@ -104,7 +104,7 @@ start_server {tags {"repl external:skip"}} {
assert_equal OK [$master set foo 123] assert_equal OK [$master set foo 123]
assert_equal OK [$master eval "return redis.call('set','foo',12345)" 0] assert_equal OK [$master eval "return redis.call('set','foo',12345)" 0]
# Killing a slave to make it become a lagged slave. # Killing a slave to make it become a lagged slave.
exec kill -SIGSTOP [srv 0 pid] pause_process [srv 0 pid]
# Waiting for slave kill. # Waiting for slave kill.
wait_for_condition 100 100 { wait_for_condition 100 100 {
[catch {$master set foo 123}] != 0 [catch {$master set foo 123}] != 0
...@@ -113,7 +113,7 @@ start_server {tags {"repl external:skip"}} { ...@@ -113,7 +113,7 @@ start_server {tags {"repl external:skip"}} {
} }
assert_error "*NOREPLICAS*" {$master set foo 123} assert_error "*NOREPLICAS*" {$master set foo 123}
assert_error "*NOREPLICAS*" {$master eval "return redis.call('set','foo',12345)" 0} assert_error "*NOREPLICAS*" {$master eval "return redis.call('set','foo',12345)" 0}
exec kill -SIGCONT [srv 0 pid] resume_process [srv 0 pid]
} }
} }
} }
...@@ -146,12 +146,12 @@ start_server {tags {"repl external:skip"}} { ...@@ -146,12 +146,12 @@ start_server {tags {"repl external:skip"}} {
$master debug set-active-expire 0 $master debug set-active-expire 0
$master set k 1 px $px_ms $master set k 1 px $px_ms
wait_for_ofs_sync $master $slave wait_for_ofs_sync $master $slave
exec kill -SIGSTOP [srv 0 pid] pause_process [srv 0 pid]
$master incr k $master incr k
after [expr $px_ms + 1] after [expr $px_ms + 1]
# Stopping the replica for one second to makes sure the INCR arrives # Stopping the replica for one second to makes sure the INCR arrives
# to the replica after the key is logically expired. # to the replica after the key is logically expired.
exec kill -SIGCONT [srv 0 pid] resume_process [srv 0 pid]
wait_for_ofs_sync $master $slave wait_for_ofs_sync $master $slave
# Check that k is logically expired but is present in the replica. # Check that k is logically expired but is present in the replica.
set res [$slave exists k] set res [$slave exists k]
......
...@@ -159,7 +159,7 @@ start_server {} { ...@@ -159,7 +159,7 @@ start_server {} {
assert {[s repl_backlog_histlen] > [expr 2*10000*10000]} assert {[s repl_backlog_histlen] > [expr 2*10000*10000]}
assert_equal [s connected_slaves] {2} assert_equal [s connected_slaves] {2}
exec kill -SIGSTOP $replica2_pid pause_process $replica2_pid
r config set client-output-buffer-limit "replica 128k 0 0" r config set client-output-buffer-limit "replica 128k 0 0"
# trigger output buffer limit check # trigger output buffer limit check
r set key [string repeat A [expr 64*1024]] r set key [string repeat A [expr 64*1024]]
...@@ -178,7 +178,7 @@ start_server {} { ...@@ -178,7 +178,7 @@ start_server {} {
} else { } else {
fail "Replication backlog memory is not smaller" fail "Replication backlog memory is not smaller"
} }
exec kill -SIGCONT $replica2_pid resume_process $replica2_pid
} }
# speed up termination # speed up termination
$master config set shutdown-timeout 0 $master config set shutdown-timeout 0
......
...@@ -9,8 +9,8 @@ ...@@ -9,8 +9,8 @@
# reconnect with the master, otherwise just the initial synchronization is # reconnect with the master, otherwise just the initial synchronization is
# checked for consistency. # checked for consistency.
proc test_psync {descr duration backlog_size backlog_ttl delay cond mdl sdl reconnect} { proc test_psync {descr duration backlog_size backlog_ttl delay cond mdl sdl reconnect} {
start_server {tags {"repl"}} { start_server {tags {"repl"} overrides {save {}}} {
start_server {} { start_server {overrides {save {}}} {
set master [srv -1 client] set master [srv -1 client]
set master_host [srv -1 host] set master_host [srv -1 host]
......
...@@ -302,7 +302,7 @@ start_server {tags {"repl external:skip"}} { ...@@ -302,7 +302,7 @@ start_server {tags {"repl external:skip"}} {
foreach mdl {no yes} { foreach mdl {no yes} {
foreach sdl {disabled swapdb} { foreach sdl {disabled swapdb} {
start_server {tags {"repl external:skip"}} { start_server {tags {"repl external:skip"} overrides {save {}}} {
set master [srv 0 client] set master [srv 0 client]
$master config set repl-diskless-sync $mdl $master config set repl-diskless-sync $mdl
$master config set repl-diskless-sync-delay 5 $master config set repl-diskless-sync-delay 5
...@@ -310,11 +310,11 @@ foreach mdl {no yes} { ...@@ -310,11 +310,11 @@ foreach mdl {no yes} {
set master_host [srv 0 host] set master_host [srv 0 host]
set master_port [srv 0 port] set master_port [srv 0 port]
set slaves {} set slaves {}
start_server {} { start_server {overrides {save {}}} {
lappend slaves [srv 0 client] lappend slaves [srv 0 client]
start_server {} { start_server {overrides {save {}}} {
lappend slaves [srv 0 client] lappend slaves [srv 0 client]
start_server {} { start_server {overrides {save {}}} {
lappend slaves [srv 0 client] lappend slaves [srv 0 client]
test "Connect multiple replicas at the same time (issue #141), master diskless=$mdl, replica diskless=$sdl" { test "Connect multiple replicas at the same time (issue #141), master diskless=$mdl, replica diskless=$sdl" {
# start load handles only inside the test, so that the test can be skipped # start load handles only inside the test, so that the test can be skipped
...@@ -391,11 +391,11 @@ foreach mdl {no yes} { ...@@ -391,11 +391,11 @@ foreach mdl {no yes} {
} }
} }
start_server {tags {"repl external:skip"}} { start_server {tags {"repl external:skip"} overrides {save {}}} {
set master [srv 0 client] set master [srv 0 client]
set master_host [srv 0 host] set master_host [srv 0 host]
set master_port [srv 0 port] set master_port [srv 0 port]
start_server {} { start_server {overrides {save {}}} {
test "Master stream is correctly processed while the replica has a script in -BUSY state" { test "Master stream is correctly processed while the replica has a script in -BUSY state" {
set load_handle0 [start_write_load $master_host $master_port 3] set load_handle0 [start_write_load $master_host $master_port 3]
set slave [srv 0 client] set slave [srv 0 client]
...@@ -705,11 +705,11 @@ foreach testType {Successful Aborted} { ...@@ -705,11 +705,11 @@ foreach testType {Successful Aborted} {
} }
test {diskless loading short read} { test {diskless loading short read} {
start_server {tags {"repl"}} { start_server {tags {"repl"} overrides {save ""}} {
set replica [srv 0 client] set replica [srv 0 client]
set replica_host [srv 0 host] set replica_host [srv 0 host]
set replica_port [srv 0 port] set replica_port [srv 0 port]
start_server {} { start_server {overrides {save ""}} {
set master [srv 0 client] set master [srv 0 client]
set master_host [srv 0 host] set master_host [srv 0 host]
set master_port [srv 0 port] set master_port [srv 0 port]
...@@ -847,7 +847,7 @@ proc compute_cpu_usage {start end} { ...@@ -847,7 +847,7 @@ proc compute_cpu_usage {start end} {
# test diskless rdb pipe with multiple replicas, which may drop half way # test diskless rdb pipe with multiple replicas, which may drop half way
start_server {tags {"repl external:skip"}} { start_server {tags {"repl external:skip"} overrides {save ""}} {
set master [srv 0 client] set master [srv 0 client]
$master config set repl-diskless-sync yes $master config set repl-diskless-sync yes
$master config set repl-diskless-sync-delay 5 $master config set repl-diskless-sync-delay 5
...@@ -868,10 +868,10 @@ start_server {tags {"repl external:skip"}} { ...@@ -868,10 +868,10 @@ start_server {tags {"repl external:skip"}} {
set replicas {} set replicas {}
set replicas_alive {} set replicas_alive {}
# start one replica that will read the rdb fast, and one that will be slow # start one replica that will read the rdb fast, and one that will be slow
start_server {} { start_server {overrides {save ""}} {
lappend replicas [srv 0 client] lappend replicas [srv 0 client]
lappend replicas_alive [srv 0 client] lappend replicas_alive [srv 0 client]
start_server {} { start_server {overrides {save ""}} {
lappend replicas [srv 0 client] lappend replicas [srv 0 client]
lappend replicas_alive [srv 0 client] lappend replicas_alive [srv 0 client]
...@@ -913,7 +913,7 @@ start_server {tags {"repl external:skip"}} { ...@@ -913,7 +913,7 @@ start_server {tags {"repl external:skip"}} {
if {$all_drop == "timeout"} { if {$all_drop == "timeout"} {
$master config set repl-timeout 2 $master config set repl-timeout 2
# we want the slow replica to hang on a key for very long so it'll reach repl-timeout # we want the slow replica to hang on a key for very long so it'll reach repl-timeout
exec kill -SIGSTOP [srv -1 pid] pause_process [srv -1 pid]
after 2000 after 2000
} }
...@@ -940,7 +940,7 @@ start_server {tags {"repl external:skip"}} { ...@@ -940,7 +940,7 @@ start_server {tags {"repl external:skip"}} {
# master disconnected the slow replica, remove from array # master disconnected the slow replica, remove from array
set replicas_alive [lreplace $replicas_alive 0 0] set replicas_alive [lreplace $replicas_alive 0 0]
# release it # release it
exec kill -SIGCONT [srv -1 pid] resume_process [srv -1 pid]
} }
# make sure we don't have a busy loop going thought epoll_wait # make sure we don't have a busy loop going thought epoll_wait
...@@ -1000,7 +1000,7 @@ test "diskless replication child being killed is collected" { ...@@ -1000,7 +1000,7 @@ test "diskless replication child being killed is collected" {
# when diskless master is waiting for the replica to become writable # when diskless master is waiting for the replica to become writable
# it removes the read event from the rdb pipe so if the child gets killed # it removes the read event from the rdb pipe so if the child gets killed
# the replica will hung. and the master may not collect the pid with waitpid # the replica will hung. and the master may not collect the pid with waitpid
start_server {tags {"repl"}} { start_server {tags {"repl"} overrides {save ""}} {
set master [srv 0 client] set master [srv 0 client]
set master_host [srv 0 host] set master_host [srv 0 host]
set master_port [srv 0 port] set master_port [srv 0 port]
...@@ -1010,7 +1010,7 @@ test "diskless replication child being killed is collected" { ...@@ -1010,7 +1010,7 @@ test "diskless replication child being killed is collected" {
# put enough data in the db that the rdb file will be bigger than the socket buffers # put enough data in the db that the rdb file will be bigger than the socket buffers
$master debug populate 20000 test 10000 $master debug populate 20000 test 10000
$master config set rdbcompression no $master config set rdbcompression no
start_server {} { start_server {overrides {save ""}} {
set replica [srv 0 client] set replica [srv 0 client]
set loglines [count_log_lines 0] set loglines [count_log_lines 0]
$replica config set repl-diskless-load swapdb $replica config set repl-diskless-load swapdb
...@@ -1044,7 +1044,7 @@ test "diskless replication child being killed is collected" { ...@@ -1044,7 +1044,7 @@ test "diskless replication child being killed is collected" {
foreach mdl {yes no} { foreach mdl {yes no} {
test "replication child dies when parent is killed - diskless: $mdl" { test "replication child dies when parent is killed - diskless: $mdl" {
# when master is killed, make sure the fork child can detect that and exit # when master is killed, make sure the fork child can detect that and exit
start_server {tags {"repl"}} { start_server {tags {"repl"} overrides {save ""}} {
set master [srv 0 client] set master [srv 0 client]
set master_host [srv 0 host] set master_host [srv 0 host]
set master_port [srv 0 port] set master_port [srv 0 port]
...@@ -1054,7 +1054,7 @@ foreach mdl {yes no} { ...@@ -1054,7 +1054,7 @@ foreach mdl {yes no} {
# create keys that will take 10 seconds to save # create keys that will take 10 seconds to save
$master config set rdb-key-save-delay 1000 $master config set rdb-key-save-delay 1000
$master debug populate 10000 $master debug populate 10000
start_server {} { start_server {overrides {save ""}} {
set replica [srv 0 client] set replica [srv 0 client]
$replica replicaof $master_host $master_port $replica replicaof $master_host $master_port
...@@ -1085,7 +1085,7 @@ test "diskless replication read pipe cleanup" { ...@@ -1085,7 +1085,7 @@ test "diskless replication read pipe cleanup" {
# When we close this pipe (fd), the read handler also needs to be removed from the event loop (if it still registered). # When we close this pipe (fd), the read handler also needs to be removed from the event loop (if it still registered).
# Otherwise, next time we will use the same fd, the registration will be fail (panic), because # Otherwise, next time we will use the same fd, the registration will be fail (panic), because
# we will use EPOLL_CTL_MOD (the fd still register in the event loop), on fd that already removed from epoll_ctl # we will use EPOLL_CTL_MOD (the fd still register in the event loop), on fd that already removed from epoll_ctl
start_server {tags {"repl"}} { start_server {tags {"repl"} overrides {save ""}} {
set master [srv 0 client] set master [srv 0 client]
set master_host [srv 0 host] set master_host [srv 0 host]
set master_port [srv 0 port] set master_port [srv 0 port]
...@@ -1097,7 +1097,7 @@ test "diskless replication read pipe cleanup" { ...@@ -1097,7 +1097,7 @@ test "diskless replication read pipe cleanup" {
$master config set rdb-key-save-delay 100000 $master config set rdb-key-save-delay 100000
$master debug populate 20000 test 10000 $master debug populate 20000 test 10000
$master config set rdbcompression no $master config set rdbcompression no
start_server {} { start_server {overrides {save ""}} {
set replica [srv 0 client] set replica [srv 0 client]
set loglines [count_log_lines 0] set loglines [count_log_lines 0]
$replica config set repl-diskless-load swapdb $replica config set repl-diskless-load swapdb
...@@ -1122,17 +1122,17 @@ test "diskless replication read pipe cleanup" { ...@@ -1122,17 +1122,17 @@ test "diskless replication read pipe cleanup" {
test {replicaof right after disconnection} { test {replicaof right after disconnection} {
# this is a rare race condition that was reproduced sporadically by the psync2 unit. # this is a rare race condition that was reproduced sporadically by the psync2 unit.
# see details in #7205 # see details in #7205
start_server {tags {"repl"}} { start_server {tags {"repl"} overrides {save ""}} {
set replica1 [srv 0 client] set replica1 [srv 0 client]
set replica1_host [srv 0 host] set replica1_host [srv 0 host]
set replica1_port [srv 0 port] set replica1_port [srv 0 port]
set replica1_log [srv 0 stdout] set replica1_log [srv 0 stdout]
start_server {} { start_server {overrides {save ""}} {
set replica2 [srv 0 client] set replica2 [srv 0 client]
set replica2_host [srv 0 host] set replica2_host [srv 0 host]
set replica2_port [srv 0 port] set replica2_port [srv 0 port]
set replica2_log [srv 0 stdout] set replica2_log [srv 0 stdout]
start_server {} { start_server {overrides {save ""}} {
set master [srv 0 client] set master [srv 0 client]
set master_host [srv 0 host] set master_host [srv 0 host]
set master_port [srv 0 port] set master_port [srv 0 port]
......
...@@ -19,8 +19,8 @@ proc fill_up_os_socket_send_buffer_for_repl {idx} { ...@@ -19,8 +19,8 @@ proc fill_up_os_socket_send_buffer_for_repl {idx} {
foreach how {sigterm shutdown} { foreach how {sigterm shutdown} {
test "Shutting down master waits for replica to catch up ($how)" { test "Shutting down master waits for replica to catch up ($how)" {
start_server {} { start_server {overrides {save ""}} {
start_server {} { start_server {overrides {save ""}} {
set master [srv -1 client] set master [srv -1 client]
set master_host [srv -1 host] set master_host [srv -1 host]
set master_port [srv -1 port] set master_port [srv -1 port]
...@@ -42,8 +42,7 @@ foreach how {sigterm shutdown} { ...@@ -42,8 +42,7 @@ foreach how {sigterm shutdown} {
wait_for_ofs_sync $master $replica wait_for_ofs_sync $master $replica
# Pause the replica. # Pause the replica.
exec kill -SIGSTOP $replica_pid pause_process $replica_pid
after 10
# Fill up the OS socket send buffer for the replica connection # Fill up the OS socket send buffer for the replica connection
# to prevent the following INCR from reaching the replica via # to prevent the following INCR from reaching the replica via
...@@ -69,7 +68,7 @@ foreach how {sigterm shutdown} { ...@@ -69,7 +68,7 @@ foreach how {sigterm shutdown} {
# Wake up replica and check if master has waited for it. # Wake up replica and check if master has waited for it.
after 20; # 2 cron intervals after 20; # 2 cron intervals
exec kill -SIGCONT $replica_pid resume_process $replica_pid
wait_for_condition 300 1000 { wait_for_condition 300 1000 {
[$replica get k] eq 2 [$replica get k] eq 2
} else { } else {
...@@ -86,8 +85,8 @@ foreach how {sigterm shutdown} { ...@@ -86,8 +85,8 @@ foreach how {sigterm shutdown} {
} }
test {Shutting down master waits for replica timeout} { test {Shutting down master waits for replica timeout} {
start_server {} { start_server {overrides {save ""}} {
start_server {} { start_server {overrides {save ""}} {
set master [srv -1 client] set master [srv -1 client]
set master_host [srv -1 host] set master_host [srv -1 host]
set master_port [srv -1 port] set master_port [srv -1 port]
...@@ -107,8 +106,7 @@ test {Shutting down master waits for replica timeout} { ...@@ -107,8 +106,7 @@ test {Shutting down master waits for replica timeout} {
wait_for_ofs_sync $master $replica wait_for_ofs_sync $master $replica
# Pause the replica. # Pause the replica.
exec kill -SIGSTOP $replica_pid pause_process $replica_pid
after 10
# Fill up the OS socket send buffer for the replica connection to # Fill up the OS socket send buffer for the replica connection to
# prevent the following INCR k from reaching the replica via the OS. # prevent the following INCR k from reaching the replica via the OS.
...@@ -129,15 +127,15 @@ test {Shutting down master waits for replica timeout} { ...@@ -129,15 +127,15 @@ test {Shutting down master waits for replica timeout} {
verify_log_message -1 "*0 of 1 replicas are in sync*" 0 verify_log_message -1 "*0 of 1 replicas are in sync*" 0
# Wake up replica. # Wake up replica.
exec kill -SIGCONT $replica_pid resume_process $replica_pid
assert_equal 1 [$replica get k] assert_equal 1 [$replica get k]
} }
} }
} {} {repl external:skip} } {} {repl external:skip}
test "Shutting down master waits for replica then fails" { test "Shutting down master waits for replica then fails" {
start_server {} { start_server {overrides {save ""}} {
start_server {} { start_server {overrides {save ""}} {
set master [srv -1 client] set master [srv -1 client]
set master_host [srv -1 host] set master_host [srv -1 host]
set master_port [srv -1 port] set master_port [srv -1 port]
...@@ -150,8 +148,7 @@ test "Shutting down master waits for replica then fails" { ...@@ -150,8 +148,7 @@ test "Shutting down master waits for replica then fails" {
wait_for_sync $replica wait_for_sync $replica
# Pause the replica and write a key on master. # Pause the replica and write a key on master.
exec kill -SIGSTOP $replica_pid pause_process $replica_pid
after 10
$master incr k $master incr k
# Two clients call blocking SHUTDOWN in parallel. # Two clients call blocking SHUTDOWN in parallel.
...@@ -168,7 +165,7 @@ test "Shutting down master waits for replica then fails" { ...@@ -168,7 +165,7 @@ test "Shutting down master waits for replica then fails" {
$master config set appendonly yes $master config set appendonly yes
# Wake up replica, causing master to continue shutting down. # Wake up replica, causing master to continue shutting down.
exec kill -SIGCONT $replica_pid resume_process $replica_pid
# SHUTDOWN returns an error to both clients blocking on SHUTDOWN. # SHUTDOWN returns an error to both clients blocking on SHUTDOWN.
catch { $rd1 read } e1 catch { $rd1 read } e1
...@@ -190,8 +187,8 @@ test "Shutting down master waits for replica then fails" { ...@@ -190,8 +187,8 @@ test "Shutting down master waits for replica then fails" {
} {} {repl external:skip} } {} {repl external:skip}
test "Shutting down master waits for replica then aborted" { test "Shutting down master waits for replica then aborted" {
start_server {} { start_server {overrides {save ""}} {
start_server {} { start_server {overrides {save ""}} {
set master [srv -1 client] set master [srv -1 client]
set master_host [srv -1 host] set master_host [srv -1 host]
set master_port [srv -1 port] set master_port [srv -1 port]
...@@ -204,8 +201,7 @@ test "Shutting down master waits for replica then aborted" { ...@@ -204,8 +201,7 @@ test "Shutting down master waits for replica then aborted" {
wait_for_sync $replica wait_for_sync $replica
# Pause the replica and write a key on master. # Pause the replica and write a key on master.
exec kill -SIGSTOP $replica_pid pause_process $replica_pid
after 10
$master incr k $master incr k
# Two clients call blocking SHUTDOWN in parallel. # Two clients call blocking SHUTDOWN in parallel.
...@@ -221,7 +217,7 @@ test "Shutting down master waits for replica then aborted" { ...@@ -221,7 +217,7 @@ test "Shutting down master waits for replica then aborted" {
$master shutdown abort $master shutdown abort
# Wake up replica, causing master to continue shutting down. # Wake up replica, causing master to continue shutting down.
exec kill -SIGCONT $replica_pid resume_process $replica_pid
# SHUTDOWN returns an error to both clients blocking on SHUTDOWN. # SHUTDOWN returns an error to both clients blocking on SHUTDOWN.
catch { $rd1 read } e1 catch { $rd1 read } e1
......
...@@ -49,9 +49,9 @@ test "SDOWN is triggered by non-responding but not crashed instance" { ...@@ -49,9 +49,9 @@ test "SDOWN is triggered by non-responding but not crashed instance" {
set master_id [get_instance_id_by_port redis [lindex $master_addr 1]] set master_id [get_instance_id_by_port redis [lindex $master_addr 1]]
set pid [get_instance_attrib redis $master_id pid] set pid [get_instance_attrib redis $master_id pid]
exec kill -SIGSTOP $pid pause_process $pid
ensure_master_down ensure_master_down
exec kill -SIGCONT $pid resume_process $pid
ensure_master_up ensure_master_up
} }
......
...@@ -537,6 +537,9 @@ proc start_server {options {code undefined}} { ...@@ -537,6 +537,9 @@ proc start_server {options {code undefined}} {
set fd [open $stdout "a+"] set fd [open $stdout "a+"]
puts $fd "### Starting server for test $::cur_test" puts $fd "### Starting server for test $::cur_test"
close $fd close $fd
if {$::verbose > 1} {
puts "### Starting server $stdout for test - $::cur_test"
}
} }
# We may have a stdout left over from the previous tests, so we need # We may have a stdout left over from the previous tests, so we need
......
...@@ -168,7 +168,9 @@ proc test {name code {okpattern undefined} {tags {}}} { ...@@ -168,7 +168,9 @@ proc test {name code {okpattern undefined} {tags {}}} {
send_data_packet $::test_server_fd skip $name send_data_packet $::test_server_fd skip $name
return return
} }
if {$::verbose > 1} {
puts "starting test $name"
}
# abort if only_tests was set but test name is not included # abort if only_tests was set but test name is not included
if {[llength $::only_tests] > 0 && ![search_pattern_list $name $::only_tests]} { if {[llength $::only_tests] > 0 && ![search_pattern_list $name $::only_tests]} {
incr ::num_skipped incr ::num_skipped
...@@ -200,11 +202,16 @@ proc test {name code {okpattern undefined} {tags {}}} { ...@@ -200,11 +202,16 @@ proc test {name code {okpattern undefined} {tags {}}} {
$r close $r close
} }
} else { } else {
set servers {}
foreach srv $::servers { foreach srv $::servers {
set stdout [dict get $srv stdout] set stdout [dict get $srv stdout]
set fd [open $stdout "a+"] set fd [open $stdout "a+"]
puts $fd "### Starting test $::cur_test" puts $fd "### Starting test $::cur_test"
close $fd close $fd
lappend servers $stdout
}
if {$::verbose > 1} {
puts "### Starting test $::cur_test - with servers: $servers"
} }
} }
......
...@@ -602,15 +602,24 @@ proc stop_bg_complex_data {handle} { ...@@ -602,15 +602,24 @@ proc stop_bg_complex_data {handle} {
# Write num keys with the given key prefix and value size (in bytes). If idx is # Write num keys with the given key prefix and value size (in bytes). If idx is
# given, it's the index (AKA level) used with the srv procedure and it specifies # given, it's the index (AKA level) used with the srv procedure and it specifies
# to which Redis instance to write the keys. # to which Redis instance to write the keys.
proc populate {num {prefix key:} {size 3} {idx 0}} { proc populate {num {prefix key:} {size 3} {idx 0} {prints false}} {
set rd [redis_deferring_client $idx] r $idx deferred 1
for {set j 0} {$j < $num} {incr j} { if {$num > 16} {set pipeline 16} else {set pipeline $num}
$rd set $prefix$j [string repeat A $size] set val [string repeat A $size]
for {set j 0} {$j < $pipeline} {incr j} {
r $idx set $prefix$j $val
if {$prints} {puts $j}
} }
for {set j 0} {$j < $num} {incr j} { for {} {$j < $num} {incr j} {
$rd read r $idx set $prefix$j $val
r $idx read
if {$prints} {puts $j}
} }
$rd close for {set j 0} {$j < $pipeline} {incr j} {
r $idx read
if {$prints} {puts $j}
}
r $idx deferred 0
} }
proc get_child_pid {idx} { proc get_child_pid {idx} {
...@@ -636,6 +645,20 @@ proc process_is_alive pid { ...@@ -636,6 +645,20 @@ proc process_is_alive pid {
} }
} }
proc pause_process pid {
exec kill -SIGSTOP $pid
wait_for_condition 50 100 {
[string match {*T*} [lindex [exec ps j $pid] 16]]
} else {
puts [exec ps j $pid]
fail "process didn't stop"
}
}
proc resume_process pid {
exec kill -SIGCONT $pid
}
proc cmdrstat {cmd r} { proc cmdrstat {cmd r} {
if {[regexp "\r\ncmdstat_$cmd:(.*?)\r\n" [$r info commandstats] _ value]} { if {[regexp "\r\ncmdstat_$cmd:(.*?)\r\n" [$r info commandstats] _ value]} {
set _ $value set _ $value
......
...@@ -526,6 +526,7 @@ proc signal_idle_client fd { ...@@ -526,6 +526,7 @@ proc signal_idle_client fd {
incr ::next_test incr ::next_test
if {$::loop && $::next_test == [llength $::all_tests]} { if {$::loop && $::next_test == [llength $::all_tests]} {
set ::next_test 0 set ::next_test 0
incr ::loop -1
} }
} elseif {[llength $::run_solo_tests] != 0 && [llength $::active_clients] == 0} { } elseif {[llength $::run_solo_tests] != 0 && [llength $::active_clients] == 0} {
if {!$::quiet} { if {!$::quiet} {
...@@ -620,6 +621,7 @@ proc print_help_screen {} { ...@@ -620,6 +621,7 @@ proc print_help_screen {} {
"--no-latency Skip latency measurements and validation by some tests." "--no-latency Skip latency measurements and validation by some tests."
"--stop Blocks once the first test fails." "--stop Blocks once the first test fails."
"--loop Execute the specified set of tests forever." "--loop Execute the specified set of tests forever."
"--loops <count> Execute the specified set of tests several times."
"--wait-server Wait after server is started (so that you can attach a debugger)." "--wait-server Wait after server is started (so that you can attach a debugger)."
"--dump-logs Dump server log on test failure." "--dump-logs Dump server log on test failure."
"--tls Run tests in TLS mode." "--tls Run tests in TLS mode."
...@@ -721,7 +723,7 @@ for {set j 0} {$j < [llength $argv]} {incr j} { ...@@ -721,7 +723,7 @@ for {set j 0} {$j < [llength $argv]} {incr j} {
} }
exit 0 exit 0
} elseif {$opt eq {--verbose}} { } elseif {$opt eq {--verbose}} {
set ::verbose 1 incr ::verbose
} elseif {$opt eq {--client}} { } elseif {$opt eq {--client}} {
set ::client 1 set ::client 1
set ::test_server_port $arg set ::test_server_port $arg
...@@ -744,7 +746,10 @@ for {set j 0} {$j < [llength $argv]} {incr j} { ...@@ -744,7 +746,10 @@ for {set j 0} {$j < [llength $argv]} {incr j} {
} elseif {$opt eq {--stop}} { } elseif {$opt eq {--stop}} {
set ::stop_on_failure 1 set ::stop_on_failure 1
} elseif {$opt eq {--loop}} { } elseif {$opt eq {--loop}} {
set ::loop 1 set ::loop 2147483647
} elseif {$opt eq {--loops}} {
set ::loop $arg
incr j
} elseif {$opt eq {--timeout}} { } elseif {$opt eq {--timeout}} {
set ::timeout $arg set ::timeout $arg
incr j incr j
......
# This unit has the potential to create huge .reqres files, causing log-req-res-validator.py to run for a very long time... # This unit has the potential to create huge .reqres files, causing log-req-res-validator.py to run for a very long time...
# Since this unit doesn't do anything worth validating, reply_schema-wise, we decided to skip it # Since this unit doesn't do anything worth validating, reply_schema-wise, we decided to skip it
start_server {tags {"aofrw external:skip logreqres:skip"}} { start_server {tags {"aofrw external:skip logreqres:skip"} overrides {save {}}} {
# Enable the AOF # Enable the AOF
r config set appendonly yes r config set appendonly yes
r config set auto-aof-rewrite-percentage 0 ; # Disable auto-rewrite. r config set auto-aof-rewrite-percentage 0 ; # Disable auto-rewrite.
......
...@@ -347,12 +347,12 @@ start_server {} { ...@@ -347,12 +347,12 @@ start_server {} {
# We use two obuf-clients to make sure that even if client eviction is attempted # We use two obuf-clients to make sure that even if client eviction is attempted
# between two command processing (with no sleep) we don't perform any client eviction # between two command processing (with no sleep) we don't perform any client eviction
# because the obuf limit is enforced with precedence. # because the obuf limit is enforced with precedence.
exec kill -SIGSTOP $server_pid pause_process $server_pid
$rr2 get k $rr2 get k
$rr2 flush $rr2 flush
$rr3 get k $rr3 get k
$rr3 flush $rr3 flush
exec kill -SIGCONT $server_pid resume_process $server_pid
r ping ;# make sure a full event loop cycle is processed before issuing CLIENT LIST r ping ;# make sure a full event loop cycle is processed before issuing CLIENT LIST
# Validate obuf-clients were disconnected (because of obuf limit) # Validate obuf-clients were disconnected (because of obuf limit)
......
...@@ -116,7 +116,7 @@ start_multiple_servers 3 [list overrides $base_conf] { ...@@ -116,7 +116,7 @@ start_multiple_servers 3 [list overrides $base_conf] {
test "Kill a cluster node and wait for fail state" { test "Kill a cluster node and wait for fail state" {
# kill node3 in cluster # kill node3 in cluster
exec kill -SIGSTOP $node3_pid pause_process $node3_pid
wait_for_condition 1000 50 { wait_for_condition 1000 50 {
[CI 0 cluster_state] eq {fail} && [CI 0 cluster_state] eq {fail} &&
...@@ -134,7 +134,7 @@ start_multiple_servers 3 [list overrides $base_conf] { ...@@ -134,7 +134,7 @@ start_multiple_servers 3 [list overrides $base_conf] {
assert_equal [s -1 blocked_clients] {0} assert_equal [s -1 blocked_clients] {0}
} }
exec kill -SIGCONT $node3_pid resume_process $node3_pid
$node1_rd close $node1_rd close
} ;# stop servers } ;# stop servers
......
...@@ -200,7 +200,7 @@ start_cluster 3 0 {tags {external:skip cluster}} { ...@@ -200,7 +200,7 @@ start_cluster 3 0 {tags {external:skip cluster}} {
# To manufacture an ever-growing send buffer from primary1 to primary2, # To manufacture an ever-growing send buffer from primary1 to primary2,
# make primary2 unresponsive. # make primary2 unresponsive.
set primary2_pid [srv [expr -1*$primary2_id] pid] set primary2_pid [srv [expr -1*$primary2_id] pid]
exec kill -SIGSTOP $primary2_pid pause_process $primary2_pid
# On primary1, send 128KB Pubsub messages in a loop until the send buffer of the link from # On primary1, send 128KB Pubsub messages in a loop until the send buffer of the link from
# primary1 to primary2 exceeds buffer limit therefore be dropped. # primary1 to primary2 exceeds buffer limit therefore be dropped.
...@@ -226,7 +226,7 @@ start_cluster 3 0 {tags {external:skip cluster}} { ...@@ -226,7 +226,7 @@ start_cluster 3 0 {tags {external:skip cluster}} {
assert {[dict get $same_link_p1_from_p2 create-time] eq [dict get $orig_link_p1_from_p2 create-time]} assert {[dict get $same_link_p1_from_p2 create-time] eq [dict get $orig_link_p1_from_p2 create-time]}
# Revive primary2 # Revive primary2
exec kill -SIGCONT $primary2_pid resume_process $primary2_pid
# Reset configs on primary1 so config changes don't leak out to other tests # Reset configs on primary1 so config changes don't leak out to other tests
$primary1 CONFIG set cluster-node-timeout $oldtimeout $primary1 CONFIG set cluster-node-timeout $oldtimeout
......
...@@ -350,7 +350,7 @@ proc test_slave_buffers {test_name cmd_count payload_len limit_memory pipeline} ...@@ -350,7 +350,7 @@ proc test_slave_buffers {test_name cmd_count payload_len limit_memory pipeline}
# put the slave to sleep # put the slave to sleep
set rd_slave [redis_deferring_client] set rd_slave [redis_deferring_client]
exec kill -SIGSTOP $slave_pid pause_process $slave_pid
# send some 10mb worth of commands that don't increase the memory usage # send some 10mb worth of commands that don't increase the memory usage
if {$pipeline == 1} { if {$pipeline == 1} {
...@@ -399,7 +399,7 @@ proc test_slave_buffers {test_name cmd_count payload_len limit_memory pipeline} ...@@ -399,7 +399,7 @@ proc test_slave_buffers {test_name cmd_count payload_len limit_memory pipeline}
} }
# unfreeze slave process (after the 'test' succeeded or failed, but before we attempt to terminate the server # unfreeze slave process (after the 'test' succeeded or failed, but before we attempt to terminate the server
exec kill -SIGCONT $slave_pid resume_process $slave_pid
} }
} }
} }
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment