Unverified Commit 997fa41e authored by Oran Agra's avatar Oran Agra Committed by GitHub
Browse files

Attempt to solve MacOS CI issues in GH Actions (#12013)

The MacOS CI in github actions often hangs without any logs. GH argues that
it's due to resource utilization, either running out of disk space, memory, or CPU
starvation, and thus the runner is terminated.

This PR contains multiple attempts to resolve this:
1. introducing pause_process instead of SIGSTOP, which waits for the process
  to stop before resuming the test, possibly resolving race conditions in some tests,
  this was a suspect since there was one test that could result in an infinite loop in that
 case, in practice this didn't help, but still a good idea to keep.
2. disable the `save` config in many tests that don't need it, specifically ones that use
  heavy writes and could create large files.
3. change the `populate` proc to use short pipeline rather than an infinite one.
4. use `--clients 1` in the macos CI so that we don't risk running multiple resource
  demanding tests in parallel.
5. enable `--verbose` to be repeated to elevate verbosity and print more info to stdout
  when a test or a server starts.
parent 45b8eea1
......@@ -132,7 +132,7 @@ start_cluster 3 0 [list config_lines $modules] {
test "Kill a cluster node and wait for fail state" {
# kill node3 in cluster
exec kill -SIGSTOP $node3_pid
pause_process $node3_pid
wait_for_condition 1000 50 {
[CI 0 cluster_state] eq {fail} &&
......@@ -158,7 +158,7 @@ start_cluster 3 0 [list config_lines $modules] {
assert_error "ERR Can not execute a command 'set' while the cluster is down" {$node1 do_rm_call set x 1}
}
exec kill -SIGCONT $node3_pid
resume_process $node3_pid
$node1_rd close
$node2_rd close
}
......
......@@ -47,25 +47,25 @@ start_server {} {
}
test {WAIT should not acknowledge 1 additional copy if slave is blocked} {
exec kill -SIGSTOP $slave_pid
pause_process $slave_pid
$master set foo 0
$master incr foo
$master incr foo
$master incr foo
assert {[$master wait 1 1000] == 0}
exec kill -SIGCONT $slave_pid
resume_process $slave_pid
assert {[$master wait 1 1000] == 1}
}
test {WAIT implicitly blocks on client pause since ACKs aren't sent} {
exec kill -SIGSTOP $slave_pid
pause_process $slave_pid
$master multi
$master incr foo
$master client pause 10000 write
$master exec
assert {[$master wait 1 1000] == 0}
$master client unpause
exec kill -SIGCONT $slave_pid
resume_process $slave_pid
assert {[$master wait 1 1000] == 1}
}
......@@ -73,7 +73,7 @@ start_server {} {
set rd [redis_deferring_client -1]
set rd2 [redis_deferring_client -1]
exec kill -SIGSTOP $slave_pid
pause_process $slave_pid
$rd incr foo
$rd read
......@@ -85,7 +85,7 @@ start_server {} {
$rd2 wait 1 0
wait_for_blocked_clients_count 2 100 10 -1
exec kill -SIGCONT $slave_pid
resume_process $slave_pid
assert_equal [$rd read] {1}
assert_equal [$rd2 read] {1}
......@@ -229,10 +229,10 @@ tags {"wait aof network external:skip"} {
}
test {WAITAOF replica copy if replica is blocked} {
exec kill -SIGSTOP $replica_pid
pause_process $replica_pid
$master incr foo
assert_equal [$master waitaof 0 1 50] {1 0} ;# exits on timeout
exec kill -SIGCONT $replica_pid
resume_process $replica_pid
assert_equal [$master waitaof 0 1 0] {1 1}
}
......@@ -240,7 +240,7 @@ tags {"wait aof network external:skip"} {
set rd [redis_deferring_client -1]
set rd2 [redis_deferring_client -1]
exec kill -SIGSTOP $replica_pid
pause_process $replica_pid
$rd incr foo
$rd read
......@@ -252,7 +252,7 @@ tags {"wait aof network external:skip"} {
$rd2 waitaof 0 1 0
wait_for_blocked_clients_count 2 100 10 -1
exec kill -SIGCONT $replica_pid
resume_process $replica_pid
assert_equal [$rd read] {1 1}
assert_equal [$rd2 read] {1 1}
......@@ -438,7 +438,7 @@ start_server {} {
waitForBgrewriteaof $replica1
waitForBgrewriteaof $replica2
exec kill -SIGSTOP $replica1_pid
pause_process $replica1_pid
$rd incr foo
$rd read
......@@ -451,7 +451,7 @@ start_server {} {
wait_for_blocked_clients_count 2
exec kill -SIGCONT $replica1_pid
resume_process $replica1_pid
# WAIT will unblock the client first.
assert_equal [$rd2 read] {2}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment