-
Notifications
You must be signed in to change notification settings - Fork 720
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Signed-off-by: Binbin <[email protected]>
- Loading branch information
1 parent
ee680e6
commit 0f34526
Showing
2 changed files
with
64 additions
and
51 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,62 +1,66 @@ | ||
# Check the basic monitoring and failover capabilities. | ||
|
||
start_cluster 3 6 {tags {external:skip cluster} overrides {cluster-ping-interval 1000 cluster-node-timeout 15000}} { | ||
|
||
test "Cluster is up" { | ||
wait_for_cluster_state ok | ||
} | ||
|
||
test "Cluster is writable" { | ||
cluster_write_test [srv 0 port] | ||
} | ||
|
||
set paused_pid [srv 0 pid] | ||
test "Killing one primary node" { | ||
pause_process $paused_pid | ||
} | ||
|
||
test "Wait for failover" { | ||
wait_for_condition 1000 50 { | ||
[s -3 role] == "master" || [s -6 role] == "master" | ||
} else { | ||
fail "No failover detected" | ||
} | ||
} | ||
|
||
test "Killing the new primary node" { | ||
if {[s -3 role] == "master"} { | ||
set replica_to_be_primary -6 | ||
set paused_pid2 [srv -3 pid] | ||
} else { | ||
set replica_to_be_primary -3 | ||
set paused_pid2 [srv -6 pid] | ||
} | ||
pause_process $paused_pid2 | ||
} | ||
|
||
test "Cluster should eventually be up again" { | ||
for {set j 0} {$j < [llength $::servers]} {incr j} { | ||
if {[process_is_paused $paused_pid]} continue | ||
if {[process_is_paused $paused_pid2]} continue | ||
start_cluster 3 4 {tags {external:skip cluster} overrides {cluster-ping-interval 1000 cluster-node-timeout 5000}} { | ||
|
||
test "Cluster is up" { | ||
wait_for_cluster_state ok | ||
} | ||
|
||
test "Cluster is writable" { | ||
cluster_write_test [srv 0 port] | ||
} | ||
|
||
set paused_pid [srv 0 pid] | ||
test "Killing one primary node" { | ||
pause_process $paused_pid | ||
} | ||
|
||
test "Wait for failover" { | ||
wait_for_condition 1000 50 { | ||
[s -3 role] == "master" || [s -6 role] == "master" | ||
} else { | ||
fail "No failover detected" | ||
} | ||
} | ||
|
||
test "Killing the new primary node" { | ||
if {[s -3 role] == "master"} { | ||
set replica_to_be_primary -6 | ||
set paused_pid2 [srv -3 pid] | ||
} else { | ||
set replica_to_be_primary -3 | ||
set paused_pid2 [srv -6 pid] | ||
} | ||
pause_process $paused_pid2 | ||
} | ||
|
||
test "Cluster should eventually be up again" { | ||
for {set j 0} {$j < [llength $::servers]} {incr j} { | ||
if {[process_is_paused [srv -$j pid]]} continue | ||
wait_for_condition 1000 50 { | ||
[CI $j cluster_state] eq "ok" | ||
} else { | ||
fail "Cluster node $j cluster_state:[CI $j cluster_state]" | ||
} | ||
} | ||
} | ||
|
||
test "wait new failover" { | ||
wait_for_condition 1000 50 { | ||
[CI $j cluster_state] eq "ok" | ||
[s $replica_to_be_primary role] == "master" | ||
} else { | ||
fail "Cluster node $j cluster_state:[CI $j cluster_state]" | ||
fail "No failover detected" | ||
} | ||
} | ||
} | ||
|
||
test "wait new failover" { | ||
wait_for_condition 1000 100 { | ||
[s $replica_to_be_primary role] == "master" | ||
} else { | ||
fail "No failover detected" | ||
test "Restarting the previously killed primary nodes" { | ||
resume_process $paused_pid | ||
resume_process $paused_pid2 | ||
} | ||
} | ||
|
||
test "Restarting the previously killed primary nodes" { | ||
resume_process $paused_pid | ||
resume_process $paused_pid2 | ||
} | ||
test "Make sure there is no failover timeout" { | ||
verify_no_log_message -3 "*Failover attempt expired*" 0 | ||
verify_no_log_message -6 "*Failover attempt expired*" 0 | ||
} | ||
|
||
} ;# start_cluster |