Skip to content

Commit

Permalink
update the tests
Browse files Browse the repository at this point in the history
Signed-off-by: Binbin <[email protected]>
  • Loading branch information
enjoy-binbin committed Jul 14, 2024
1 parent ee680e6 commit 0f34526
Show file tree
Hide file tree
Showing 2 changed files with 64 additions and 51 deletions.
9 changes: 9 additions & 0 deletions tests/support/util.tcl
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,15 @@ proc verify_log_message {srv_idx pattern from_line} {
}
}

# verify pattern does not exists in server's stout after a certain line number
proc verify_no_log_message {srv_idx pattern from_line} {
incr from_line
set result [exec tail -n +$from_line < [srv $srv_idx stdout]]
if {[string match $pattern $result]} {
error "assertion:expected message found in log file: $pattern"
}
}

# wait for pattern to be found in server's stdout after certain line number
# return value is a list containing the line that matched the pattern and the line number
proc wait_for_log_messages {srv_idx patterns from_line maxtries delay} {
Expand Down
106 changes: 55 additions & 51 deletions tests/unit/cluster/failover2.tcl
Original file line number Diff line number Diff line change
@@ -1,62 +1,66 @@
# Check the basic monitoring and failover capabilities.

start_cluster 3 6 {tags {external:skip cluster} overrides {cluster-ping-interval 1000 cluster-node-timeout 15000}} {

test "Cluster is up" {
wait_for_cluster_state ok
}

test "Cluster is writable" {
cluster_write_test [srv 0 port]
}

set paused_pid [srv 0 pid]
test "Killing one primary node" {
pause_process $paused_pid
}

test "Wait for failover" {
wait_for_condition 1000 50 {
[s -3 role] == "master" || [s -6 role] == "master"
} else {
fail "No failover detected"
}
}

test "Killing the new primary node" {
if {[s -3 role] == "master"} {
set replica_to_be_primary -6
set paused_pid2 [srv -3 pid]
} else {
set replica_to_be_primary -3
set paused_pid2 [srv -6 pid]
}
pause_process $paused_pid2
}

test "Cluster should eventually be up again" {
for {set j 0} {$j < [llength $::servers]} {incr j} {
if {[process_is_paused $paused_pid]} continue
if {[process_is_paused $paused_pid2]} continue
start_cluster 3 4 {tags {external:skip cluster} overrides {cluster-ping-interval 1000 cluster-node-timeout 5000}} {

test "Cluster is up" {
wait_for_cluster_state ok
}

test "Cluster is writable" {
cluster_write_test [srv 0 port]
}

set paused_pid [srv 0 pid]
test "Killing one primary node" {
pause_process $paused_pid
}

test "Wait for failover" {
wait_for_condition 1000 50 {
[s -3 role] == "master" || [s -6 role] == "master"
} else {
fail "No failover detected"
}
}

test "Killing the new primary node" {
if {[s -3 role] == "master"} {
set replica_to_be_primary -6
set paused_pid2 [srv -3 pid]
} else {
set replica_to_be_primary -3
set paused_pid2 [srv -6 pid]
}
pause_process $paused_pid2
}

test "Cluster should eventually be up again" {
for {set j 0} {$j < [llength $::servers]} {incr j} {
if {[process_is_paused [srv -$j pid]]} continue
wait_for_condition 1000 50 {
[CI $j cluster_state] eq "ok"
} else {
fail "Cluster node $j cluster_state:[CI $j cluster_state]"
}
}
}

test "wait new failover" {
wait_for_condition 1000 50 {
[CI $j cluster_state] eq "ok"
[s $replica_to_be_primary role] == "master"
} else {
fail "Cluster node $j cluster_state:[CI $j cluster_state]"
fail "No failover detected"
}
}
}

test "wait new failover" {
wait_for_condition 1000 100 {
[s $replica_to_be_primary role] == "master"
} else {
fail "No failover detected"
test "Restarting the previously killed primary nodes" {
resume_process $paused_pid
resume_process $paused_pid2
}
}

test "Restarting the previously killed primary nodes" {
resume_process $paused_pid
resume_process $paused_pid2
}
test "Make sure there is no failover timeout" {
verify_no_log_message -3 "*Failover attempt expired*" 0
verify_no_log_message -6 "*Failover attempt expired*" 0
}

} ;# start_cluster

0 comments on commit 0f34526

Please sign in to comment.