Skip to content

Commit

Permalink
Reduce dual channel testing time (#1477)
Browse files Browse the repository at this point in the history 
- By not waiting `repl-diskless-sync-delay` when we don't have to, we
can reduce ~30% of dual channel tests execution time.
- This commit also drops one test which is not required for regular sync
(`Sync should continue if not all slaves dropped`).
- Skip dual channel test with master diskless disabled because it will
initiate the same synchronization process as the non-dual channel test,
making it redundant.


Before:
```
Execution time of different units:
  171 seconds - integration/dual-channel-replication
  305 seconds - integration/replication-psync

\o/ All tests passed without errors!
```
After:
```
Execution time of different units:
  120 seconds - integration/dual-channel-replication
  236 seconds - integration/replication-psync

\o/ All tests passed without errors!
```

Discused on #1173

---------

Signed-off-by: naglera <[email protected]>
  • Loading branch information
@naglera
naglera authored Dec 24, 2024
1 parent 2ee06e7 commit f1b7f30
Show file tree
Hide file tree
Showing 2 changed files with 36 additions and 28 deletions.
60 changes: 32 additions & 28 deletions tests/integration/dual-channel-replication.tcl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,7 @@ start_server {tags {"dual-channel-replication external:skip"}} {

$primary config set rdb-key-save-delay 200
$primary config set dual-channel-replication-enabled yes
$primary config set repl-diskless-sync-delay 0
$replica config set dual-channel-replication-enabled yes
$replica config set repl-diskless-sync no

Expand Down Expand Up @@ -201,6 +202,7 @@ start_server {tags {"dual-channel-replication external:skip"}} {
# a replication buffer block.
$primary config set client-output-buffer-limit "replica 1100k 0 0"
$primary config set dual-channel-replication-enabled $enable
$primary config set repl-diskless-sync-delay 0
$replica config set dual-channel-replication-enabled $enable

test "Toggle dual-channel-replication-enabled: $enable start" {
Expand Down Expand Up @@ -506,6 +508,7 @@ start_server {tags {"dual-channel-replication external:skip"}} {
$primary config set dual-channel-replication-enabled yes
$primary config set repl-backlog-size $backlog_size
$primary config set loglevel debug
$primary config set repl-diskless-sync-delay 0
if {$::valgrind} {
$primary config set repl-timeout 100
} else {
Expand Down Expand Up @@ -877,7 +880,6 @@ start_server {tags {"dual-channel-replication external:skip"}} {
}
}

foreach dualchannel {yes no} {
start_server {tags {"dual-channel-replication external:skip"}} {
set primary [srv 0 client]
set primary_host [srv 0 host]
Expand All @@ -893,20 +895,20 @@ start_server {tags {"dual-channel-replication external:skip"}} {
# Generating RDB will cost 5s(10000 * 0.0005s)
$primary debug populate 10000 primary 1
$primary config set rdb-key-save-delay 500
$primary config set dual-channel-replication-enabled $dualchannel
$primary config set dual-channel-replication-enabled yes

start_server {} {
set replica1 [srv 0 client]
$replica1 config set dual-channel-replication-enabled $dualchannel
$replica1 config set dual-channel-replication-enabled yes
$replica1 config set loglevel debug
start_server {} {
set replica2 [srv 0 client]
$replica2 config set dual-channel-replication-enabled $dualchannel
$replica2 config set dual-channel-replication-enabled yes
$replica2 config set loglevel debug
$replica2 config set repl-timeout 60

set load_handle [start_one_key_write_load $primary_host $primary_port 100 "mykey1"]
test "Sync should continue if not all slaves dropped dual-channel-replication $dualchannel" {
test "Sync should continue if not all slaves dropped" {
$replica1 replicaof $primary_host $primary_port
$replica2 replicaof $primary_host $primary_port

Expand All @@ -915,20 +917,17 @@ start_server {tags {"dual-channel-replication external:skip"}} {
} else {
fail "Sync did not start"
}
if {$dualchannel == "yes"} {
# Wait for both replicas main conns to establish psync
wait_for_condition 50 1000 {
[status $primary sync_partial_ok] == 2
} else {
fail "Replicas main conns didn't establish psync [status $primary sync_partial_ok]"
}
# Wait for both replicas main conns to establish psync
wait_for_condition 50 1000 {
[status $primary sync_partial_ok] == 2
} else {
fail "Replicas main conns didn't establish psync [status $primary sync_partial_ok]"
}

catch {$replica1 shutdown nosave}
wait_for_condition 50 2000 {
[status $replica2 master_link_status] == "up" &&
[status $primary sync_full] == 2 &&
(($dualchannel == "yes" && [status $primary sync_partial_ok] == 2) || $dualchannel == "no")
([status $primary sync_partial_ok] == 2)
} else {
fail "Sync session interapted\n
sync_full:[status $primary sync_full]\n
Expand All @@ -942,7 +941,7 @@ start_server {tags {"dual-channel-replication external:skip"}} {
$primary debug populate 1000000 primary 1
$primary config set rdb-key-save-delay 100

test "Primary abort sync if all slaves dropped dual-channel-replication $dualchannel" {
test "Primary abort sync if all slaves dropped dual-channel-replication" {
set cur_psync [status $primary sync_partial_ok]
$replica2 replicaof $primary_host $primary_port

Expand All @@ -951,13 +950,11 @@ start_server {tags {"dual-channel-replication external:skip"}} {
} else {
fail "Sync did not start"
}
if {$dualchannel == "yes"} {
# Wait for both replicas main conns to establish psync
wait_for_condition 50 1000 {
[status $primary sync_partial_ok] == $cur_psync + 1
} else {
fail "Replicas main conns didn't establish psync [status $primary sync_partial_ok]"
}
# Wait for both replicas main conns to establish psync
wait_for_condition 50 1000 {
[status $primary sync_partial_ok] == $cur_psync + 1
} else {
fail "Replicas main conns didn't establish psync [status $primary sync_partial_ok]"
}

catch {$replica2 shutdown nosave}
Expand All @@ -971,7 +968,7 @@ start_server {tags {"dual-channel-replication external:skip"}} {
}
}
}
}


start_server {tags {"dual-channel-replication external:skip"}} {
set primary [srv 0 client]
Expand All @@ -982,8 +979,7 @@ start_server {tags {"dual-channel-replication external:skip"}} {
$primary config set repl-diskless-sync yes
$primary config set dual-channel-replication-enabled yes
$primary config set loglevel debug
$primary config set repl-diskless-sync-delay 5; # allow catch failed sync before retry

$primary config set repl-diskless-sync-delay 0
# Generating RDB will cost 500s(1000000 * 0.0001s)
$primary debug populate 1000000 primary 1
$primary config set rdb-key-save-delay 100
Expand Down Expand Up @@ -1014,6 +1010,7 @@ start_server {tags {"dual-channel-replication external:skip"}} {
set replica_main_conn_id [get_client_id_by_last_cmd $primary "psync"]
assert {$replica_main_conn_id != ""}
set loglines [count_log_lines -1]
$primary config set repl-diskless-sync-delay 5; # allow catch failed sync before retry
$primary client kill id $replica_main_conn_id
# Wait for primary to abort the sync
wait_for_condition 50 1000 {
Expand All @@ -1034,6 +1031,7 @@ start_server {tags {"dual-channel-replication external:skip"}} {
}

test "Test dual-channel-replication replica rdb connection disconnected" {
$primary config set repl-diskless-sync-delay 0
$replica replicaof $primary_host $primary_port
# Wait for sync session to start
wait_for_condition 500 1000 {
Expand All @@ -1048,6 +1046,7 @@ start_server {tags {"dual-channel-replication external:skip"}} {
$primary debug log "killing replica rdb connection $replica_rdb_channel_id"
assert {$replica_rdb_channel_id != ""}
set loglines [count_log_lines -1]
$primary config set repl-diskless-sync-delay 5; # allow catch failed sync before retry
$primary client kill id $replica_rdb_channel_id
# Wait for primary to abort the sync
wait_for_log_messages -1 {"*Background RDB transfer error*"} $loglines 1000 10
Expand All @@ -1063,6 +1062,7 @@ start_server {tags {"dual-channel-replication external:skip"}} {
}

test "Test dual-channel-replication primary reject set-rdb-client after client killed" {
$primary config set repl-diskless-sync-delay 0
# Ensure replica main channel will not handshake before rdb client is killed
$replica debug pause-after-fork 1
$replica replicaof $primary_host $primary_port
Expand All @@ -1077,6 +1077,7 @@ start_server {tags {"dual-channel-replication external:skip"}} {
set replica_rdb_channel_id [get_client_id_by_last_cmd $primary "sync"]
assert {$replica_rdb_channel_id != ""}
$primary debug log "killing replica rdb connection $replica_rdb_channel_id"
$primary config set repl-diskless-sync-delay 5; # allow catch failed sync before retry
$primary client kill id $replica_rdb_channel_id
# Wait for primary to abort the sync
wait_and_resume_process 0
Expand Down Expand Up @@ -1154,7 +1155,7 @@ start_server {tags {"dual-channel-replication external:skip"}} {
$primary config set repl-diskless-sync yes
$primary config set dual-channel-replication-enabled yes
$primary config set loglevel debug
$primary config set repl-diskless-sync-delay 5; # allow catch failed sync before retry
$primary config set repl-diskless-sync-delay 0

# Generating RDB will cost 100 sec to generate
$primary debug populate 10000 primary 1
Expand Down Expand Up @@ -1185,13 +1186,15 @@ start_server {tags {"dual-channel-replication external:skip"}} {
set replica_rdb_channel_id [get_client_id_by_last_cmd $primary "sync"]
assert {$replica_rdb_channel_id != ""}
set loglines [count_log_lines -1]
$primary config set repl-diskless-sync-delay 5; # allow catch failed sync before retry
$primary client kill id $replica_rdb_channel_id
# Wait for primary to abort the sync
wait_for_condition 50 1000 {
[string match {*replicas_waiting_psync:0*} [$primary info replication]]
} else {
fail "Primary did not free repl buf block after sync failure"
}
$primary config set repl-diskless-sync-delay 0
wait_for_log_messages -1 {"*Background RDB transfer error*"} $loglines 1000 10
# Replica should retry
wait_for_condition 500 1000 {
Expand All @@ -1200,7 +1203,7 @@ start_server {tags {"dual-channel-replication external:skip"}} {
[s -1 rdb_bgsave_in_progress] eq 1
} else {
fail "replica didn't retry after connection close"
}
}
}
$replica replicaof no one
wait_for_condition 500 1000 {
Expand All @@ -1218,18 +1221,19 @@ start_server {tags {"dual-channel-replication external:skip"}} {
} else {
fail "replica didn't start sync session in time"
}

$primary debug log "killing replica main connection"
set replica_main_conn_id [get_client_id_by_last_cmd $primary "sync"]
assert {$replica_main_conn_id != ""}
set loglines [count_log_lines -1]
$primary config set repl-diskless-sync-delay 5; # allow catch failed sync before retry
$primary client kill id $replica_main_conn_id
# Wait for primary to abort the sync
wait_for_condition 50 1000 {
[string match {*replicas_waiting_psync:0*} [$primary info replication]]
} else {
fail "Primary did not free repl buf block after sync failure"
}
$primary config set repl-diskless-sync-delay 0
wait_for_log_messages -1 {"*Background RDB transfer error*"} $loglines 1000 10
# Replica should retry
wait_for_condition 500 1000 {
Expand Down
4 changes: 4 additions & 0 deletions tests/integration/replication-psync.tcl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,10 @@ tags {"external:skip"} {
foreach mdl {no yes} {
foreach sdl {disabled swapdb} {
foreach dualchannel {yes no} {
# Skip dual channel test with master diskless disabled
if {$dualchannel == "yes" && $mdl == "no"} {
continue
}
test_psync {no reconnection, just sync} 6 1000000 3600 0 {
} $mdl $sdl $dualchannel 0

Expand Down

0 comments on commit f1b7f30

Please sign in to comment.