start_server {tags {"repl network external:skip singledb:skip"} overrides {save {}}} { start_server { overrides {save {}}} { set master [srv -1 client] set master_host [srv -1 host] set master_port [srv -1 port] set slave [srv 0 client] set load_handle0 [start_bg_complex_data $master_host $master_port 9 100000] set load_handle1 [start_bg_complex_data $master_host $master_port 11 100000] set load_handle2 [start_bg_complex_data $master_host $master_port 12 100000] test {First server should have role slave after SLAVEOF} { $slave slaveof $master_host $master_port wait_for_condition 50 100 { [s 0 role] eq {slave} } else { fail "Replication not started." } } test {Test replication with parallel clients writing in different DBs} { # Gives the random workloads a chance to add some complex commands. after 5000 # Make sure all parallel clients have written data. wait_for_condition 1000 50 { [$master select 9] == {OK} && [$master dbsize] > 0 && [$master select 11] == {OK} && [$master dbsize] > 0 && [$master select 12] == {OK} && [$master dbsize] > 0 } else { fail "Parallel clients are not writing in different DBs." } stop_bg_complex_data $load_handle0 stop_bg_complex_data $load_handle1 stop_bg_complex_data $load_handle2 wait_for_condition 100 100 { [$master debug digest] == [$slave debug digest] } else { set csv1 [csvdump r] set csv2 [csvdump {r -1}] set fd [open /tmp/repldump1.txt w] puts -nonewline $fd $csv1 close $fd set fd [open /tmp/repldump2.txt w] puts -nonewline $fd $csv2 close $fd fail "Master - Replica inconsistency, Run diff -u against /tmp/repldump*.txt for more info" } } } } start_server {tags {"repl external:skip"}} { start_server {} { set master [srv -1 client] set master_host [srv -1 host] set master_port [srv -1 port] set slave [srv 0 client] # Load some functions to be used later $master FUNCTION load replace {#!lua name=test server.register_function{function_name='f_default_flags', callback=function(keys, args) return redis.call('get',keys[1]) end, flags={}} server.register_function{function_name='f_no_writes', callback=function(keys, args) return redis.call('get',keys[1]) end, flags={'no-writes'}} } test {First server should have role slave after SLAVEOF} { $slave slaveof $master_host $master_port wait_replica_online $master } test {With min-slaves-to-write (1,3): master should be writable} { $master config set min-slaves-max-lag 3 $master config set min-slaves-to-write 1 assert_equal OK [$master set foo 123] assert_equal OK [$master eval "return redis.call('set','foo',12345)" 0] } test {With min-slaves-to-write (2,3): master should not be writable} { $master config set min-slaves-max-lag 3 $master config set min-slaves-to-write 2 assert_error "*NOREPLICAS*" {$master set foo bar} assert_error "*NOREPLICAS*" {$master eval "redis.call('set','foo','bar')" 0} } test {With min-slaves-to-write function without no-write flag} { assert_error "*NOREPLICAS*" {$master fcall f_default_flags 1 foo} assert_equal "12345" [$master fcall f_no_writes 1 foo] } test {With not enough good slaves, read in Lua script is still accepted} { $master config set min-slaves-max-lag 3 $master config set min-slaves-to-write 1 $master eval "redis.call('set','foo','bar')" 0 $master config set min-slaves-to-write 2 $master eval "return redis.call('get','foo')" 0 } {bar} test {With min-slaves-to-write: master not writable with lagged slave} { $master config set min-slaves-max-lag 2 $master config set min-slaves-to-write 1 assert_equal OK [$master set foo 123] assert_equal OK [$master eval "return redis.call('set','foo',12345)" 0] # Killing a slave to make it become a lagged slave. pause_process [srv 0 pid] # Waiting for slave kill. wait_for_condition 100 100 { [catch {$master set foo 123}] != 0 } else { fail "Master didn't become readonly" } assert_error "*NOREPLICAS*" {$master set foo 123} assert_error "*NOREPLICAS*" {$master eval "return redis.call('set','foo',12345)" 0} resume_process [srv 0 pid] } } } start_server {tags {"repl external:skip"}} { start_server {} { set master [srv -1 client] set master_host [srv -1 host] set master_port [srv -1 port] set slave [srv 0 client] test {First server should have role slave after SLAVEOF} { $slave slaveof $master_host $master_port wait_for_condition 50 100 { [s 0 master_link_status] eq {up} } else { fail "Replication not started." } } test {Replication of an expired key does not delete the expired key} { # This test is very likely to do a false positive if the wait_for_ofs_sync # takes longer than the expiration time, so give it a few more chances. # Go with 5 retries of increasing timeout, i.e. start with 500ms, then go # to 1000ms, 2000ms, 4000ms, 8000ms. set px_ms 500 for {set i 0} {$i < 5} {incr i} { wait_for_ofs_sync $master $slave $master debug set-active-expire 0 $master set k 1 px $px_ms wait_for_ofs_sync $master $slave pause_process [srv 0 pid] $master incr k after [expr $px_ms + 1] # Stopping the replica for one second to makes sure the INCR arrives # to the replica after the key is logically expired. resume_process [srv 0 pid] wait_for_ofs_sync $master $slave # Check that k is logically expired but is present in the replica. set res [$slave exists k] set errcode [catch {$slave debug object k} err] ; # Raises exception if k is gone. if {$res == 0 && $errcode == 0} { break } set px_ms [expr $px_ms * 2] } ;# for if {$::verbose} { puts "Replication of an expired key does not delete the expired key test attempts: $i" } assert_equal $res 0 assert_equal $errcode 0 } } } start_server {tags {"repl external:skip"}} { start_server {} { set master [srv -1 client] set master_host [srv -1 host] set master_port [srv -1 port] set slave [srv 0 client] test {First server should have role slave after SLAVEOF} { $slave slaveof $master_host $master_port wait_for_condition 50 100 { [s 0 role] eq {slave} } else { fail "Replication not started." } } test {Replication: commands with many arguments (issue #1221)} { # We now issue large MSET commands, that may trigger a specific # class of bugs, see issue #1221. for {set j 0} {$j < 100} {incr j} { set cmd [list mset] for {set x 0} {$x < 1000} {incr x} { lappend cmd [randomKey] [randomValue] } $master {*}$cmd } set retry 10 while {$retry && ([$master debug digest] ne [$slave debug digest])}\ { after 1000 incr retry -1 } assert {[$master dbsize] > 0} } test {spopwithcount rewrite srem command} { $master del myset set content {} for {set j 0} {$j < 4000} {} { lappend content [incr j] } $master sadd myset {*}$content $master spop myset 1023 $master spop myset 1024 $master spop myset 1025 assert_match 928 [$master scard myset] assert_match {*calls=3,*} [cmdrstat spop $master] wait_for_condition 50 100 { [status $slave master_repl_offset] == [status $master master_repl_offset] } else { fail "SREM replication inconsistency." } assert_match {*calls=4,*} [cmdrstat srem $slave] assert_match 928 [$slave scard myset] } test {Replication of SPOP command -- alsoPropagate() API} { $master del myset set size [expr 1+[randomInt 100]] set content {} for {set j 0} {$j < $size} {incr j} { lappend content [randomValue] } $master sadd myset {*}$content set count [randomInt 100] set result [$master spop myset $count] wait_for_condition 50 100 { [$master debug digest] eq [$slave debug digest] } else { fail "SPOP replication inconsistency" } } } } start_server {tags {"repl external:skip"}} { start_server {} { set master [srv -1 client] set master_host [srv -1 host] set master_port [srv -1 port] set replica [srv 0 client] test {First server should have role slave after SLAVEOF} { $replica slaveof $master_host $master_port wait_for_condition 50 100 { [s 0 role] eq {slave} } else { fail "Replication not started." } wait_for_sync $replica } test {Data divergence can happen under default conditions} { $replica config set propagation-error-behavior ignore $master debug replicate fake-command-1 # Wait for replication to normalize $master set foo bar2 $master wait 1 2000 # Make sure we triggered the error, by finding the critical # message and the fake command. assert_equal [count_log_message 0 "fake-command-1"] 1 assert_equal [count_log_message 0 "== CRITICAL =="] 1 } test {Data divergence is allowed on writable replicas} { $replica config set replica-read-only no $replica set number2 foo $master incrby number2 1 $master wait 1 2000 assert_equal [$master get number2] 1 assert_equal [$replica get number2] foo assert_equal [count_log_message 0 "incrby"] 1 } } }