Unverified Commit 7b9e9606 authored by debing.sun's avatar debing.sun Committed by GitHub
Browse files

Hash Field Expiration (#13303)

## Background

This PR introduces support for field-level expiration in Redis hashes. Previously, Redis supported expiration only at the key level, but this enhancement allows setting expiration times for individual fields within a hash.

## New commands
* HEXPIRE
* HEXPIREAT
* HEXPIRETIME
* HPERSIST
* HPEXPIRE
* HPEXPIREAT
* HPEXPIRETIME
* HPTTL
* HTTL

## Short example
from @moticless
```sh
127.0.0.1:6379>  hset myhash f1 v1 f2 v2 f3 v3                                                   
(integer) 3
127.0.0.1:6379>  hpexpire myhash 10000 NX fields 2 f2 f3                                         
1) (integer) 1
2) (integer) 1
127.0.0.1:6379>  hpttl myhash fields 3 f1 f2 f3                                                                                                                                                                         
1) (integer) -1
2) (integer) 9997
3) (integer) 9997
127.0.0.1:6379>  hgetall myhash  
1) "f3"
2) "v3"
3) "f2"
4) "v2"
5) "f1"
6) "v1"

... after 10 seconds ...

127.0.0.1:6379>  hgetall myhash  
1) "f1"
2) "v1"
127.0.0.1:6379>
```

## Expiration strategy
1. Integrate active
    Redis periodically performs active expiration and deletion of hash keys that contain expired fields, with a maximum attempt limit.
3. Lazy expiration
    When a client touches fields within a hash, Redis checks if the fields are expired. If a field is expired, it will be deleted. However, we do not delete expired fields during a traversal, we implicitly skip over them.

## RDB changes
Add two new rdb type s`RDB_TYPE_HASH_METADATA` and `RDB_TYPE_HASH_LISTPACK_EX`.

## Notification
1. Add `hpersist` notification for `HPERSIST` command.
5. Add `hexpire` notification for `HEXPIRE`, `HEXPIREAT`, `HPEXPIRE` and `HPEXPIREAT` commands.

## Internal
1. Add new data structure `ebuckets`, which is used to store TTL and keys, enabling quick retrieval of keys based on TTL.
2. Add new data structure `mstr` like sds, which is used to store a string with TTL.

This work was done by @moticless, @tezc, @ronen-kalish, @sundb, I just release it.
parents 2d1bb42c f0389f28
......@@ -1754,7 +1754,7 @@ void zsetTypeRandomElement(robj *zsetobj, unsigned long zsetsize, listpackEntry
*score = *(double*)dictGetVal(de);
} else if (zsetobj->encoding == OBJ_ENCODING_LISTPACK) {
listpackEntry val;
lpRandomPair(zsetobj->ptr, zsetsize, key, &val);
lpRandomPair(zsetobj->ptr, zsetsize, key, &val, 2);
if (score) {
if (val.sval) {
*score = zzlStrtod(val.sval,val.slen);
......@@ -4263,7 +4263,7 @@ void zrandmemberWithCountCommand(client *c, long l, int withscores) {
while (count) {
sample_count = count > limit ? limit : count;
count -= sample_count;
lpRandomPairs(zsetobj->ptr, sample_count, keys, vals);
lpRandomPairs(zsetobj->ptr, sample_count, keys, vals, 2);
zrandmemberReplyWithListpack(c, sample_count, keys, vals);
if (c->flags & CLIENT_CLOSE_ASAP)
break;
......@@ -4317,7 +4317,7 @@ void zrandmemberWithCountCommand(client *c, long l, int withscores) {
keys = zmalloc(sizeof(listpackEntry)*count);
if (withscores)
vals = zmalloc(sizeof(listpackEntry)*count);
serverAssert(lpRandomPairsUnique(zsetobj->ptr, count, keys, vals) == count);
serverAssert(lpRandomPairsUnique(zsetobj->ptr, count, keys, vals, 2) == count);
zrandmemberReplyWithListpack(c, count, keys, vals);
zfree(keys);
zfree(vals);
......
......@@ -21,6 +21,8 @@
#define REDIS_TEST_ACCURATE (1<<0)
#define REDIS_TEST_LARGE_MEMORY (1<<1)
#define REDIS_TEST_VALGRIND (1<<2)
#define REDIS_TEST_VERBOSE (1<<3)
extern int __failed_tests;
extern int __test_num;
......
......@@ -19,12 +19,14 @@ proc generate_collections {suffix elements} {
# add both string values and integers
if {$j % 2 == 0} {set val $j} else {set val "_$j"}
$rd hset hash$suffix $j $val
$rd hset hashmd$suffix $j $val
$rd hexpire hashmd$suffix [expr {int(rand() * 10000)}] FIELDS 1 $j
$rd lpush list$suffix $val
$rd zadd zset$suffix $j $val
$rd sadd set$suffix $val
$rd xadd stream$suffix * item 1 value $val
}
for {set j 0} {$j < $elements * 5} {incr j} {
for {set j 0} {$j < $elements * 7} {incr j} {
$rd read ; # Discard replies
}
$rd close
......
......@@ -59,6 +59,62 @@ test {corrupt payload: valid zipped hash header, dup records} {
}
}
test {corrupt payload: hash listpackex with invalid string TTL} {
start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
r config set sanitize-dump-payload yes
catch {
r restore key 0 "\x17\x2d\x2d\x00\x00\x00\x09\x00\x81\x61\x02\x01\x01\xf4\xa6\x96\x18\xb8\x8f\x01\x00\x00\x09\x82\x66\x31\x03\x82\x76\x31\x03\x83\x66\x6f\x6f\x04\x82\x66\x32\x03\x82\x76\x32\x03\x00\x01\xff\x0c\x00\xde\x40\xe5\x37\x51\x1c\x12\x56" replace
} err
assert_match "*Bad data format*" $err
r ping
}
}
test {corrupt payload: hash listpackex with TTL large than EB_EXPIRE_TIME_MAX} {
start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
r config set sanitize-dump-payload yes
catch {
r restore key 0 "\x17\x33\x33\x00\x00\x00\x09\x00\x00\x01\x00\x01\xf4\x01\xc5\x89\x95\x8f\x01\x00\x00\x09\x01\x01\x82\x5f\x31\x03\xf4\x29\x94\x97\x95\x8f\x01\x00\x00\x09\x02\x01\x02\x01\xf4\x01\x5e\xaf\x95\x8f\x01\x33\x00\x09\xff\x0c\x00\x7e\x4f\xf4\x33\xe9\xc5\x3e\x56" replace
} err
assert_match "*Bad data format*" $err
r ping
}
}
test {corrupt payload: hash listpackex with unordered TTL fields} {
start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
r config set sanitize-dump-payload yes
catch {
r restore key 0 "\x17\xc3\x30\x35\x14\x35\x00\x00\x00\t\x00\x82\x66\x32\x03\x82\x76\x32\x03\xf4\x80\x73\x16\xd1\x8f\x01\x20\x12\x02\x82\x66\x31\x20\x11\x03\x31\x03\xf4\x7f\xe0\x01\x11\x00\x33\x20\x11\x04\x33\x03\x00\x01\xff\x0c\x00\xf6\x70\x29\x57\x11\x68\x9d\xe5" replace
} err
assert_match "*Bad data format*" $err
r ping
}
}
test {corrupt payload: hash listpackex field without TTL should not be followed by field with TTL} {
start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
r config set sanitize-dump-payload yes
catch {
r restore key 0 "\x17\x2d\x2d\x00\x00\x00\x09\x00\x82\x66\x31\x03\x82\x76\x31\x03\x00\x01\x82\x66\x32\x03\x82\x76\x32\x03\xf4\xe0\x59\x7a\x96\x00\x00\x00\x00\x09\x82\x66\x33\x03\x82\x76\x33\x03\x00\x01\xff\x0c\x00\x42\x66\xd4\xbe\x17\xc3\x96\x72" replace
} err
assert_match "*Bad data format*" $err
r ping
}
}
test {corrupt payload: hash hashtable with TTL large than EB_EXPIRE_TIME_MAX} {
start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
r config set hash-max-listpack-entries 0
r config set sanitize-dump-payload yes
catch {
r restore key 0 "\x16\x02\x81\x00\x01\x00\x00\x00\x00\x00\x00\x02\x66\x31\x02\x76\x31\x81\x00\x01\x00\x00\x00\x00\x00\x00\x02\x66\x32\x02\x76\x32\x0c\x00\xb9\x3c\x65\x28\x40\x94\x58\x36" replace
} err
assert_match "*Bad data format*" $err
r ping
}
}
test {corrupt payload: quicklist big ziplist prev len} {
start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
r config set sanitize-dump-payload no
......
......@@ -179,6 +179,17 @@ start_server {} {
$master set $j somevalue px 10
}
##### hash-field-expiration
# Hashes of type OBJ_ENCODING_LISTPACK_EX won't be discarded during
# RDB load, even if they are expired.
$master hset myhash1 f1 v1 f2 v2 f3 v3
$master hpexpire myhash1 10 FIELDS 3 f1 f2 f3
# Hashes of type RDB_TYPE_HASH_METADATA will be discarded during RDB load.
$master config set hash-max-listpack-entries 0
$master hset myhash2 f1 v1 f2 v2
$master hpexpire myhash2 10 FIELDS 2 f1 f2
$master config set hash-max-listpack-entries 1
after 20
wait_for_condition 500 100 {
......
......@@ -416,4 +416,228 @@ start_server {} {
} {OK}
}
set server_path [tmpdir "server.partial-hfield-exp-test"]
# verifies writing and reading hash key with expiring and persistent fields
start_server [list overrides [list "dir" $server_path]] {
foreach {type lp_entries} {listpack 512 dict 0} {
test "HFE - save and load expired fields, expired soon after, or long after ($type)" {
r config set hash-max-listpack-entries $lp_entries
r FLUSHALL
r HMSET key a 1 b 2 c 3 d 4 e 5
# expected to be expired long after restart
r HEXPIREAT key 2524600800 FIELDS 1 a
# expected long TTL value (6 bytes) is saved and loaded correctly
r HPEXPIREAT key 188900976391764 FIELDS 1 b
# expected to be already expired after restart
r HPEXPIRE key 80 FIELDS 1 d
# expected to be expired soon after restart
r HPEXPIRE key 200 FIELDS 1 e
r save
# sleep 101 ms to make sure d will expire after restart
after 101
restart_server 0 true false
wait_done_loading r
assert_equal [lsort [r hgetall key]] "1 2 3 a b c"
assert_equal [r hpexpiretime key FIELDS 3 a b c] {2524600800000 188900976391764 -1}
assert_equal [s rdb_last_load_keys_loaded] 1
# wait until expired_hash_fields equals 2
wait_for_condition 10 100 {
[s expired_hash_fields] == 2
} else {
fail "Value of expired_hash_fields is not as expected"
}
}
}
}
set server_path [tmpdir "server.all-hfield-exp-test"]
# verifies writing hash with several expired keys, and active-expiring it on load
start_server [list overrides [list "dir" $server_path]] {
foreach {type lp_entries} {listpack 512 dict 0} {
test "HFE - save and load rdb all fields expired, ($type)" {
r config set hash-max-listpack-entries $lp_entries
r FLUSHALL
r HMSET key a 1 b 2 c 3 d 4
r HPEXPIRE key 100 FIELDS 4 a b c d
r save
# sleep 101 ms to make sure all fields will expire after restart
after 101
restart_server 0 true false
wait_done_loading r
# it is expected that no field was expired on load and the key was
# loaded, even though all its fields are actually expired.
assert_equal [s rdb_last_load_keys_loaded] 1
assert_equal [r hgetall key] {}
}
}
}
set server_path [tmpdir "server.listpack-to-dict-test"]
test "save listpack, load dict" {
start_server [list overrides [list "dir" $server_path enable-debug-command yes]] {
r config set hash-max-listpack-entries 512
r FLUSHALL
r HMSET key a 1 b 2 c 3 d 4
assert_match "*encoding:listpack*" [r debug object key]
r HPEXPIRE key 100 FIELDS 1 d
r save
# sleep 200 ms to make sure 'd' will expire after when reloading
after 200
# change configuration and reload - result should be dict-encoded key
r config set hash-max-listpack-entries 0
r debug reload nosave
# first verify d was not expired during load (no expiry when loading
# a hash that was saved listpack-encoded)
assert_equal [s rdb_last_load_keys_loaded] 1
# d should be lazy expired in hgetall
assert_equal [lsort [r hgetall key]] "1 2 3 a b c"
assert_match "*encoding:hashtable*" [r debug object key]
}
}
set server_path [tmpdir "server.dict-to-listpack-test"]
test "save dict, load listpack" {
start_server [list overrides [list "dir" $server_path enable-debug-command yes]] {
r config set hash-max-listpack-entries 0
r FLUSHALL
r HMSET key a 1 b 2 c 3 d 4
assert_match "*encoding:hashtable*" [r debug object key]
r HPEXPIRE key 200 FIELDS 1 d
r save
# sleep 201 ms to make sure 'd' will expire during reload
after 201
# change configuration and reload - result should be LP-encoded key
r config set hash-max-listpack-entries 512
r debug reload nosave
# verify d was expired during load
assert_equal [s rdb_last_load_keys_loaded] 1
assert_equal [lsort [r hgetall key]] "1 2 3 a b c"
assert_match "*encoding:listpack*" [r debug object key]
}
}
set server_path [tmpdir "server.active-expiry-after-load"]
# verifies a field is correctly expired by active expiry AFTER loading from RDB
foreach {type lp_entries} {listpack 512 dict 0} {
start_server [list overrides [list "dir" $server_path enable-debug-command yes]] {
test "active field expiry after load, ($type)" {
r config set hash-max-listpack-entries $lp_entries
r FLUSHALL
r HMSET key a 1 b 2 c 3 d 4 e 5 f 6
r HEXPIREAT key 2524600800 FIELDS 2 a b
r HPEXPIRE key 200 FIELDS 2 c d
r save
r debug reload nosave
# wait at most 2 secs to make sure 'c' and 'd' will active-expire
wait_for_condition 20 100 {
[s expired_hash_fields] == 2
} else {
fail "expired hash fields is [s expired_hash_fields] != 2"
}
assert_equal [s rdb_last_load_keys_loaded] 1
# hgetall might lazy expire fields, so it's only called after the stat asserts
assert_equal [lsort [r hgetall key]] "1 2 5 6 a b e f"
assert_equal [r hexpiretime key FIELDS 6 a b c d e f] {2524600800 2524600800 -2 -2 -1 -1}
}
}
}
set server_path [tmpdir "server.lazy-expiry-after-load"]
foreach {type lp_entries} {listpack 512 dict 0} {
start_server [list overrides [list "dir" $server_path enable-debug-command yes]] {
test "lazy field expiry after load, ($type)" {
r config set hash-max-listpack-entries $lp_entries
r debug set-active-expire 0
r FLUSHALL
r HMSET key a 1 b 2 c 3 d 4 e 5 f 6
r HEXPIREAT key 2524600800 FIELDS 2 a b
r HPEXPIRE key 200 FIELDS 2 c d
r save
r debug reload nosave
# sleep 500 msec to make sure 'c' and 'd' will lazy-expire when calling hgetall
after 500
assert_equal [s rdb_last_load_keys_loaded] 1
assert_equal [s expired_hash_fields] 0
# hgetall will lazy expire fields, so it's only called after the stat asserts
assert_equal [lsort [r hgetall key]] "1 2 5 6 a b e f"
assert_equal [r hexpiretime key FIELDS 6 a b c d e f] {2524600800 2524600800 -2 -2 -1 -1}
}
}
}
set server_path [tmpdir "server.unexpired-items-rax-list-boundary"]
foreach {type lp_entries} {listpack 512 dict 0} {
start_server [list overrides [list "dir" $server_path enable-debug-command yes]] {
test "load un-expired items below and above rax-list boundary, ($type)" {
r config set hash-max-listpack-entries $lp_entries
r flushall
set hash_sizes {15 16 17 31 32 33}
foreach h $hash_sizes {
for {set i 1} {$i <= $h} {incr i} {
r hset key$h f$i v$i
r hexpireat key$h 2524600800 FIELDS 1 f$i
}
}
r save
restart_server 0 true false
wait_done_loading r
set hash_sizes {15 16 17 31 32 33}
foreach h $hash_sizes {
for {set i 1} {$i <= $h} {incr i} {
# random expiration time
assert_equal [r hget key$h f$i] v$i
assert_equal [r hexpiretime key$h FIELDS 1 f$i] 2524600800
}
}
}
}
}
} ;# tags
......@@ -293,6 +293,9 @@ proc findKeyWithType {r type} {
proc createComplexDataset {r ops {opt {}}} {
set useexpire [expr {[lsearch -exact $opt useexpire] != -1}]
# TODO: Remove usehexpire on next commit, when RDB will support replication
set usehexpire [expr {[lsearch -exact $opt usehexpire] != -1}]
if {[lsearch -exact $opt usetag] != -1} {
set tag "{t}"
} else {
......@@ -386,6 +389,10 @@ proc createComplexDataset {r ops {opt {}}} {
{hash} {
randpath {{*}$r hset $k $f $v} \
{{*}$r hdel $k $f}
if { [{*}$r hexists $k $f] && $usehexpire && rand() < 0.5} {
{*}$r hexpire $k 1000 FIELDS 1 $f
}
}
}
}
......@@ -438,8 +445,14 @@ proc csvdump r {
hash {
set fields [{*}$r hgetall $k]
set newfields {}
foreach {k v} $fields {
lappend newfields [list $k $v]
foreach {f v} $fields {
set expirylist [{*}$r hexpiretime $k FIELDS 1 $f]
if {$expirylist eq (-1)} {
lappend newfields [list $f $v]
} else {
set e [lindex $expirylist 0]
lappend newfields [list $f $e $v] # TODO: extract the actual ttl value from the list in $e
}
}
set fields [lsort -index 0 $newfields]
foreach kv $fields {
......
......@@ -34,6 +34,7 @@ set ::all_tests {
unit/type/set
unit/type/zset
unit/type/hash
unit/type/hash-field-expire
unit/type/stream
unit/type/stream-cgroups
unit/sort
......
......@@ -503,6 +503,105 @@ run_solo {defrag} {
$rd_pubsub close
}
test "Active Defrag HFE: $type" {
r flushdb
r config resetstat
r config set hz 100
r config set activedefrag no
# TODO: Lower the threshold after defraging the ebuckets.
# Now just to ensure that the reference is updated correctly.
r config set active-defrag-threshold-lower 12
r config set active-defrag-cycle-min 65
r config set active-defrag-cycle-max 75
r config set active-defrag-ignore-bytes 1500kb
r config set maxmemory 0
r config set hash-max-listpack-value 512
r config set hash-max-listpack-entries 10
# Populate memory with interleaving hash field of same size
set n 3000
set fields 16 ;# make all the fields in an eblist.
set dummy_field "[string repeat x 400]"
set rd [redis_deferring_client]
for {set i 0} {$i < $n} {incr i} {
for {set j 0} {$j < $fields} {incr j} {
$rd hset h$i f$j $dummy_field
$rd hexpire h$i 9999999 FIELDS 1 f$j
$rd set "k$i$j" $dummy_field
}
}
for {set j 0} {$j < [expr $n*$fields]} {incr j} {
$rd read ; # Discard hset replies
$rd read ; # Discard hexpire replies
$rd read ; # Discard set replies
}
# Coverage for listpackex.
r hset h_lpex f0 $dummy_field
r hexpire h_lpex 9999999 FIELDS 1 f0
assert_encoding listpackex h_lpex
after 120 ;# serverCron only updates the info once in 100ms
if {$::verbose} {
puts "used [s allocator_allocated]"
puts "rss [s allocator_active]"
puts "frag [s allocator_frag_ratio]"
puts "frag_bytes [s allocator_frag_bytes]"
}
assert_lessthan [s allocator_frag_ratio] 1.05
# Delete all the keys to create fragmentation
for {set i 0} {$i < $n} {incr i} {
for {set j 0} {$j < $fields} {incr j} {
r del "k$i$j"
}
}
$rd close
after 120 ;# serverCron only updates the info once in 100ms
if {$::verbose} {
puts "used [s allocator_allocated]"
puts "rss [s allocator_active]"
puts "frag [s allocator_frag_ratio]"
puts "frag_bytes [s allocator_frag_bytes]"
}
assert_morethan [s allocator_frag_ratio] 1.35
catch {r config set activedefrag yes} e
if {[r config get activedefrag] eq "activedefrag yes"} {
# wait for the active defrag to start working (decision once a second)
wait_for_condition 50 100 {
[s total_active_defrag_time] ne 0
} else {
after 120 ;# serverCron only updates the info once in 100ms
puts [r info memory]
puts [r info stats]
puts [r memory malloc-stats]
fail "defrag not started."
}
# wait for the active defrag to stop working
wait_for_condition 500 100 {
[s active_defrag_running] eq 0
} else {
after 120 ;# serverCron only updates the info once in 100ms
puts [r info memory]
puts [r memory malloc-stats]
fail "defrag didn't stop."
}
# test the fragmentation is lower
after 120 ;# serverCron only updates the info once in 100ms
if {$::verbose} {
puts "used [s allocator_allocated]"
puts "rss [s allocator_active]"
puts "frag [s allocator_frag_ratio]"
puts "frag_bytes [s allocator_frag_bytes]"
}
assert_lessthan_equal [s allocator_frag_ratio] 1.5
}
}
if {$type eq "standalone"} { ;# skip in cluster mode
test "Active defrag big list: $type" {
r flushdb
......
......@@ -124,7 +124,8 @@ start_server {tags {"other"}} {
if {$::accurate} {set numops 10000} else {set numops 1000}
test {Check consistency of different data types after a reload} {
r flushdb
createComplexDataset r $numops usetag
# TODO: integrate usehexpire following next commit that will support replication
createComplexDataset r $numops {usetag usehexpire}
if {$::ignoredigest} {
set _ 1
} else {
......
......@@ -353,17 +353,29 @@ start_server {tags {"pubsub network"}} {
$rd1 close
}
test "Keyspace notifications: hash events test" {
foreach {type max_lp_entries} {listpackex 512 hashtable 0} {
test "Keyspace notifications: hash events test ($type)" {
r config set hash-max-listpack-entries $max_lp_entries
r config set notify-keyspace-events Kh
r del myhash
set rd1 [redis_deferring_client]
assert_equal {1} [psubscribe $rd1 *]
r hmset myhash yes 1 no 0
r hincrby myhash yes 10
r hexpire myhash 999999 FIELDS 1 yes
r hexpireat myhash [expr {[clock seconds] + 999999}] NX FIELDS 1 no
r hpexpire myhash 5 FIELDS 1 yes
r hpersist myhash FIELDS 1 yes
assert_encoding $type myhash
assert_equal "pmessage * __keyspace@${db}__:myhash hset" [$rd1 read]
assert_equal "pmessage * __keyspace@${db}__:myhash hincrby" [$rd1 read]
assert_equal "pmessage * __keyspace@${db}__:myhash hexpire" [$rd1 read]
assert_equal "pmessage * __keyspace@${db}__:myhash hexpire" [$rd1 read]
assert_equal "pmessage * __keyspace@${db}__:myhash hexpire" [$rd1 read]
assert_equal "pmessage * __keyspace@${db}__:myhash hpersist" [$rd1 read]
$rd1 close
}
} ;# foreach
test "Keyspace notifications: stream events test" {
r config set notify-keyspace-events Kt
......
......@@ -277,6 +277,32 @@ proc test_scan {type} {
set res [r hscan hash 0 count 1000 novalues]
assert_equal [lsort $keys2] [lsort [lindex $res 1]]
}
test "{$type} HSCAN with large value $enc" {
r del hash
if {$enc eq {listpack}} {
set count 60
} else {
set count 170
}
set val1 [string repeat "1" $count]
r hset hash $val1 $val1
set val2 [string repeat "2" $count]
r hset hash $val2 $val2
set res [lsort [lindex [r hscan hash 0] 1]]
assert_equal $val1 [lindex $res 0]
assert_equal $val1 [lindex $res 1]
assert_equal $val2 [lindex $res 2]
assert_equal $val2 [lindex $res 3]
set res [lsort [lindex [r hscan hash 0 novalues] 1]]
assert_equal $val1 [lindex $res 0]
assert_equal $val2 [lindex $res 1]
}
}
foreach enc {listpack skiplist} {
......
######## HEXPIRE family commands
# Field does not exists
set E_NO_FIELD -2
# Specified NX | XX | GT | LT condition not met
set E_FAIL 0
# expiration time set/updated
set E_OK 1
# Field deleted because the specified expiration time is in the past
set E_DELETED 2
######## HTTL family commands
set T_NO_FIELD -2
set T_NO_EXPIRY -1
######## HPERIST
set P_NO_FIELD -2
set P_NO_EXPIRY -1
set P_OK 1
############################### AUX FUNCS ######################################
proc get_hashes_with_expiry_fields {r} {
set input_string [r info keyspace]
set hash_count 0
foreach line [split $input_string \n] {
if {[regexp {hashes_with_expiry_fields=(\d+)} $line -> value]} {
return $value
}
}
return 0
}
proc create_hash {key entries} {
r del $key
foreach entry $entries {
r hset $key [lindex $entry 0] [lindex $entry 1]
}
}
proc get_keys {l} {
set res {}
foreach entry $l {
set key [lindex $entry 0]
lappend res $key
}
return $res
}
proc cmp_hrandfield_result {hash_name expected_result} {
# Accumulate hrandfield results
unset -nocomplain myhash
array set myhash {}
for {set i 0} {$i < 100} {incr i} {
set key [r hrandfield $hash_name]
set myhash($key) 1
}
set res [lsort [array names myhash]]
if {$res eq $expected_result} {
return 1
} else {
return $res
}
}
proc dumpAllHashes {client} {
set keyAndFields(0,0) 0
unset keyAndFields
# keep keys sorted for comparison
foreach key [lsort [$client keys *]] {
set fields [$client hgetall $key]
foreach f $fields {
set keyAndFields($key,$f) [$client hpexpiretime $key FIELDS 1 $f]
}
}
return [array get keyAndFields]
}
proc hrandfieldTest {activeExpireConfig} {
r debug set-active-expire $activeExpireConfig
r del myhash
set contents {{field1 1} {field2 2} }
create_hash myhash $contents
set factorValgrind [expr {$::valgrind ? 2 : 1}]
# Set expiration time for field1 and field2 such that field1 expires first
r hpexpire myhash 1 NX FIELDS 1 field1
r hpexpire myhash 100 NX FIELDS 1 field2
# On call hrandfield command lazy expire deletes field1 first
wait_for_condition 8 10 {
[cmp_hrandfield_result myhash "field2"] == 1
} else {
fail "Expected field2 to be returned by HRANDFIELD."
}
# On call hrandfield command lazy expire deletes field2 as well
wait_for_condition 8 20 {
[cmp_hrandfield_result myhash "{}"] == 1
} else {
fail "Expected {} to be returned by HRANDFIELD."
}
# restore the default value
r debug set-active-expire 1
}
############################### TESTS #########################################
start_server {tags {"external:skip needs:debug"}} {
foreach type {listpackex hashtable} {
if {$type eq "hashtable"} {
r config set hash-max-listpack-entries 0
} else {
r config set hash-max-listpack-entries 512
}
test "HEXPIRE/HEXPIREAT/HPEXPIRE/HPEXPIREAT - Returns empty array if key does not exist" {
r del myhash
# Make sure we can distinguish between an empty array and a null response
r readraw 1
assert_equal {*0} [r HEXPIRE myhash 1000 FIELDS 1 a]
assert_equal {*0} [r HEXPIREAT myhash 1000 FIELDS 1 a]
assert_equal {*0} [r HPEXPIRE myhash 1000 FIELDS 1 a]
assert_equal {*0} [r HPEXPIREAT myhash 1000 FIELDS 1 a]
r readraw 0
}
test "HPEXPIRE(AT) - Test 'NX' flag ($type)" {
r del myhash
r hset myhash field1 value1 field2 value2 field3 value3
assert_equal [r hpexpire myhash 1000 NX FIELDS 1 field1] [list $E_OK]
assert_equal [r hpexpire myhash 1000 NX FIELDS 2 field1 field2] [list $E_FAIL $E_OK]
r del myhash
r hset myhash field1 value1 field2 value2 field3 value3
assert_equal [r hpexpireat myhash [expr {([clock seconds]+1000)*1000}] NX FIELDS 1 field1] [list $E_OK]
assert_equal [r hpexpireat myhash [expr {([clock seconds]+1000)*1000}] NX FIELDS 2 field1 field2] [list $E_FAIL $E_OK]
}
test "HPEXPIRE(AT) - Test 'XX' flag ($type)" {
r del myhash
r hset myhash field1 value1 field2 value2 field3 value3
assert_equal [r hpexpire myhash 1000 NX FIELDS 2 field1 field2] [list $E_OK $E_OK]
assert_equal [r hpexpire myhash 1000 XX FIELDS 2 field1 field3] [list $E_OK $E_FAIL]
r del myhash
r hset myhash field1 value1 field2 value2 field3 value3
assert_equal [r hpexpireat myhash [expr {([clock seconds]+1000)*1000}] NX FIELDS 2 field1 field2] [list $E_OK $E_OK]
assert_equal [r hpexpireat myhash [expr {([clock seconds]+1000)*1000}] XX FIELDS 2 field1 field3] [list $E_OK $E_FAIL]
}
test "HPEXPIRE(AT) - Test 'GT' flag ($type)" {
r del myhash
r hset myhash field1 value1 field2 value2
assert_equal [r hpexpire myhash 1000 NX FIELDS 1 field1] [list $E_OK]
assert_equal [r hpexpire myhash 2000 NX FIELDS 1 field2] [list $E_OK]
assert_equal [r hpexpire myhash 1500 GT FIELDS 2 field1 field2] [list $E_OK $E_FAIL]
r del myhash
r hset myhash field1 value1 field2 value2
assert_equal [r hpexpireat myhash [expr {([clock seconds]+1000)*1000}] NX FIELDS 1 field1] [list $E_OK]
assert_equal [r hpexpireat myhash [expr {([clock seconds]+2000)*1000}] NX FIELDS 1 field2] [list $E_OK]
assert_equal [r hpexpireat myhash [expr {([clock seconds]+1500)*1000}] GT FIELDS 2 field1 field2] [list $E_OK $E_FAIL]
}
test "HPEXPIRE(AT) - Test 'LT' flag ($type)" {
r del myhash
r hset myhash field1 value1 field2 value2 field3 value3
assert_equal [r hpexpire myhash 1000 NX FIELDS 1 field1] [list $E_OK]
assert_equal [r hpexpire myhash 2000 NX FIELDS 1 field2] [list $E_OK]
assert_equal [r hpexpire myhash 1500 LT FIELDS 3 field1 field2 field3] [list $E_FAIL $E_OK $E_OK]
r del myhash
r hset myhash field1 value1 field2 value2 field3 value3
assert_equal [r hpexpireat myhash [expr {([clock seconds]+1000)*1000}] NX FIELDS 1 field1] [list $E_OK]
assert_equal [r hpexpireat myhash [expr {([clock seconds]+2000)*1000}] NX FIELDS 1 field2] [list $E_OK]
assert_equal [r hpexpireat myhash [expr {([clock seconds]+1500)*1000}] LT FIELDS 3 field1 field2 field3] [list $E_FAIL $E_OK $E_OK]
}
test "HPEXPIREAT - field not exists or TTL is in the past ($type)" {
r del myhash
r hset myhash f1 v1 f2 v2 f4 v4
r hexpire myhash 1000 NX FIELDS 1 f4
assert_equal [r hpexpireat myhash [expr {([clock seconds]-1)*1000}] NX FIELDS 4 f1 f2 f3 f4] "$E_DELETED $E_DELETED $E_NO_FIELD $E_FAIL"
assert_equal [r hexists myhash field1] 0
}
test "HPEXPIRE - wrong number of arguments ($type)" {
r del myhash
r hset myhash f1 v1
assert_error {*Parameter `numFields` should be greater than 0} {r hpexpire myhash 1000 NX FIELDS 0 f1 f2 f3}
assert_error {*Parameter `numFileds` is more than number of arguments} {r hpexpire myhash 1000 NX FIELDS 4 f1 f2 f3}
}
test "HPEXPIRE - parameter expire-time near limit of 2^48 ($type)" {
r del myhash
r hset myhash f1 v1
# below & above
assert_equal [r hpexpire myhash [expr (1<<48) - [clock milliseconds] - 1000 ] FIELDS 1 f1] [list $E_OK]
assert_error {*invalid expire time*} {r hpexpire myhash [expr (1<<48) - [clock milliseconds] + 100 ] FIELDS 1 f1}
}
test "Lazy Expire - fields are lazy deleted ($type)" {
# TODO remove the SELECT once dbid will be embedded inside dict/listpack
r select 0
r debug set-active-expire 0
r del myhash
r hset myhash f1 v1 f2 v2 f3 v3
r hpexpire myhash 1 NX FIELDS 3 f1 f2 f3
after 5
# Verify that still exists even if all fields are expired
assert_equal 1 [r EXISTS myhash]
# Verify that len counts also expired fields
assert_equal 3 [r HLEN myhash]
# Trying access to expired field should delete it. Len should be updated
assert_equal 0 [r hexists myhash f1]
assert_equal 2 [r HLEN myhash]
# Trying access another expired field should delete it. Len should be updated
assert_equal "" [r hget myhash f2]
assert_equal 1 [r HLEN myhash]
# Trying access last expired field should delete it. hash shouldn't exists afterward.
assert_equal 0 [r hstrlen myhash f3]
assert_equal 0 [r HLEN myhash]
assert_equal 0 [r EXISTS myhash]
# Restore default
r debug set-active-expire 1
}
test "Active Expire - deletes hash that all its fields got expired ($type)" {
r flushall
set hash_sizes {1 15 16 17 31 32 33 40}
foreach h $hash_sizes {
for {set i 1} {$i <= $h} {incr i} {
# random expiration time
r hset hrand$h f$i v$i
r hpexpire hrand$h [expr {50 + int(rand() * 50)}] FIELDS 1 f$i
assert_equal 1 [r HEXISTS hrand$h f$i]
# same expiration time
r hset same$h f$i v$i
r hpexpire same$h 100 FIELDS 1 f$i
assert_equal 1 [r HEXISTS same$h f$i]
# same expiration time
r hset mix$h f$i v$i fieldWithoutExpire$i v$i
r hpexpire mix$h 100 FIELDS 1 f$i
assert_equal 1 [r HEXISTS mix$h f$i]
}
}
# Wait for active expire
wait_for_condition 50 20 { [r EXISTS same40] == 0 } else { fail "hash `same40` should be expired" }
# Verify that all fields got expired and keys got deleted
foreach h $hash_sizes {
wait_for_condition 50 20 {
[r HLEN mix$h] == $h
} else {
fail "volatile fields of hash `mix$h` should be expired"
}
for {set i 1} {$i <= $h} {incr i} {
assert_equal 0 [r HEXISTS mix$h f$i]
}
assert_equal 0 [r EXISTS hrand$h]
assert_equal 0 [r EXISTS same$h]
}
}
test "HPEXPIRE - Flushall deletes all pending expired fields ($type)" {
r del myhash
r hset myhash field1 value1 field2 value2
r hpexpire myhash 10000 NX FIELDS 1 field1
r hpexpire myhash 10000 NX FIELDS 1 field2
r flushall
r del myhash
r hset myhash field1 value1 field2 value2
r hpexpire myhash 10000 NX FIELDS 1 field1
r hpexpire myhash 10000 NX FIELDS 1 field2
r flushall async
}
test "HTTL/HPTTL - Returns empty array if key does not exist" {
r del myhash
# Make sure we can distinguish between an empty array and a null response
r readraw 1
assert_equal {*0} [r HTTL myhash FIELDS 1 a]
assert_equal {*0} [r HPTTL myhash FIELDS 1 a]
r readraw 0
}
test "HTTL/HPTTL - Input validation gets failed on nonexists field or field without expire ($type)" {
r del myhash
r HSET myhash field1 value1 field2 value2
r HPEXPIRE myhash 1000 NX FIELDS 1 field1
foreach cmd {HTTL HPTTL} {
assert_equal [r $cmd non_exists_key FIELDS 1 f] {}
assert_equal [r $cmd myhash FIELDS 2 field2 non_exists_field] "$T_NO_EXPIRY $T_NO_FIELD"
# Set numFields less than actual number of fields. Fine.
assert_equal [r $cmd myhash FIELDS 1 non_exists_field1 non_exists_field2] "$T_NO_FIELD"
}
}
test "HTTL/HPTTL - returns time to live in seconds/msillisec ($type)" {
r del myhash
r HSET myhash field1 value1 field2 value2
r HPEXPIRE myhash 2000 NX FIELDS 2 field1 field2
set ttlArray [r HTTL myhash FIELDS 2 field1 field2]
assert_range [lindex $ttlArray 0] 1 2
set ttl [r HPTTL myhash FIELDS 1 field1]
assert_range $ttl 1000 2000
}
test "HEXPIRETIME/HPEXPIRETIME - Returns empty array if key does not exist" {
r del myhash
# Make sure we can distinguish between an empty array and a null response
r readraw 1
assert_equal {*0} [r HEXPIRETIME myhash FIELDS 1 a]
assert_equal {*0} [r HPEXPIRETIME myhash FIELDS 1 a]
r readraw 0
}
test "HEXPIRETIME - returns TTL in Unix timestamp ($type)" {
r del myhash
r HSET myhash field1 value1
r HPEXPIRE myhash 1000 NX FIELDS 1 field1
set lo [expr {[clock seconds] + 1}]
set hi [expr {[clock seconds] + 2}]
assert_range [r HEXPIRETIME myhash FIELDS 1 field1] $lo $hi
assert_range [r HPEXPIRETIME myhash FIELDS 1 field1] [expr $lo*1000] [expr $hi*1000]
}
test "HTTL/HPTTL - Verify TTL progress until expiration ($type)" {
r del myhash
r hset myhash field1 value1 field2 value2
r hpexpire myhash 1000 NX FIELDS 1 field1
assert_range [r HPTTL myhash FIELDS 1 field1] 100 1000
assert_range [r HTTL myhash FIELDS 1 field1] 0 1
after 100
assert_range [r HPTTL myhash FIELDS 1 field1] 1 901
after 910
assert_equal [r HPTTL myhash FIELDS 1 field1] $T_NO_FIELD
assert_equal [r HTTL myhash FIELDS 1 field1] $T_NO_FIELD
}
test "HPEXPIRE - DEL hash with non expired fields (valgrind test) ($type)" {
r del myhash
r hset myhash field1 value1 field2 value2
r hpexpire myhash 10000 NX FIELDS 1 field1
r del myhash
}
test "HEXPIREAT - Set time in the past ($type)" {
r del myhash
r hset myhash field1 value1
assert_equal [r hexpireat myhash [expr {[clock seconds] - 1}] NX FIELDS 1 field1] $E_DELETED
assert_equal [r hexists myhash field1] 0
}
test "HEXPIREAT - Set time and then get TTL ($type)" {
r del myhash
r hset myhash field1 value1
r hexpireat myhash [expr {[clock seconds] + 2}] NX FIELDS 1 field1
assert_range [r hpttl myhash FIELDS 1 field1] 1000 2000
assert_range [r httl myhash FIELDS 1 field1] 1 2
r hexpireat myhash [expr {[clock seconds] + 5}] XX FIELDS 1 field1
assert_range [r httl myhash FIELDS 1 field1] 4 5
}
test "Lazy Expire - delete hash with expired fields ($type)" {
r del myhash
r debug set-active-expire 0
r hset myhash k v
r hpexpire myhash 1 NX FIELDS 1 k
after 5
r del myhash
r debug set-active-expire 1
}
# OPEN: To decide if to delete expired fields at start of HRANDFIELD.
# test "Test HRANDFIELD does not return expired fields ($type)" {
# hrandfieldTest 0
# hrandfieldTest 1
# }
test "Test HRANDFIELD can return expired fields ($type)" {
r debug set-active-expire 0
r del myhash
r hset myhash f1 v1 f2 v2 f3 v3 f4 v4 f5 v5
r hpexpire myhash 1 NX FIELDS 4 f1 f2 f3 f4
after 5
set res [cmp_hrandfield_result myhash "f1 f2 f3 f4 f5"]
assert {$res == 1}
r debug set-active-expire 1
}
test "Lazy Expire - HLEN does count expired fields ($type)" {
# Enforce only lazy expire
r debug set-active-expire 0
r del h1 h4 h18 h20
r hset h1 k1 v1
r hpexpire h1 1 NX FIELDS 1 k1
r hset h4 k1 v1 k2 v2 k3 v3 k4 v4
r hpexpire h4 1 NX FIELDS 3 k1 k3 k4
# beyond 16 fields: HFE DS (ebuckets) converts from list to rax
r hset h18 k1 v1 k2 v2 k3 v3 k4 v4 k5 v5 k6 v6 k7 v7 k8 v8 k9 v9 k10 v10 k11 v11 k12 v12 k13 v13 k14 v14 k15 v15 k16 v16 k17 v17 k18 v18
r hpexpire h18 1 NX FIELDS 18 k1 k2 k3 k4 k5 k6 k7 k8 k9 k10 k11 k12 k13 k14 k15 k16 k17 k18
r hset h20 k1 v1 k2 v2 k3 v3 k4 v4 k5 v5 k6 v6 k7 v7 k8 v8 k9 v9 k10 v10 k11 v11 k12 v12 k13 v13 k14 v14 k15 v15 k16 v16 k17 v17 k18 v18 k19 v19 k20 v20
r hpexpire h20 1 NX FIELDS 2 k1 k2
after 10
assert_equal [r hlen h1] 1
assert_equal [r hlen h4] 4
assert_equal [r hlen h18] 18
assert_equal [r hlen h20] 20
# Restore to support active expire
r debug set-active-expire 1
}
test "Lazy Expire - HSCAN does not report expired fields ($type)" {
# Enforce only lazy expire
r debug set-active-expire 0
r del h1 h20 h4 h18 h20
r hset h1 01 01
r hpexpire h1 1 NX FIELDS 1 01
r hset h4 01 01 02 02 03 03 04 04
r hpexpire h4 1 NX FIELDS 3 01 03 04
# beyond 16 fields hash-field expiration DS (ebuckets) converts from list to rax
r hset h18 01 01 02 02 03 03 04 04 05 05 06 06 07 07 08 08 09 09 10 10 11 11 12 12 13 13 14 14 15 15 16 16 17 17 18 18
r hpexpire h18 1 NX FIELDS 18 01 02 03 04 05 06 07 08 09 10 11 12 13 14 15 16 17 18
r hset h20 01 01 02 02 03 03 04 04 05 05 06 06 07 07 08 08 09 09 10 10 11 11 12 12 13 13 14 14 15 15 16 16 17 17 18 18 19 19 20 20
r hpexpire h20 1 NX FIELDS 2 01 02
after 10
# Verify SCAN does not report expired fields
assert_equal [lsort -unique [lindex [r hscan h1 0 COUNT 10] 1]] ""
assert_equal [lsort -unique [lindex [r hscan h4 0 COUNT 10] 1]] "02"
assert_equal [lsort -unique [lindex [r hscan h18 0 COUNT 10] 1]] ""
assert_equal [lsort -unique [lindex [r hscan h20 0 COUNT 100] 1]] "03 04 05 06 07 08 09 10 11 12 13 14 15 16 17 18 19 20"
# Restore to support active expire
r debug set-active-expire 1
}
test "Test HSCAN with mostly expired fields return empty result ($type)" {
r debug set-active-expire 0
# Create hash with 1000 fields and 999 of them will be expired
r del myhash
for {set i 1} {$i <= 1000} {incr i} {
r hset myhash field$i value$i
if {$i > 1} {
r hpexpire myhash 1 NX FIELDS 1 field$i
}
}
after 3
# Verify iterative HSCAN returns either empty result or only the first field
set countEmptyResult 0
set cur 0
while 1 {
set res [r hscan myhash $cur]
set cur [lindex $res 0]
# if the result is not empty, it should contain only the first field
if {[llength [lindex $res 1]] > 0} {
assert_equal [lindex $res 1] "field1 value1"
} else {
incr countEmptyResult
}
if {$cur == 0} break
}
assert {$countEmptyResult > 0}
r debug set-active-expire 1
}
test "Lazy Expire - verify various HASH commands handling expired fields ($type)" {
# Enforce only lazy expire
r debug set-active-expire 0
r del h1 h2 h3 h4 h5 h18
r hset h1 01 01
r hset h2 01 01 02 02
r hset h3 01 01 02 02 03 03
r hset h4 1 99 2 99 3 99 4 99
r hset h5 1 1 2 22 3 333 4 4444 5 55555
r hset h6 01 01 02 02 03 03 04 04 05 05 06 06
r hset h18 01 01 02 02 03 03 04 04 05 05 06 06 07 07 08 08 09 09 10 10 11 11 12 12 13 13 14 14 15 15 16 16 17 17 18 18
r hpexpire h1 100 NX FIELDS 1 01
r hpexpire h2 100 NX FIELDS 1 01
r hpexpire h2 100 NX FIELDS 1 02
r hpexpire h3 100 NX FIELDS 1 01
r hpexpire h4 100 NX FIELDS 1 2
r hpexpire h5 100 NX FIELDS 1 3
r hpexpire h6 100 NX FIELDS 1 05
r hpexpire h18 100 NX FIELDS 17 01 02 03 04 05 06 07 08 09 10 11 12 13 14 15 16 17
after 150
# Verify HDEL not ignore expired field. It is too much overhead to check
# if the field is expired before deletion.
assert_equal [r HDEL h1 01] "1"
# Verify HGET ignore expired field
assert_equal [r HGET h2 01] ""
assert_equal [r HGET h2 02] ""
assert_equal [r HGET h3 01] ""
assert_equal [r HGET h3 02] "02"
assert_equal [r HGET h3 03] "03"
# Verify HINCRBY ignore expired field
assert_equal [r HINCRBY h4 2 1] "1"
assert_equal [r HINCRBY h4 3 1] "100"
# Verify HSTRLEN ignore expired field
assert_equal [r HSTRLEN h5 3] "0"
assert_equal [r HSTRLEN h5 4] "4"
assert_equal [lsort [r HKEYS h6]] "01 02 03 04 06"
# Verify HEXISTS ignore expired field
assert_equal [r HEXISTS h18 07] "0"
assert_equal [r HEXISTS h18 18] "1"
# Verify HVALS ignore expired field
assert_equal [lsort [r HVALS h18]] "18"
# Restore to support active expire
r debug set-active-expire 1
}
test "A field with TTL overridden with another value (TTL discarded) ($type)" {
r del myhash
r hset myhash field1 value1 field2 value2 field3 value3
r hpexpire myhash 10000 NX FIELDS 1 field1
r hpexpire myhash 1 NX FIELDS 1 field2
# field2 TTL will be discarded
r hset myhash field2 value4
after 5
# Expected TTL will be discarded
assert_equal [r hget myhash field2] "value4"
assert_equal [r httl myhash FIELDS 2 field2 field3] "$T_NO_EXPIRY $T_NO_EXPIRY"
assert_not_equal [r httl myhash FIELDS 1 field1] "$T_NO_EXPIRY"
}
test "Modify TTL of a field ($type)" {
r del myhash
r hset myhash field1 value1
r hpexpire myhash 200000 NX FIELDS 1 field1
r hpexpire myhash 1000000 XX FIELDS 1 field1
after 15
assert_equal [r hget myhash field1] "value1"
assert_range [r hpttl myhash FIELDS 1 field1] 900000 1000000
}
test "Test return value of set operation ($type)" {
r del myhash
r hset myhash f1 v1 f2 v2
r hexpire myhash 100000 FIELDS 1 f1
assert_equal [r hset myhash f2 v2] 0
assert_equal [r hset myhash f3 v3] 1
assert_equal [r hset myhash f3 v3 f4 v4] 1
assert_equal [r hset myhash f3 v3 f5 v5 f6 v6] 2
}
test "Test HGETALL not return expired fields ($type)" {
# Test with small hash
r debug set-active-expire 0
r del myhash
r hset myhash1 f1 v1 f2 v2 f3 v3 f4 v4 f5 v5 f6 v6
r hpexpire myhash1 1 NX FIELDS 3 f2 f4 f6
after 10
assert_equal [lsort [r hgetall myhash1]] "f1 f3 f5 v1 v3 v5"
# Test with large hash
r del myhash
for {set i 1} {$i <= 600} {incr i} {
r hset myhash f$i v$i
if {$i > 3} { r hpexpire myhash 1 NX FIELDS 1 f$i }
}
after 10
assert_equal [lsort [r hgetall myhash]] [lsort "f1 f2 f3 v1 v2 v3"]
r debug set-active-expire 1
}
test "Test RENAME hash with fields to be expired ($type)" {
r debug set-active-expire 0
r del myhash
r hset myhash field1 value1
r hpexpire myhash 20 NX FIELDS 1 field1
r rename myhash myhash2
assert_equal [r exists myhash] 0
assert_range [r hpttl myhash2 FIELDS 1 field1] 1 20
after 25
# Verify the renamed key exists
assert_equal [r exists myhash2] 1
r debug set-active-expire 1
# Only active expire will delete the key
wait_for_condition 30 10 { [r exists myhash2] == 0 } else { fail "`myhash2` should be expired" }
}
test "MOVE to another DB hash with fields to be expired ($type)" {
r select 9
r flushall
r hset myhash field1 value1
r hpexpire myhash 100 NX FIELDS 1 field1
r move myhash 10
assert_equal [r exists myhash] 0
assert_equal [r dbsize] 0
# Verify the key and its field exists in the target DB
r select 10
assert_equal [r hget myhash field1] "value1"
assert_equal [r exists myhash] 1
# Eventually the field will be expired and the key will be deleted
wait_for_condition 40 10 { [r hget myhash field1] == "" } else { fail "`field1` should be expired" }
wait_for_condition 40 10 { [r exists myhash] == 0 } else { fail "db should be empty" }
} {} {singledb:skip}
test "Test COPY hash with fields to be expired ($type)" {
r flushall
r hset h1 f1 v1 f2 v2
r hset h2 f1 v1 f2 v2 f3 v3 f4 v4 f5 v5 f6 v6 f7 v7 f8 v8 f9 v9 f10 v10 f11 v11 f12 v12 f13 v13 f14 v14 f15 v15 f16 v16 f17 v17 f18 v18
r hpexpire h1 100 NX FIELDS 1 f1
r hpexpire h2 100 NX FIELDS 18 f1 f2 f3 f4 f5 f6 f7 f8 f9 f10 f11 f12 f13 f14 f15 f16 f17 f18
r COPY h1 h1copy
r COPY h2 h2copy
assert_equal [r hget h1 f1] "v1"
assert_equal [r hget h1copy f1] "v1"
assert_equal [r exists h2] 1
assert_equal [r exists h2copy] 1
after 105
# Verify lazy expire of field in h1 and its copy
assert_equal [r hget h1 f1] ""
assert_equal [r hget h1copy f1] ""
# Verify lazy expire of field in h2 and its copy. Verify the key deleted as well.
wait_for_condition 40 10 { [r exists h2] == 0 } else { fail "`h2` should be expired" }
wait_for_condition 40 10 { [r exists h2copy] == 0 } else { fail "`h2copy` should be expired" }
} {} {singledb:skip}
test "Test SWAPDB hash-fields to be expired ($type)" {
r select 9
r flushall
r hset myhash field1 value1
r hpexpire myhash 50 NX FIELDS 1 field1
r swapdb 9 10
# Verify the key and its field doesn't exist in the source DB
assert_equal [r exists myhash] 0
assert_equal [r dbsize] 0
# Verify the key and its field exists in the target DB
r select 10
assert_equal [r hget myhash field1] "value1"
assert_equal [r dbsize] 1
# Eventually the field will be expired and the key will be deleted
wait_for_condition 20 10 { [r exists myhash] == 0 } else { fail "'myhash' should be expired" }
} {} {singledb:skip}
test "HMGET - returns empty entries if fields or hash expired ($type)" {
r debug set-active-expire 0
r del h1 h2
r hset h1 f1 v1 f2 v2 f3 v3
r hset h2 f1 v1 f2 v2 f3 v3
r hpexpire h1 10000000 NX FIELDS 1 f1
r hpexpire h1 1 NX FIELDS 2 f2 f3
r hpexpire h2 1 NX FIELDS 3 f1 f2 f3
after 5
assert_equal [r hmget h1 f1 f2 f3] {v1 {} {}}
assert_equal [r hmget h2 f1 f2 f3] {{} {} {}}
r debug set-active-expire 1
}
test "HPERSIST - Returns empty array if key does not exist ($type)" {
r del myhash
# Make sure we can distinguish between an empty array and a null response
r readraw 1
assert_equal {*0} [r HPERSIST myhash FIELDS 1 a]
r readraw 0
}
test "HPERSIST - input validation ($type)" {
# HPERSIST key <num-fields> <field [field ...]>
r del myhash
r hset myhash f1 v1 f2 v2
r hexpire myhash 1000 NX FIELDS 1 f1
assert_error {*wrong number of arguments*} {r hpersist myhash}
assert_error {*wrong number of arguments*} {r hpersist myhash FIELDS 1}
assert_equal [r hpersist not-exists-key FIELDS 1 f1] {}
assert_equal [r hpersist myhash FIELDS 2 f1 not-exists-field] "$P_OK $P_NO_FIELD"
assert_equal [r hpersist myhash FIELDS 1 f2] "$P_NO_EXPIRY"
}
test "HPERSIST - verify fields with TTL are persisted ($type)" {
r del myhash
r hset myhash f1 v1 f2 v2
r hexpire myhash 20 NX FIELDS 2 f1 f2
r hpersist myhash FIELDS 2 f1 f2
after 25
assert_equal [r hget myhash f1] "v1"
assert_equal [r hget myhash f2] "v2"
assert_equal [r HTTL myhash FIELDS 2 f1 f2] "$T_NO_EXPIRY $T_NO_EXPIRY"
}
test "HTTL/HPERSIST - Test expiry commands with non-volatile hash ($type)" {
r del myhash
r hset myhash field1 value1 field2 value2 field3 value3
assert_equal [r httl myhash FIELDS 1 field1] $T_NO_EXPIRY
assert_equal [r httl myhash FIELDS 1 fieldnonexist] $E_NO_FIELD
assert_equal [r hpersist myhash FIELDS 1 field1] $P_NO_EXPIRY
assert_equal [r hpersist myhash FIELDS 1 fieldnonexist] $P_NO_FIELD
}
test {DUMP / RESTORE are able to serialize / unserialize a hash} {
r config set sanitize-dump-payload yes
r del myhash
r hmset myhash a 1 b 2 c 3
r hexpireat myhash 2524600800 fields 1 a
r hexpireat myhash 2524600801 fields 1 b
set encoded [r dump myhash]
r del myhash
r restore myhash 0 $encoded
assert_equal [lsort [r hgetall myhash]] "1 2 3 a b c"
assert_equal [r hexpiretime myhash FIELDS 3 a b c] {2524600800 2524600801 -1}
}
test {DUMP / RESTORE are able to serialize / unserialize a hash with TTL 0 for all fields} {
r config set sanitize-dump-payload yes
r del myhash
r hmset myhash a 1 b 2 c 3
r hexpire myhash 9999999 fields 1 a ;# make all TTLs of fields to 0
r hpersist myhash fields 1 a
assert_encoding $type myhash
set encoded [r dump myhash]
r del myhash
r restore myhash 0 $encoded
assert_equal [lsort [r hgetall myhash]] "1 2 3 a b c"
assert_equal [r hexpiretime myhash FIELDS 3 a b c] {-1 -1 -1}
}
test {HINCRBY - discards pending expired field and reset its value} {
r debug set-active-expire 0
r del h1 h2
r hset h1 f1 10 f2 2
r hset h2 f1 10
assert_equal [r HINCRBY h1 f1 2] 12
assert_equal [r HINCRBY h2 f1 2] 12
r HPEXPIRE h1 10 FIELDS 1 f1
r HPEXPIRE h2 10 FIELDS 1 f1
after 15
assert_equal [r HINCRBY h1 f1 1] 1
assert_equal [r HINCRBY h2 f1 1] 1
r debug set-active-expire 1
}
test {HINCRBY - preserve expiration time of the field} {
r del h1
r hset h1 f1 10
r hpexpire h1 20 FIELDS 1 f1
assert_equal [r HINCRBY h1 f1 2] 12
assert_range [r HPTTL h1 FIELDS 1 f1] 1 20
}
test {HINCRBYFLOAT - discards pending expired field and reset its value} {
r debug set-active-expire 0
r del h1 h2
r hset h1 f1 10 f2 2
r hset h2 f1 10
assert_equal [r HINCRBYFLOAT h1 f1 2] 12
assert_equal [r HINCRBYFLOAT h2 f1 2] 12
r HPEXPIRE h1 10 FIELDS 1 f1
r HPEXPIRE h2 10 FIELDS 1 f1
after 15
assert_equal [r HINCRBYFLOAT h1 f1 1] 1
assert_equal [r HINCRBYFLOAT h2 f1 1] 1
r debug set-active-expire 1
}
test {HINCRBYFLOAT - preserve expiration time of the field} {
r del h1
r hset h1 f1 10
r hpexpire h1 20 FIELDS 1 f1
assert_equal [r HINCRBYFLOAT h1 f1 2.5] 12.5
assert_range [r HPTTL h1 FIELDS 1 f1] 1 20
}
}
r config set hash-max-listpack-entries 512
}
start_server {tags {"external:skip needs:debug"}} {
# Tests that only applies to listpack
test "Test listpack memory usage" {
r hset myhash f1 v1 f2 v2 f3 v3 f4 v4 f5 v5
r hpexpire myhash 5 FIELDS 2 f2 f4
# Just to have code coverage for the new listpack encoding
r memory usage myhash
}
test "Test listpack object encoding" {
r hset myhash f1 v1 f2 v2 f3 v3 f4 v4 f5 v5
r hpexpire myhash 5 FIELDS 2 f2 f4
# Just to have code coverage for the listpackex encoding
assert_equal [r object encoding myhash] "listpackex"
}
test "Test listpack debug listpack" {
r hset myhash f1 v1 f2 v2 f3 v3 f4 v4 f5 v5
# Just to have code coverage for the listpackex encoding
r debug listpack myhash
}
test "Test listpack converts to ht and passive expiry works" {
set prev [lindex [r config get hash-max-listpack-entries] 1]
r config set hash-max-listpack-entries 10
r debug set-active-expire 0
r hset myhash f1 v1 f2 v2 f3 v3 f4 v4 f5 v5
r hpexpire myhash 5 FIELDS 2 f2 f4
for {set i 6} {$i < 11} {incr i} {
r hset myhash f$i v$i
}
after 50
assert_equal [lsort [r hgetall myhash]] [lsort "f1 f3 f5 f6 f7 f8 f9 f10 v1 v3 v5 v6 v7 v8 v9 v10"]
r config set hash-max-listpack-entries $prev
r debug set-active-expire 1
}
test "Test listpack converts to ht and active expiry works" {
r del myhash
r debug set-active-expire 0
r hset myhash f1 v1 f2 v2 f3 v3 f4 v4 f5 v5
r hpexpire myhash 10 FIELDS 1 f1
for {set i 0} {$i < 2048} {incr i} {
r hset myhash f$i v$i
}
for {set i 0} {$i < 2048} {incr i} {
r hpexpire myhash 10 FIELDS 1 f$i
}
r debug set-active-expire 1
wait_for_condition 50 20 { [r EXISTS myhash] == 0 } else { fail "'myhash' should be expired" }
}
test "Test listpack converts to ht and active expiry works" {
r del myhash
r debug set-active-expire 0
# Check expiry works after listpack converts to ht
for {set i 0} {$i < 1024} {incr i} {
r hset myhash f1_$i v1_$i f2_$i v2_$i f3_$i v3_$i f4_$i v4_$i
r hpexpire myhash 10 FIELDS 4 f1_$i f2_$i f3_$i f4_$i
}
assert_encoding hashtable myhash
assert_equal [r hlen myhash] 4096
r debug set-active-expire 1
wait_for_condition 50 20 { [r EXISTS myhash] == 0 } else { fail "'myhash' should be expired" }
}
test "HPERSIST/HEXPIRE - Test listpack with large values" {
r del myhash
# Test with larger values to verify we successfully move fields in
# listpack when we are ordering according to TTL. This config change
# will make code to use temporary heap allocation when moving fields.
# See listpackExUpdateExpiry() for details.
r config set hash-max-listpack-value 2048
set payload1 [string repeat v3 1024]
set payload2 [string repeat v1 1024]
# Test with single item list
r hset myhash f1 $payload1
r hexpire myhash 2000 FIELDS 1 f1
assert_equal [r hget myhash f1] $payload1
r del myhash
# Test with multiple items
r hset myhash f1 $payload2 f2 v2 f3 $payload1 f4 v4
r hexpire myhash 100000 FIELDS 1 f3
r hpersist myhash FIELDS 1 f3
assert_equal [r hpersist myhash FIELDS 1 f3] $P_NO_EXPIRY
r hpexpire myhash 10 FIELDS 1 f1
after 20
assert_equal [lsort [r hgetall myhash]] [lsort "f2 f3 f4 v2 $payload1 v4"]
r config set hash-max-listpack-value 64
}
test "Statistics - Hashes with HFEs" {
r config resetstat
r del myhash
r hset myhash f1 v1 f2 v2 f3 v3 f4 v4 f5 v5
r hpexpire myhash 100 FIELDS 3 f1 f2 f3
assert_match [get_hashes_with_expiry_fields r] 1
r hset myhash2 f1 v1 f2 v2 f3 v3 f4 v4 f5 v5
assert_match [get_hashes_with_expiry_fields r] 1
r hpexpire myhash2 100 FIELDS 3 f1 f2 f3
assert_match [get_hashes_with_expiry_fields r] 2
wait_for_condition 50 50 {
[get_hashes_with_expiry_fields r] == 0
} else {
fail "Hash field expiry statistics failed"
}
}
}
start_server {tags {"external:skip needs:debug"}} {
foreach type {listpack ht} {
if {$type eq "ht"} {
r config set hash-max-listpack-entries 0
} else {
r config set hash-max-listpack-entries 512
}
test "Command rewrite and expired hash fields are propagated to replica ($type)" {
start_server {overrides {appendonly {yes} appendfsync always} tags {external:skip}} {
set aof [get_last_incr_aof_path r]
r hset h1 f1 v1 f2 v2
r hpexpire h1 20 FIELDS 1 f1
r hpexpire h1 30 FIELDS 1 f2
r hpexpire h1 30 FIELDS 1 non_exists_field
r hset h2 f1 v1 f2 v2 f3 v3 f4 v4
r hpexpire h2 40 FIELDS 2 f1 non_exists_field
r hpexpire h2 50 FIELDS 1 f2
r hpexpireat h2 [expr [clock seconds]*1000+100000] LT FIELDS 1 f3
r hexpireat h2 [expr [clock seconds]+10] NX FIELDS 1 f4
wait_for_condition 50 100 {
[r hlen h2] eq 2
} else {
fail "Field f2 of hash h2 wasn't deleted"
}
# Assert that each TTL-related command are persisted with absolute timestamps in AOF
assert_aof_content $aof {
{select *}
{hset h1 f1 v1 f2 v2}
{hpexpireat h1 * FIELDS 1 f1}
{hpexpireat h1 * FIELDS 1 f2}
{hset h2 f1 v1 f2 v2 f3 v3 f4 v4}
{hpexpireat h2 * FIELDS 2 f1 non_exists_field}
{hpexpireat h2 * FIELDS 1 f2}
{hpexpireat h2 * FIELDS 1 f3}
{hpexpireat h2 * FIELDS 1 f4}
{hdel h1 f1}
{hdel h1 f2}
{hdel h2 f1}
{hdel h2 f2}
}
}
}
test "Lazy Expire - fields are lazy deleted and propagated to replicas ($type)" {
start_server {overrides {appendonly {yes} appendfsync always} tags {external:skip}} {
r debug set-active-expire 0
set aof [get_last_incr_aof_path r]
r del myhash
r hset myhash f1 v1 f2 v2 f3 v3
r hpexpire myhash 1 NX FIELDS 3 f1 f2 f3
after 5
# Verify that still exists even if all fields are expired
assert_equal 1 [r EXISTS myhash]
# Verify that len counts also expired fields
assert_equal 3 [r HLEN myhash]
# Trying access to expired field should delete it. Len should be updated
assert_equal 0 [r hexists myhash f1]
assert_equal 2 [r HLEN myhash]
# Trying access another expired field should delete it. Len should be updated
assert_equal "" [r hget myhash f2]
assert_equal 1 [r HLEN myhash]
# Trying access last expired field should delete it. hash shouldn't exists afterward.
assert_equal 0 [r hstrlen myhash f3]
assert_equal 0 [r HLEN myhash]
assert_equal 0 [r EXISTS myhash]
wait_for_condition 50 100 { [r exists h1] == 0 } else { fail "hash h1 wasn't deleted" }
# HDEL are propagated as expected
assert_aof_content $aof {
{select *}
{hset myhash f1 v1 f2 v2 f3 v3}
{hpexpireat myhash * NX FIELDS 3 f1 f2 f3}
{hdel myhash f1}
{hdel myhash f2}
{hdel myhash f3}
}
r debug set-active-expire 1
}
}
# Start a new server with empty data and AOF file.
start_server {overrides {appendonly {yes} appendfsync always} tags {external:skip}} {
# Based on test at expire.tcl: " All time-to-live(TTL) in commands are propagated as absolute ..."
test {All TTLs in commands are propagated as absolute timestamp in milliseconds in AOF} {
set aof [get_last_incr_aof_path r]
r hset h1 f1 v1 f2 v2 f3 v3 f4 v4 f5 v5 f6 v6
r hexpireat h1 [expr [clock seconds]+100] NX FIELDS 1 f1
r hpexpireat h1 [expr [clock seconds]*1000+100000] NX FIELDS 1 f2
r hpexpire h1 100000 NX FIELDS 3 f3 f4 f5
r hexpire h1 100000 FIELDS 1 f6
r hset h5 f1 v1
assert_aof_content $aof {
{select *}
{hset h1 f1 v1 f2 v2 f3 v3 f4 v4 f5 v5 f6 v6}
{hpexpireat h1 * FIELDS 1 f1}
{hpexpireat h1 * FIELDS 1 f2}
{hpexpireat h1 * NX FIELDS 3 f3 f4 f5}
{hpexpireat h1 * FIELDS 1 f6}
{hset h5 f1 v1}
}
array set keyAndFields1 [dumpAllHashes r]
# Let some time pass and reload data from AOF
after 2000
r debug loadaof
array set keyAndFields2 [dumpAllHashes r]
# Assert that absolute TTLs are the same
assert_equal [array get keyAndFields1] [array get keyAndFields2]
} {} {needs:debug}
}
# Based on test, with same name, at expire.tcl:
test {All TTL in commands are propagated as absolute timestamp in replication stream} {
# Make sure that both relative and absolute expire commands are propagated
# Consider also comment of the test, with same name, at expire.tcl
r flushall ; # Clean up keyspace to avoid interference by keys from other tests
set repl [attach_to_replication_stream]
r hset h1 f1 v1
r hexpireat h1 [expr [clock seconds]+100] NX FIELDS 1 f1
r hset h2 f2 v2
r hpexpireat h2 [expr [clock seconds]*1000+100000] NX FIELDS 1 f2
r hset h3 f3 v3 f4 v4
r hexpire h3 100 FIELDS 3 f3 f4 non_exists_field
assert_replication_stream $repl {
{select *}
{hset h1 f1 v1}
{hpexpireat h1 * NX FIELDS 1 f1}
{hset h2 f2 v2}
{hpexpireat h2 * NX FIELDS 1 f2}
{hset h3 f3 v3 f4 v4}
{hpexpireat h3 * FIELDS 3 f3 f4 non_exists_field}
}
close_replication_stream $repl
} {} {needs:repl}
# Start another server to test replication of TTLs
start_server {tags {needs:repl external:skip}} {
# Set the outer layer server as primary
set primary [srv -1 client]
set primary_host [srv -1 host]
set primary_port [srv -1 port]
# Set this inner layer server as replica
set replica [srv 0 client]
# Server should have role slave
$replica replicaof $primary_host $primary_port
wait_for_condition 50 100 {
[s 0 role] eq {slave}
} else {
fail "Replication not started."
}
# Based on test, with same name, at expire.tcl
test {For all replicated TTL-related commands, absolute expire times are identical on primary and replica} {
# Apply each TTL-related command to a unique key on primary
$primary flushall
$primary hset h1 f v
$primary hexpireat h1 [expr [clock seconds]+10000] FIELDS 1 f
$primary hset h2 f v
$primary hpexpireat h2 [expr [clock milliseconds]+100000] FIELDS 1 f
$primary hset h3 f v
$primary hexpire h3 100 NX FIELDS 1 f
$primary hset h4 f v
$primary hpexpire h4 100000 NX FIELDS 1 f
$primary hset h5 f v
$primary hpexpireat h5 [expr [clock milliseconds]-100000] FIELDS 1 f
$primary hset h9 f v
# Wait for replica to get the keys and TTLs
assert {[$primary wait 1 0] == 1}
# Verify absolute TTLs are identical on primary and replica for all keys
# This is because TTLs are always replicated as absolute values
assert_equal [dumpAllHashes $primary] [dumpAllHashes $replica]
}
}
}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment