Unverified Commit e26ea35c authored by Moti Cohen's avatar Moti Cohen Committed by GitHub
Browse files

Adapt HRANDFIELD to HFE feature (#13348)

Considerations for the selected imp of HRANDFIELD & HFE feature:

HRANDFIELD might access any of the fields in the hash as some of them
might be expired. And so the Implementation of HRANDFIELD along with HFEs
might be one of the two options:
1. Expire hash-fields before diving into handling HRANDFIELD.
2. Refine HRANDFIELD cases to deal with expired fields.

Regarding the first option, as reference, the command RANDOMKEY also
declareson O(1) complexity, yet might be stuck on a very long (but not infinite)
loop trying to find non-expired keys. Furthermore RANDOMKEY also evicts expired 
keys along the way even though it is categorized as a read-only command. Note
that the case of HRANDFIELD is more lightweight versus RANDOMKEY since 
HFEs have much more effective and aggressive active-expiration for fields behind.

The second option introduces additional implementation complexity to HRANDFIELD.
We could further refine HRANDFIELD cases to differentiate between scenarios
with many expired fields versus few expired fields, and adjust based on the
percentage of expired fields. However, this approach could still lead to long
loops or necessitate expiring fields before selecting them. For the “lightweight”
cases it is also expected to have a lightweight expiration.

Considering the pros and cons, and the fact that HRANDFIELD is an infrequent
command (particularly with HFEs) and the fact we have effective active-expiration
behind for hash-fields, it is better to keep it simple and choose option number 1.

Other changes:
* Don't mark command dirty by internal hashTypeExpire(). It causes to read 
  only command of HRANDFIELD to be accidently propagated (This flag
  should be indicated at higher level, by the command functions).
* Align `hashTypeExpireIfNeeded()` and `hashTypeGetValue()` to be more
  aligned with `expireIfNeeded()` logic of keyspace.
parent 811c5d7a
...@@ -1968,7 +1968,7 @@ int rewriteHashObject(rio *r, robj *key, robj *o) { ...@@ -1968,7 +1968,7 @@ int rewriteHashObject(rio *r, robj *key, robj *o) {
hashTypeIterator *hi; hashTypeIterator *hi;
long long count = 0, items = hashTypeLength(o, 0); long long count = 0, items = hashTypeLength(o, 0);
int isHFE = hashTypeGetMinExpire(o) != EB_EXPIRE_TIME_INVALID; int isHFE = hashTypeGetMinExpire(o, 0) != EB_EXPIRE_TIME_INVALID;
hi = hashTypeInitIterator(o); hi = hashTypeInitIterator(o);
if (!isHFE) { if (!isHFE) {
......
...@@ -270,7 +270,7 @@ void restoreCommand(client *c) { ...@@ -270,7 +270,7 @@ void restoreCommand(client *c) {
/* If minExpiredField was set, then the object is hash with expiration /* If minExpiredField was set, then the object is hash with expiration
* on fields and need to register it in global HFE DS */ * on fields and need to register it in global HFE DS */
if (obj->type == OBJ_HASH) { if (obj->type == OBJ_HASH) {
uint64_t minExpiredField = hashTypeGetNextTimeToExpire(obj); uint64_t minExpiredField = hashTypeGetMinExpire(obj, 1);
if (minExpiredField != EB_EXPIRE_TIME_INVALID) if (minExpiredField != EB_EXPIRE_TIME_INVALID)
hashTypeAddToExpires(c->db, dictGetKey(de), obj, minExpiredField); hashTypeAddToExpires(c->db, dictGetKey(de), obj, minExpiredField);
} }
......
...@@ -751,7 +751,7 @@ void defragKey(defragCtx *ctx, dictEntry *de) { ...@@ -751,7 +751,7 @@ void defragKey(defragCtx *ctx, dictEntry *de) {
} }
/* Try to defrag robj and / or string value. */ /* Try to defrag robj and / or string value. */
if (unlikely(ob->type == OBJ_HASH && hashTypeGetMinExpire(ob) != EB_EXPIRE_TIME_INVALID)) { if (unlikely(ob->type == OBJ_HASH && hashTypeGetMinExpire(ob, 0) != EB_EXPIRE_TIME_INVALID)) {
/* Update its reference in the ebucket while defragging it. */ /* Update its reference in the ebucket while defragging it. */
newob = ebDefragItem(&db->hexpires, &hashExpireBucketsType, ob, newob = ebDefragItem(&db->hexpires, &hashExpireBucketsType, ob,
(ebDefragFunction *)activeDefragStringOb); (ebDefragFunction *)activeDefragStringOb);
......
...@@ -699,7 +699,7 @@ int rdbSaveObjectType(rio *rdb, robj *o) { ...@@ -699,7 +699,7 @@ int rdbSaveObjectType(rio *rdb, robj *o) {
else if (o->encoding == OBJ_ENCODING_LISTPACK_EX) else if (o->encoding == OBJ_ENCODING_LISTPACK_EX)
return rdbSaveType(rdb,RDB_TYPE_HASH_LISTPACK_EX); return rdbSaveType(rdb,RDB_TYPE_HASH_LISTPACK_EX);
else if (o->encoding == OBJ_ENCODING_HT) { else if (o->encoding == OBJ_ENCODING_HT) {
if (hashTypeGetMinExpire(o) == EB_EXPIRE_TIME_INVALID) if (hashTypeGetMinExpire(o, 0) == EB_EXPIRE_TIME_INVALID)
return rdbSaveType(rdb,RDB_TYPE_HASH); return rdbSaveType(rdb,RDB_TYPE_HASH);
else else
return rdbSaveType(rdb,RDB_TYPE_HASH_METADATA); return rdbSaveType(rdb,RDB_TYPE_HASH_METADATA);
...@@ -960,7 +960,7 @@ ssize_t rdbSaveObject(rio *rdb, robj *o, robj *key, int dbid) { ...@@ -960,7 +960,7 @@ ssize_t rdbSaveObject(rio *rdb, robj *o, robj *key, int dbid) {
* RDB_TYPE_HASH_METADATA layout, including tuples of [ttl][field][value]. * RDB_TYPE_HASH_METADATA layout, including tuples of [ttl][field][value].
* Otherwise, use the standard RDB_TYPE_HASH layout containing only * Otherwise, use the standard RDB_TYPE_HASH layout containing only
* the tuples [field][value]. */ * the tuples [field][value]. */
int with_ttl = (hashTypeGetMinExpire(o) != EB_EXPIRE_TIME_INVALID); int with_ttl = (hashTypeGetMinExpire(o, 0) != EB_EXPIRE_TIME_INVALID);
/* save number of fields in hash */ /* save number of fields in hash */
if ((n = rdbSaveLen(rdb,dictSize((dict*)o->ptr))) == -1) { if ((n = rdbSaveLen(rdb,dictSize((dict*)o->ptr))) == -1) {
...@@ -3562,7 +3562,7 @@ int rdbLoadRioWithLoadingCtx(rio *rdb, int rdbflags, rdbSaveInfo *rsi, rdbLoadin ...@@ -3562,7 +3562,7 @@ int rdbLoadRioWithLoadingCtx(rio *rdb, int rdbflags, rdbSaveInfo *rsi, rdbLoadin
/* If minExpiredField was set, then the object is hash with expiration /* If minExpiredField was set, then the object is hash with expiration
* on fields and need to register it in global HFE DS */ * on fields and need to register it in global HFE DS */
if (val->type == OBJ_HASH) { if (val->type == OBJ_HASH) {
uint64_t minExpiredField = hashTypeGetNextTimeToExpire(val); uint64_t minExpiredField = hashTypeGetMinExpire(val, 1);
if (minExpiredField != EB_EXPIRE_TIME_INVALID) if (minExpiredField != EB_EXPIRE_TIME_INVALID)
hashTypeAddToExpires(db, key, val, minExpiredField); hashTypeAddToExpires(db, key, val, minExpiredField);
} }
......
...@@ -334,6 +334,10 @@ uint64_t dictObjHash(const void *key) { ...@@ -334,6 +334,10 @@ uint64_t dictObjHash(const void *key) {
return dictGenHashFunction(o->ptr, sdslen((sds)o->ptr)); return dictGenHashFunction(o->ptr, sdslen((sds)o->ptr));
} }
uint64_t dictPtrHash(const void *key) {
return dictGenHashFunction((unsigned char*)&key,sizeof(key));
}
uint64_t dictSdsHash(const void *key) { uint64_t dictSdsHash(const void *key) {
return dictGenHashFunction((unsigned char*)key, sdslen((char*)key)); return dictGenHashFunction((unsigned char*)key, sdslen((char*)key));
} }
......
...@@ -3164,7 +3164,9 @@ robj *setTypeDup(robj *o); ...@@ -3164,7 +3164,9 @@ robj *setTypeDup(robj *o);
typedef struct listpackEx { typedef struct listpackEx {
ExpireMeta meta; /* To be used in order to register the hash in the ExpireMeta meta; /* To be used in order to register the hash in the
global ebuckets (i.e. db->hexpires) with next, global ebuckets (i.e. db->hexpires) with next,
minimum, hash-field to expire. */ minimum, hash-field to expire. TTL value might be
inaccurate up-to few seconds due to optimization
consideration. */
sds key; /* reference to the key, same one that stored in sds key; /* reference to the key, same one that stored in
db->dict. Will be used from active-expiration flow db->dict. Will be used from active-expiration flow
for notification and deletion of the object, if for notification and deletion of the object, if
...@@ -3179,7 +3181,9 @@ typedef struct dictExpireMetadata { ...@@ -3179,7 +3181,9 @@ typedef struct dictExpireMetadata {
ExpireMeta expireMeta; /* embedded ExpireMeta in dict. ExpireMeta expireMeta; /* embedded ExpireMeta in dict.
To be used in order to register the hash in the To be used in order to register the hash in the
global ebuckets (i.e db->hexpires) with next, global ebuckets (i.e db->hexpires) with next,
minimum, hash-field to expire */ minimum, hash-field to expire. TTL value might be
inaccurate up-to few seconds due to optimization
consideration. */
ebuckets hfe; /* DS of Hash Fields Expiration, associated to each hash */ ebuckets hfe; /* DS of Hash Fields Expiration, associated to each hash */
sds key; /* reference to the key, same one that stored in sds key; /* reference to the key, same one that stored in
db->dict. Will be used from active-expiration flow db->dict. Will be used from active-expiration flow
...@@ -3225,13 +3229,10 @@ uint64_t hashTypeRemoveFromExpires(ebuckets *hexpires, robj *o); ...@@ -3225,13 +3229,10 @@ uint64_t hashTypeRemoveFromExpires(ebuckets *hexpires, robj *o);
void hashTypeAddToExpires(redisDb *db, sds key, robj *hashObj, uint64_t expireTime); void hashTypeAddToExpires(redisDb *db, sds key, robj *hashObj, uint64_t expireTime);
void hashTypeFree(robj *o); void hashTypeFree(robj *o);
int hashTypeIsExpired(const robj *o, uint64_t expireAt); int hashTypeIsExpired(const robj *o, uint64_t expireAt);
uint64_t hashTypeGetMinExpire(robj *o);
unsigned char *hashTypeListpackGetLp(robj *o); unsigned char *hashTypeListpackGetLp(robj *o);
uint64_t hashTypeGetMinExpire(robj *o); uint64_t hashTypeGetMinExpire(robj *o, int accurate);
void hashTypeUpdateKeyRef(robj *o, sds newkey); void hashTypeUpdateKeyRef(robj *o, sds newkey);
ebuckets *hashTypeGetDictMetaHFE(dict *d); ebuckets *hashTypeGetDictMetaHFE(dict *d);
uint64_t hashTypeGetMinExpire(robj *keyObj);
uint64_t hashTypeGetNextTimeToExpire(robj *o);
void initDictExpireMetadata(sds key, robj *o); void initDictExpireMetadata(sds key, robj *o);
struct listpackEx *listpackExCreate(void); struct listpackEx *listpackExCreate(void);
void listpackExAddNew(robj *o, char *field, size_t flen, void listpackExAddNew(robj *o, char *field, size_t flen,
...@@ -3539,6 +3540,7 @@ void startEvictionTimeProc(void); ...@@ -3539,6 +3540,7 @@ void startEvictionTimeProc(void);
/* Keys hashing / comparison functions for dict.c hash tables. */ /* Keys hashing / comparison functions for dict.c hash tables. */
uint64_t dictSdsHash(const void *key); uint64_t dictSdsHash(const void *key);
uint64_t dictPtrHash(const void *key);
uint64_t dictSdsCaseHash(const void *key); uint64_t dictSdsCaseHash(const void *key);
int dictSdsKeyCompare(dict *d, const void *key1, const void *key2); int dictSdsKeyCompare(dict *d, const void *key1, const void *key2);
int dictSdsMstrKeyCompare(dict *d, const void *sdsLookup, const void *mstrStored); int dictSdsMstrKeyCompare(dict *d, const void *sdsLookup, const void *mstrStored);
......
This diff is collapsed.
...@@ -32,13 +32,6 @@ proc get_hashes_with_expiry_fields {r} { ...@@ -32,13 +32,6 @@ proc get_hashes_with_expiry_fields {r} {
return 0 return 0
} }
proc create_hash {key entries} {
r del $key
foreach entry $entries {
r hset $key [lindex $entry 0] [lindex $entry 1]
}
}
proc get_keys {l} { proc get_keys {l} {
set res {} set res {}
foreach entry $l { foreach entry $l {
...@@ -48,22 +41,6 @@ proc get_keys {l} { ...@@ -48,22 +41,6 @@ proc get_keys {l} {
return $res return $res
} }
proc cmp_hrandfield_result {hash_name expected_result} {
# Accumulate hrandfield results
unset -nocomplain myhash
array set myhash {}
for {set i 0} {$i < 100} {incr i} {
set key [r hrandfield $hash_name]
set myhash($key) 1
}
set res [lsort [array names myhash]]
if {$res eq $expected_result} {
return 1
} else {
return $res
}
}
proc dumpAllHashes {client} { proc dumpAllHashes {client} {
set keyAndFields(0,0) 0 set keyAndFields(0,0) 0
unset keyAndFields unset keyAndFields
...@@ -77,36 +54,6 @@ proc dumpAllHashes {client} { ...@@ -77,36 +54,6 @@ proc dumpAllHashes {client} {
return [array get keyAndFields] return [array get keyAndFields]
} }
proc hrandfieldTest {activeExpireConfig} {
r debug set-active-expire $activeExpireConfig
r del myhash
set contents {{field1 1} {field2 2} }
create_hash myhash $contents
set factorValgrind [expr {$::valgrind ? 2 : 1}]
# Set expiration time for field1 and field2 such that field1 expires first
r hpexpire myhash 1 NX FIELDS 1 field1
r hpexpire myhash 100 NX FIELDS 1 field2
# On call hrandfield command lazy expire deletes field1 first
wait_for_condition 8 10 {
[cmp_hrandfield_result myhash "field2"] == 1
} else {
fail "Expected field2 to be returned by HRANDFIELD."
}
# On call hrandfield command lazy expire deletes field2 as well
wait_for_condition 8 20 {
[cmp_hrandfield_result myhash "{}"] == 1
} else {
fail "Expected {} to be returned by HRANDFIELD."
}
# restore the default value
r debug set-active-expire 1
}
############################### TESTS ######################################### ############################### TESTS #########################################
start_server {tags {"external:skip needs:debug"}} { start_server {tags {"external:skip needs:debug"}} {
...@@ -396,22 +343,33 @@ start_server {tags {"external:skip needs:debug"}} { ...@@ -396,22 +343,33 @@ start_server {tags {"external:skip needs:debug"}} {
r debug set-active-expire 1 r debug set-active-expire 1
} }
# OPEN: To decide if to delete expired fields at start of HRANDFIELD. test "Test HRANDFIELD deletes all expired fields ($type)" {
# test "Test HRANDFIELD does not return expired fields ($type)" {
# hrandfieldTest 0
# hrandfieldTest 1
# }
test "Test HRANDFIELD can return expired fields ($type)" {
r debug set-active-expire 0 r debug set-active-expire 0
r del myhash r flushall
r hset myhash f1 v1 f2 v2 f3 v3 f4 v4 f5 v5 r hset myhash f1 v1 f2 v2 f3 v3 f4 v4 f5 v5
r hpexpire myhash 1 NX FIELDS 4 f1 f2 f3 f4 r hpexpire myhash 1 FIELDS 2 f1 f2
after 5 after 5
set res [cmp_hrandfield_result myhash "f1 f2 f3 f4 f5"] assert_equal [lsort [r hrandfield myhash 5]] "f3 f4 f5"
assert {$res == 1} r hpexpire myhash 1 FIELDS 3 f3 f4 f5
r debug set-active-expire 1 after 5
assert_equal [lsort [r hrandfield myhash 5]] ""
assert_equal [r keys *] ""
r del myhash
r hset myhash f1 v1 f2 v2 f3 v3
r hpexpire myhash 1 FIELDS 1 f1
after 5
set res [r hrandfield myhash]
assert {$res == "f2" || $res == "f3"}
r hpexpire myhash 1 FIELDS 1 f2
after 5
assert_equal [lsort [r hrandfield myhash 5]] "f3"
r hpexpire myhash 1 FIELDS 1 f3
after 5
assert_equal [r hrandfield myhash] ""
assert_equal [r keys *] ""
r debug set-active-expire 1
} }
test "Lazy Expire - HLEN does count expired fields ($type)" { test "Lazy Expire - HLEN does count expired fields ($type)" {
...@@ -1054,7 +1012,13 @@ start_server {tags {"external:skip needs:debug"}} { ...@@ -1054,7 +1012,13 @@ start_server {tags {"external:skip needs:debug"}} {
r hpexpireat h1 [expr [clock seconds]*1000+100000] NX FIELDS 1 f2 r hpexpireat h1 [expr [clock seconds]*1000+100000] NX FIELDS 1 f2
r hpexpire h1 100000 NX FIELDS 3 f3 f4 f5 r hpexpire h1 100000 NX FIELDS 3 f3 f4 f5
r hexpire h1 100000 FIELDS 1 f6 r hexpire h1 100000 FIELDS 1 f6
r hset h5 f1 v1
# Verify HRANDFIELD deletes expired fields and propagates it
r hset h2 f1 v1 f2 v2
r hpexpire h2 1 FIELDS 1 f1
r hpexpire h2 50 FIELDS 1 f2
assert_equal [r hrandfield h4 2] ""
after 200
assert_aof_content $aof { assert_aof_content $aof {
{select *} {select *}
...@@ -1063,7 +1027,11 @@ start_server {tags {"external:skip needs:debug"}} { ...@@ -1063,7 +1027,11 @@ start_server {tags {"external:skip needs:debug"}} {
{hpexpireat h1 * FIELDS 1 f2} {hpexpireat h1 * FIELDS 1 f2}
{hpexpireat h1 * NX FIELDS 3 f3 f4 f5} {hpexpireat h1 * NX FIELDS 3 f3 f4 f5}
{hpexpireat h1 * FIELDS 1 f6} {hpexpireat h1 * FIELDS 1 f6}
{hset h5 f1 v1} {hset h2 f1 v1 f2 v2}
{hpexpireat h2 * FIELDS 1 f1}
{hpexpireat h2 * FIELDS 1 f2}
{hdel h2 f1}
{hdel h2 f2}
} }
array set keyAndFields1 [dumpAllHashes r] array set keyAndFields1 [dumpAllHashes r]
...@@ -1086,6 +1054,7 @@ start_server {tags {"external:skip needs:debug"}} { ...@@ -1086,6 +1054,7 @@ start_server {tags {"external:skip needs:debug"}} {
r flushall ; # Clean up keyspace to avoid interference by keys from other tests r flushall ; # Clean up keyspace to avoid interference by keys from other tests
set repl [attach_to_replication_stream] set repl [attach_to_replication_stream]
# HEXPIRE/HPEXPIRE should be translated into HPEXPIREAT
r hset h1 f1 v1 r hset h1 f1 v1
r hexpireat h1 [expr [clock seconds]+100] NX FIELDS 1 f1 r hexpireat h1 [expr [clock seconds]+100] NX FIELDS 1 f1
r hset h2 f2 v2 r hset h2 f2 v2
...@@ -1109,6 +1078,28 @@ start_server {tags {"external:skip needs:debug"}} { ...@@ -1109,6 +1078,28 @@ start_server {tags {"external:skip needs:debug"}} {
close_replication_stream $repl close_replication_stream $repl
} {} {needs:repl} } {} {needs:repl}
test {HRANDFIELD delete expired fields and propagate DELs to replica} {
r flushall
set repl [attach_to_replication_stream]
r hset h4 f1 v1 f2 v2
r hpexpire h4 1 FIELDS 1 f1
r hpexpire h4 2 FIELDS 1 f2
after 100
assert_equal [r hrandfield h4 2] ""
assert_replication_stream $repl {
{select *}
{hset h4 f1 v1 f2 v2}
{hpexpireat h4 * FIELDS 1 f1}
{hpexpireat h4 * FIELDS 1 f2}
{hdel h4 f1}
{hdel h4 f2}
}
close_replication_stream $repl
} {} {needs:repl}
# Start another server to test replication of TTLs # Start another server to test replication of TTLs
start_server {tags {needs:repl external:skip}} { start_server {tags {needs:repl external:skip}} {
# Set the outer layer server as primary # Set the outer layer server as primary
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment