Unverified Commit 323be4d6 authored by Ronen Kalish's avatar Ronen Kalish Committed by GitHub
Browse files

Hfe serialization listpack (#13243)

Add RDB de/serialization for HFE

This PR adds two new RDB types: `RDB_TYPE_HASH_METADATA` and
`RDB_TYPE_HASH_LISTPACK_TTL` to save HFE data.
When the hash RAM encoding is dict, it will be saved in the former, and
when it is listpack it will be saved in the latter.
Both formats just add the TTL value for each field after the data that
was previously saved, i.e HASH_METADATA will save the number of entries
and, for each entry, key, value and TTL, whereas listpack is saved as a
blob.
On read, the usual dict <--> listpack conversion takes place if
required.
In addition, when reading a hash that was saved as a dict fields are
actively expired if expiry is due. Currently this slao holds for
listpack encoding, but it is supposed to be removed.

TODO:
Remove active expiry on load when loading from listpack format (unless
we'll decide to keep it)
parent 71676513
......@@ -237,7 +237,7 @@ void restoreCommand(client *c) {
rioInitWithBuffer(&payload,c->argv[3]->ptr);
if (((type = rdbLoadObjectType(&payload)) == -1) ||
((obj = rdbLoadObject(type,&payload,key->ptr,c->db->id,NULL)) == NULL))
((obj = rdbLoadObject(type,&payload,key->ptr,c->db,0,NULL)) == NULL))
{
addReplyError(c,"Bad data format");
return;
......
......@@ -1403,7 +1403,8 @@ int ebRemove(ebuckets *eb, EbucketsType *type, eItem item) {
* @param item - The eItem to be added to the ebucket.
* @param expireTime - The expiration time of the item.
*
* @return 1 if the item was successfully added; Otherwise, return 0 on failure.
* @return 0 (C_OK) if the item was successfully added;
* Otherwise, return -1 (C_ERR) on failure.
*/
int ebAdd(ebuckets *eb, EbucketsType *type, eItem item, uint64_t expireTime) {
int res;
......
......@@ -245,51 +245,61 @@ unsigned char* lpShrinkToFit(unsigned char *lp) {
static inline void lpEncodeIntegerGetType(int64_t v, unsigned char *intenc, uint64_t *enclen) {
if (v >= 0 && v <= 127) {
/* Single byte 0-127 integer. */
intenc[0] = v;
*enclen = 1;
if (intenc != NULL) intenc[0] = v;
if (enclen != NULL) *enclen = 1;
} else if (v >= -4096 && v <= 4095) {
/* 13 bit integer. */
if (v < 0) v = ((int64_t)1<<13)+v;
intenc[0] = (v>>8)|LP_ENCODING_13BIT_INT;
intenc[1] = v&0xff;
*enclen = 2;
if (intenc != NULL) {
intenc[0] = (v>>8)|LP_ENCODING_13BIT_INT;
intenc[1] = v&0xff;
}
if (enclen != NULL) *enclen = 2;
} else if (v >= -32768 && v <= 32767) {
/* 16 bit integer. */
if (v < 0) v = ((int64_t)1<<16)+v;
intenc[0] = LP_ENCODING_16BIT_INT;
intenc[1] = v&0xff;
intenc[2] = v>>8;
*enclen = 3;
if (intenc != NULL) {
intenc[0] = LP_ENCODING_16BIT_INT;
intenc[1] = v&0xff;
intenc[2] = v>>8;
}
if (enclen != NULL) *enclen = 3;
} else if (v >= -8388608 && v <= 8388607) {
/* 24 bit integer. */
if (v < 0) v = ((int64_t)1<<24)+v;
intenc[0] = LP_ENCODING_24BIT_INT;
intenc[1] = v&0xff;
intenc[2] = (v>>8)&0xff;
intenc[3] = v>>16;
*enclen = 4;
if (intenc != NULL) {
intenc[0] = LP_ENCODING_24BIT_INT;
intenc[1] = v&0xff;
intenc[2] = (v>>8)&0xff;
intenc[3] = v>>16;
}
if (enclen != NULL) *enclen = 4;
} else if (v >= -2147483648 && v <= 2147483647) {
/* 32 bit integer. */
if (v < 0) v = ((int64_t)1<<32)+v;
intenc[0] = LP_ENCODING_32BIT_INT;
intenc[1] = v&0xff;
intenc[2] = (v>>8)&0xff;
intenc[3] = (v>>16)&0xff;
intenc[4] = v>>24;
*enclen = 5;
if (intenc != NULL) {
intenc[0] = LP_ENCODING_32BIT_INT;
intenc[1] = v&0xff;
intenc[2] = (v>>8)&0xff;
intenc[3] = (v>>16)&0xff;
intenc[4] = v>>24;
}
if (enclen != NULL) *enclen = 5;
} else {
/* 64 bit integer. */
uint64_t uv = v;
intenc[0] = LP_ENCODING_64BIT_INT;
intenc[1] = uv&0xff;
intenc[2] = (uv>>8)&0xff;
intenc[3] = (uv>>16)&0xff;
intenc[4] = (uv>>24)&0xff;
intenc[5] = (uv>>32)&0xff;
intenc[6] = (uv>>40)&0xff;
intenc[7] = (uv>>48)&0xff;
intenc[8] = uv>>56;
*enclen = 9;
if (intenc != NULL) {
intenc[0] = LP_ENCODING_64BIT_INT;
intenc[1] = uv&0xff;
intenc[2] = (uv>>8)&0xff;
intenc[3] = (uv>>16)&0xff;
intenc[4] = (uv>>24)&0xff;
intenc[5] = (uv>>32)&0xff;
intenc[6] = (uv>>40)&0xff;
intenc[7] = (uv>>48)&0xff;
intenc[8] = uv>>56;
}
if (enclen != NULL) *enclen = 9;
}
}
......@@ -1199,13 +1209,17 @@ size_t lpBytes(unsigned char *lp) {
return lpGetTotalBytes(lp);
}
/* Returns the size of a listpack consisting of an integer repeated 'rep' times. */
size_t lpEstimateBytesRepeatedInteger(long long lval, unsigned long rep) {
/* Returns the size 'lval' will require when encoded, in bytes */
size_t lpEntrySizeInteger(long long lval) {
uint64_t enclen;
unsigned char intenc[LP_MAX_INT_ENCODING_LEN];
lpEncodeIntegerGetType(lval, intenc, &enclen);
lpEncodeIntegerGetType(lval, NULL, &enclen);
unsigned long backlen = lpEncodeBacklen(NULL, enclen);
return LP_HDR_SIZE + (enclen + backlen) * rep + 1;
return enclen + backlen;
}
/* Returns the size of a listpack consisting of an integer repeated 'rep' times. */
size_t lpEstimateBytesRepeatedInteger(long long lval, unsigned long rep) {
return LP_HDR_SIZE + lpEntrySizeInteger(lval) * rep + 1;
}
/* Seek the specified element and returns the pointer to the seeked element.
......
......@@ -61,6 +61,7 @@ unsigned char *lpLast(unsigned char *lp);
unsigned char *lpNext(unsigned char *lp, unsigned char *p);
unsigned char *lpPrev(unsigned char *lp, unsigned char *p);
size_t lpBytes(unsigned char *lp);
size_t lpEntrySizeInteger(long long lval);
size_t lpEstimateBytesRepeatedInteger(long long lval, unsigned long rep);
unsigned char *lpSeek(unsigned char *lp, long index);
typedef int (*listpackValidateEntryCB)(unsigned char *p, unsigned int head_count, void *userdata);
......
This diff is collapsed.
......@@ -73,10 +73,12 @@
#define RDB_TYPE_STREAM_LISTPACKS_2 19
#define RDB_TYPE_SET_LISTPACK 20
#define RDB_TYPE_STREAM_LISTPACKS_3 21
#define RDB_TYPE_HASH_METADATA 22
#define RDB_TYPE_HASH_LISTPACK_EX 23
/* NOTE: WHEN ADDING NEW RDB TYPE, UPDATE rdbIsObjectType(), and rdb_type_string[] */
/* Test if a type is an object type. */
#define rdbIsObjectType(t) (((t) >= 0 && (t) <= 7) || ((t) >= 9 && (t) <= 21))
#define rdbIsObjectType(t) (((t) >= 0 && (t) <= 7) || ((t) >= 9 && (t) <= 23))
/* Special RDB opcodes (saved/loaded with rdbSaveType/rdbLoadType). */
#define RDB_OPCODE_SLOT_INFO 244 /* Individual slot info, such as slot id and size (cluster mode only). */
......@@ -139,7 +141,7 @@ int rdbSaveToFile(const char *filename);
int rdbSave(int req, char *filename, rdbSaveInfo *rsi, int rdbflags);
ssize_t rdbSaveObject(rio *rdb, robj *o, robj *key, int dbid);
size_t rdbSavedObjectLen(robj *o, robj *key, int dbid);
robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error);
robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, redisDb *db, int rdbflags, int *error);
void backgroundSaveDoneHandler(int exitcode, int bysignal);
int rdbSaveKeyValuePair(rio *rdb, robj *key, robj *val, long long expiretime,int dbid);
ssize_t rdbSaveSingleModuleAux(rio *rdb, int when, moduleType *mt);
......
......@@ -80,6 +80,8 @@ char *rdb_type_string[] = {
"stream-v2",
"set-listpack",
"stream-v3",
"hash-hashtable-md",
"hash-listpack-md",
};
/* Show a few stats collected into 'rdbstate' */
......@@ -173,7 +175,6 @@ void rdbCheckSetupSignals(void) {
* otherwise the already open file 'fp' is checked. */
int redis_check_rdb(char *rdbfilename, FILE *fp) {
uint64_t dbid;
int selected_dbid = -1;
int type, rdbver;
char buf[1024];
long long expiretime, now = mstime();
......@@ -245,7 +246,6 @@ int redis_check_rdb(char *rdbfilename, FILE *fp) {
if ((dbid = rdbLoadLen(&rdb,NULL)) == RDB_LENERR)
goto eoferr;
rdbCheckInfo("Selecting DB ID %llu", (unsigned long long)dbid);
selected_dbid = dbid;
continue; /* Read type again. */
} else if (type == RDB_OPCODE_RESIZEDB) {
/* RESIZEDB: Hint about the size of the keys in the currently
......@@ -331,7 +331,7 @@ int redis_check_rdb(char *rdbfilename, FILE *fp) {
rdbstate.keys++;
/* Read value */
rdbstate.doing = RDB_CHECK_DOING_READ_OBJECT_VALUE;
if ((val = rdbLoadObject(type,&rdb,key->ptr,selected_dbid,NULL)) == NULL) goto eoferr;
if ((val = rdbLoadObject(type,&rdb,key->ptr,NULL,0,NULL)) == NULL) goto eoferr;
/* Check if the key already expired. */
if (expiretime != -1 && expiretime < now)
rdbstate.already_expired++;
......
......@@ -2707,6 +2707,7 @@ void initServer(void) {
server.rdb_save_time_start = -1;
server.rdb_last_load_keys_expired = 0;
server.rdb_last_load_keys_loaded = 0;
server.rdb_last_load_hash_fields_expired = 0;
server.dirty = 0;
resetServerStats();
/* A few stats we don't want to reset: server startup time, and peak mem. */
......@@ -5770,6 +5771,7 @@ sds genRedisInfoString(dict *section_dict, int all_sections, int everything) {
"rdb_last_cow_size:%zu\r\n", server.stat_rdb_cow_bytes,
"rdb_last_load_keys_expired:%lld\r\n", server.rdb_last_load_keys_expired,
"rdb_last_load_keys_loaded:%lld\r\n", server.rdb_last_load_keys_loaded,
"rdb_last_load_hash_fields_expired:%lld\r\n", server.rdb_last_load_hash_fields_expired,
"aof_enabled:%d\r\n", server.aof_state != AOF_OFF,
"aof_rewrite_in_progress:%d\r\n", server.child_type == CHILD_TYPE_AOF,
"aof_rewrite_scheduled:%d\r\n", server.aof_rewrite_scheduled,
......
......@@ -1802,6 +1802,7 @@ struct redisServer {
long long dirty_before_bgsave; /* Used to restore dirty on failed BGSAVE */
long long rdb_last_load_keys_expired; /* number of expired keys when loading RDB */
long long rdb_last_load_keys_loaded; /* number of loaded keys when loading RDB */
long long rdb_last_load_hash_fields_expired; /* number of expired hash fields when loading RDB */
struct saveparam *saveparams; /* Save points array for RDB */
int saveparamslen; /* Number of saving points */
char *rdb_filename; /* Name of RDB file */
......@@ -3200,6 +3201,13 @@ unsigned char *hashTypeListpackGetLp(robj *o);
uint64_t hashTypeGetMinExpire(robj *o);
void hashTypeUpdateKeyRef(robj *o, sds newkey);
ebuckets *hashTypeGetDictMetaHFE(dict *d);
void listpackExExpire(robj *o, ExpireInfo *info);
int hashTypeSetExRdb(redisDb *db, robj *o, sds field, sds value, uint64_t expire_at);
uint64_t hashTypeGetMinExpire(robj *keyObj);
uint64_t hashTypeGetNextTimeToExpire(robj *o);
void initDictExpireMetadata(sds key, robj *o);
struct listpackEx *listpackExCreate(void);
void listpackExAddNew(robj *o, sds field, sds value, uint64_t expireAt);
/* Hash-Field data type (of t_hash.c) */
hfield hfieldNew(const void *field, size_t fieldlen, int withExpireMeta);
......@@ -3210,6 +3218,7 @@ uint64_t hfieldGetExpireTime(hfield field);
static inline void hfieldFree(hfield field) { mstrFree(&mstrFieldKind, field); }
static inline void *hfieldGetAllocPtr(hfield field) { return mstrGetAllocPtr(&mstrFieldKind, field); }
static inline size_t hfieldlen(hfield field) { return mstrlen(field);}
uint64_t hfieldGetExpireTime(hfield field);
/* Pub / Sub */
int pubsubUnsubscribeAllChannels(client *c, int notify);
......
......@@ -22,7 +22,6 @@ static void hexpireGenericCommand(client *c, const char *cmd, long long basetime
static ExpireAction hashTypeActiveExpire(eItem hashObj, void *ctx);
static void hfieldPersist(robj *hashObj, hfield field);
static void updateGlobalHfeDs(redisDb *db, robj *o, uint64_t minExpire, uint64_t minExpireFields);
static uint64_t hashTypeGetNextTimeToExpire(robj *o);
/* hash dictType funcs */
static int dictHfieldKeyCompare(dict *d, const void *key1, const void *key2);
......@@ -317,7 +316,7 @@ static void hashDictWithExpireOnRelease(dict *d) {
#define HASH_LP_NO_TTL 0
static struct listpackEx *listpackExCreate(void) {
struct listpackEx *listpackExCreate(void) {
listpackEx *lpt = zcalloc(sizeof(*lpt));
lpt->meta.trash = 1;
lpt->lp = NULL;
......@@ -384,7 +383,7 @@ static uint64_t listpackExGetMinExpire(robj *o) {
}
/* Walk over fields and delete the expired ones. */
static void listpackExExpire(robj *o, ExpireInfo *info) {
void listpackExExpire(robj *o, ExpireInfo *info) {
serverAssert(o->encoding == OBJ_ENCODING_LISTPACK_EX);
uint64_t min = EB_EXPIRE_TIME_INVALID;
unsigned char *ptr, *field, *s;
......@@ -408,7 +407,6 @@ static void listpackExExpire(robj *o, ExpireInfo *info) {
if (val == HASH_LP_NO_TTL || (uint64_t) val > info->now)
break;
server.stat_expired_hash_fields++;
lpt->lp = lpDeleteRangeWithEntry(lpt->lp, &field, 3);
ptr = field;
info->itemsExpired++;
......@@ -542,7 +540,7 @@ out:
}
/* Add new field ordered by expire time. */
static void listpackExAddNew(robj *o, sds field, sds value, uint64_t expireAt) {
void listpackExAddNew(robj *o, sds field, sds value, uint64_t expireAt) {
unsigned char *fptr, *s, *elem;
listpackEx *lpt = o->ptr;
......@@ -968,6 +966,9 @@ SetExRes hashTypeSetExpiry(HashTypeSetEx *ex, sds field, uint64_t expireAt, dict
*
* Take care to call first hashTypeSetExInit() and then call this function.
* Finally, call hashTypeSetExDone() to notify and update global HFE DS.
*
* NOTE: this functions is also called during RDB load to set dict-encoded
* fields with and without expiration.
*/
SetExRes hashTypeSetEx(redisDb *db, robj *o, sds field, HashTypeSet *setKeyVal,
uint64_t expireAt, HashTypeSetEx *exInfo)
......@@ -1048,6 +1049,42 @@ SetExDone:
return res;
}
/*
* hashTypeSetExRdb provide a simplified API for setting fields & expiry by RDB load
*
* It is the duty of RDB reading process to track minimal expiration time of the
* fields and eventually call hashTypeAddToExpires() to update global HFE DS with
* next expiration time.
*
* To just add a field with no expiry, use hashTypeSet instead.
*/
int hashTypeSetExRdb(redisDb *db, robj *o, sds field, sds value, uint64_t expire_at) {
/* Dummy struct to be used in hashTypeSetEx() */
HashTypeSetEx setEx = {
.fieldSetCond = FIELD_DONT_OVRWRT, /* Shouldn't be any duplication */
.expireSetCond = HFE_NX, /* Should set expiry once each field */
.minExpire = EB_EXPIRE_TIME_INVALID, /* Won't be used. Accounting made by RDB already */
.key = NULL, /* Not going to call hashTypeSetExDone() */
.hashObj = o,
.minExpireFields = EB_EXPIRE_TIME_INVALID, /* Not needed by RDB */
.c = NULL, /* No notification required */
.cmd = NULL, /* No notification required */
};
HashTypeSet setKeyVal = {.value = value, .flags = 0};
SetExRes res = hashTypeSetEx(db, o, field, &setKeyVal, expire_at, (expire_at) ? &setEx : NULL);
return (res == HSETEX_OK || res == HSET_UPDATE) ? C_OK : C_ERR;
}
void initDictExpireMetadata(sds key, robj *o) {
dict *ht = o->ptr;
dictExpireMetadata *m = (dictExpireMetadata *) dictMetadata(ht);
m->key = key;
m->hfe = ebCreate(); /* Allocate HFE DS */
m->expireMeta.trash = 1; /* mark as trash (as long it wasn't ebAdd()) */
}
/*
* Init HashTypeSetEx struct before calling hashTypeSetEx()
*
......@@ -1645,6 +1682,7 @@ void hashTypeConvertListpackEx(robj *o, int enc, ebuckets *hexpires) {
}
}
/* NOTE: hexpires can be NULL (Won't attempt to register in global HFE DS) */
void hashTypeConvert(robj *o, int enc, ebuckets *hexpires) {
if (o->encoding == OBJ_ENCODING_LISTPACK) {
hashTypeConvertListpack(o, enc);
......@@ -1816,6 +1854,7 @@ static ExpireAction hashTypeActiveExpire(eItem _hashObj, void *ctx) {
.itemsExpired = 0};
listpackExExpire(hashObj, &info);
server.stat_expired_hash_fields += info.itemsExpired;
keystr = ((listpackEx*)hashObj->ptr)->key;
} else {
serverAssert(hashObj->encoding == OBJ_ENCODING_HT);
......@@ -2650,6 +2689,7 @@ static ExpireMeta* hfieldGetExpireMeta(const eItem field) {
return mstrMetaRef(field, &mstrFieldKind, (int) HFIELD_META_EXPIRE);
}
/* returned value is unix time in milliseconds */
uint64_t hfieldGetExpireTime(hfield field) {
if (!hfieldIsExpireAttached(field))
return EB_EXPIRE_TIME_INVALID;
......
......@@ -416,4 +416,264 @@ start_server {} {
} {OK}
}
set server_path [tmpdir "server.partial-hfield-exp-test"]
# verifies writing and reading hash key with expiring and persistent fields
start_server [list overrides [list "dir" $server_path]] {
foreach {type lp_entries} {listpack 512 dict 0} {
test "hash field expiration save and load rdb one expired field, ($type)" {
r config set hash-max-listpack-entries $lp_entries
r FLUSHALL
r HMSET key a 1 b 2 c 3 d 4
r HEXPIREAT key 2524600800 2 a b
r HPEXPIRE key 100 1 d
r save
# sleep 101 ms to make sure d will expire after restart
after 101
restart_server 0 true false
wait_done_loading r
assert_equal [lsort [r hgetall key]] "1 2 3 a b c"
assert_equal [r hexpiretime key 3 a b c] {2524600800 2524600800 -1}
assert_equal [s rdb_last_load_keys_loaded] 1
# hash keys saved in listpack encoding are loaded as a blob,
# so individual field expiry is not verified on load
if {$type eq "dict"} {
assert_equal [s rdb_last_load_hash_fields_expired] 1
} else {
assert_equal [s rdb_last_load_hash_fields_expired] 0
}
}
}
}
set server_path [tmpdir "server.all-hfield-exp-test"]
# verifies writing hash with several expired keys, and active-expiring it on load
start_server [list overrides [list "dir" $server_path]] {
foreach {type lp_entries} {listpack 512 dict 0} {
test "hash field expiration save and load rdb all fields expired, ($type)" {
r config set hash-max-listpack-entries $lp_entries
r FLUSHALL
r HMSET key a 1 b 2 c 3 d 4
r HPEXPIRE key 100 4 a b c d
r save
# sleep 101 ms to make sure all fields will expire after restart
after 101
restart_server 0 true false
wait_done_loading r
# hash keys saved as listpack-encoded are saved and loaded as a blob
# so individual field validation is not checked during load.
# Therefore, if the key was saved as dict it is expected that
# all 4 fields were expired during load, and thus the key was
# "declared" an empty key.
# On the other hand, if the key was saved as listpack, it is
# expected that no field was expired on load and the key was loaded,
# even though all its fields are actually expired.
if {$type eq "dict"} {
assert_equal [s rdb_last_load_keys_loaded] 0
assert_equal [s rdb_last_load_hash_fields_expired] 4
} else {
assert_equal [s rdb_last_load_keys_loaded] 1
assert_equal [s rdb_last_load_hash_fields_expired] 0
}
# in listpack encoding, the fields (and key) will be expired by
# lazy expiry
assert_equal [r hgetall key] {}
}
}
}
set server_path [tmpdir "server.long-ttl-test"]
# verifies a long TTL value (6 bytes) is saved and loaded correctly
start_server [list overrides [list "dir" $server_path]] {
foreach {type lp_entries} {listpack 512 dict 0} {
test "hash field expiration save and load rdb long TTL, ($type)" {
r config set hash-max-listpack-entries $lp_entries
r FLUSHALL
r HSET key a 1
# set expiry to 0xabcdef987654 (6 bytes)
r HPEXPIREAT key 188900976391764 1 a
r save
restart_server 0 true false
wait_done_loading r
assert_equal [r hget key a ] 1
assert_equal [r hpexpiretime key 1 a] {188900976391764}
}
}
}
set server_path [tmpdir "server.listpack-to-dict-test"]
test "save listpack, load dict" {
start_server [list overrides [list "dir" $server_path enable-debug-command yes]] {
r config set hash-max-listpack-entries 512
r FLUSHALL
r HMSET key a 1 b 2 c 3 d 4
assert_match "*encoding:listpack*" [r debug object key]
r HPEXPIRE key 100 1 d
r save
# sleep 200 ms to make sure 'd' will expire after when reloading
after 200
# change configuration and reload - result should be dict-encoded key
r config set hash-max-listpack-entries 0
r debug reload nosave
# first verify d was not expired during load (no expiry when loading
# a hash that was saved listpack-encoded)
assert_equal [s rdb_last_load_keys_loaded] 1
assert_equal [s rdb_last_load_hash_fields_expired] 0
# d should be lazy expired in hgetall
assert_equal [lsort [r hgetall key]] "1 2 3 a b c"
assert_match "*encoding:hashtable*" [r debug object key]
}
}
set server_path [tmpdir "server.dict-to-listpack-test"]
test "save dict, load listpack" {
start_server [list overrides [list "dir" $server_path enable-debug-command yes]] {
r config set hash-max-listpack-entries 0
r FLUSHALL
r HMSET key a 1 b 2 c 3 d 4
assert_match "*encoding:hashtable*" [r debug object key]
r HPEXPIRE key 200 1 d
r save
# sleep 201 ms to make sure 'd' will expire during reload
after 201
# change configuration and reload - result should be LP-encoded key
r config set hash-max-listpack-entries 512
r debug reload nosave
# verify d was expired during load
assert_equal [s rdb_last_load_keys_loaded] 1
assert_equal [s rdb_last_load_hash_fields_expired] 1
assert_equal [lsort [r hgetall key]] "1 2 3 a b c"
assert_match "*encoding:listpack*" [r debug object key]
}
}
set server_path [tmpdir "server.active-expiry-after-load"]
# verifies a field is correctly expired by active expiry AFTER loading from RDB
foreach {type lp_entries} {listpack 512 dict 0} {
start_server [list overrides [list "dir" $server_path enable-debug-command yes]] {
test "active field expiry after load, ($type)" {
r config set hash-max-listpack-entries $lp_entries
r FLUSHALL
r HMSET key a 1 b 2 c 3 d 4 e 5 f 6
r HEXPIREAT key 2524600800 2 a b
r HPEXPIRE key 200 2 c d
r save
r debug reload nosave
# wait at most 2 secs to make sure 'c' and 'd' will active-expire
wait_for_condition 20 100 {
[s expired_hash_fields] == 2
} else {
fail "expired hash fields is [s expired_hash_fields] != 2"
}
assert_equal [s rdb_last_load_keys_loaded] 1
assert_equal [s rdb_last_load_hash_fields_expired] 0
# hgetall might lazy expire fields, so it's only called after the stat asserts
assert_equal [lsort [r hgetall key]] "1 2 5 6 a b e f"
assert_equal [r hexpiretime key 6 a b c d e f] {2524600800 2524600800 -2 -2 -1 -1}
}
}
}
set server_path [tmpdir "server.lazy-expiry-after-load"]
foreach {type lp_entries} {listpack 512 dict 0} {
start_server [list overrides [list "dir" $server_path enable-debug-command yes]] {
test "lazy field expiry after load, ($type)" {
r config set hash-max-listpack-entries $lp_entries
r debug set-active-expire 0
r FLUSHALL
r HMSET key a 1 b 2 c 3 d 4 e 5 f 6
r HEXPIREAT key 2524600800 2 a b
r HPEXPIRE key 200 2 c d
r save
r debug reload nosave
# sleep 500 msec to make sure 'c' and 'd' will lazy-expire when calling hgetall
after 500
assert_equal [s rdb_last_load_keys_loaded] 1
assert_equal [s rdb_last_load_hash_fields_expired] 0
assert_equal [s expired_hash_fields] 0
# hgetall will lazy expire fields, so it's only called after the stat asserts
assert_equal [lsort [r hgetall key]] "1 2 5 6 a b e f"
assert_equal [r hexpiretime key 6 a b c d e f] {2524600800 2524600800 -2 -2 -1 -1}
}
}
}
set server_path [tmpdir "server.unexpired-items-rax-list-boundary"]
foreach {type lp_entries} {listpack 512 dict 0} {
start_server [list overrides [list "dir" $server_path enable-debug-command yes]] {
test "load un-expired items below and above rax-list boundary, ($type)" {
r config set hash-max-listpack-entries $lp_entries
r flushall
set hash_sizes {15 16 17 31 32 33}
foreach h $hash_sizes {
for {set i 1} {$i <= $h} {incr i} {
r hset key$h f$i v$i
r hexpireat key$h 2524600800 1 f$i
}
}
r save
restart_server 0 true false
wait_done_loading r
set hash_sizes {15 16 17 31 32 33}
foreach h $hash_sizes {
for {set i 1} {$i <= $h} {incr i} {
# random expiration time
assert_equal [r hget key$h f$i] v$i
assert_equal [r hexpiretime key$h 1 f$i] 2524600800
}
}
}
}
}
} ;# tags
......@@ -438,8 +438,14 @@ proc csvdump r {
hash {
set fields [{*}$r hgetall $k]
set newfields {}
foreach {k v} $fields {
lappend newfields [list $k $v]
foreach {f v} $fields {
set expirylist [{*}$r hexpiretime $k 1 $f]
if {$expirylist eq (-1)} {
lappend newfields [list $f $v]
} else {
set e [lindex $expirylist 0]
lappend newfields [list $f $e $v] # TODO: extract the actual ttl value from the list in $e
}
}
set fields [lsort -index 0 $newfields]
foreach kv $fields {
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment