Unverified Commit ce121b92 authored by Moti Cohen's avatar Moti Cohen Committed by GitHub
Browse files

HFE - Avoid lazy expire if called by modules + cleanup (#13326)

Need to be carefull if called by modules since modules API allow to open
and close key handler. We don't want to invalidate the handler
underneath.

* hashTypeExists(), hashTypeGetValueObject() - will return the logical
state of the field. A flag will indicate noExpire.
* RM_HashGet() - Will get NULL if the field expired. Fields won’t be
deleted.
* RM_ScanKey() - might return 0 items if all fields got expired. Fields
won’t be deleted.
* RM_HashSet() - If set, then override expired field. If delete, we can
either delete or leave it to active-expiration. XX/NX - logically
correct (Verify with tests).

Nice to have (not implemented):
* RedisModule_CloseKey() - We can local active-expire up-to 100 items. 

Note:
Length will be wrong to modules just like redis (Count expired fields).
parent 690ef363
...@@ -5271,10 +5271,21 @@ int RM_HashSet(RedisModuleKey *key, int flags, ...) { ...@@ -5271,10 +5271,21 @@ int RM_HashSet(RedisModuleKey *key, int flags, ...) {
   
/* Handle XX and NX */ /* Handle XX and NX */
if (flags & (REDISMODULE_HASH_XX|REDISMODULE_HASH_NX)) { if (flags & (REDISMODULE_HASH_XX|REDISMODULE_HASH_NX)) {
int isHashDeleted; int hfeFlags = HFE_LAZY_AVOID_HASH_DEL; /* Avoid invalidate the key */
int exists = hashTypeExists(key->db, key->value, field->ptr, &isHashDeleted);
/* hash-field-expiration is not exposed to modules */ /*
serverAssert(isHashDeleted == 0); * The hash might contain expired fields. If we lazily delete expired
* field and the command was sent with XX flag, the operation could
* fail and leave the hash empty, which the caller might not expect.
* To prevent unexpected behavior, we avoid lazy deletion in this case
* yet let the operation fail. Note that moduleDelKeyIfEmpty()
* below won't delete the hash if it left with single expired key
* because hash counts blindly expired fields as well.
*/
if (flags & REDISMODULE_HASH_XX)
hfeFlags |= HFE_LAZY_AVOID_FIELD_DEL;
int exists = hashTypeExists(key->db, key->value, field->ptr, hfeFlags, NULL);
if (((flags & REDISMODULE_HASH_XX) && !exists) || if (((flags & REDISMODULE_HASH_XX) && !exists) ||
((flags & REDISMODULE_HASH_NX) && exists)) ((flags & REDISMODULE_HASH_NX) && exists))
{ {
...@@ -5357,6 +5368,7 @@ int RM_HashSet(RedisModuleKey *key, int flags, ...) { ...@@ -5357,6 +5368,7 @@ int RM_HashSet(RedisModuleKey *key, int flags, ...) {
* RedisModule_FreeString(), or by enabling automatic memory management. * RedisModule_FreeString(), or by enabling automatic memory management.
*/ */
int RM_HashGet(RedisModuleKey *key, int flags, ...) { int RM_HashGet(RedisModuleKey *key, int flags, ...) {
int hfeFlags = HFE_LAZY_AVOID_FIELD_DEL | HFE_LAZY_AVOID_HASH_DEL;
va_list ap; va_list ap;
if (key->value && key->value->type != OBJ_HASH) return REDISMODULE_ERR; if (key->value && key->value->type != OBJ_HASH) return REDISMODULE_ERR;
   
...@@ -5378,21 +5390,16 @@ int RM_HashGet(RedisModuleKey *key, int flags, ...) { ...@@ -5378,21 +5390,16 @@ int RM_HashGet(RedisModuleKey *key, int flags, ...) {
if (flags & REDISMODULE_HASH_EXISTS) { if (flags & REDISMODULE_HASH_EXISTS) {
existsptr = va_arg(ap,int*); existsptr = va_arg(ap,int*);
if (key->value) { if (key->value) {
int isHashDeleted; *existsptr = hashTypeExists(key->db, key->value, field->ptr, hfeFlags, NULL);
*existsptr = hashTypeExists(key->db, key->value, field->ptr, &isHashDeleted);
/* hash-field-expiration is not exposed to modules */
serverAssert(isHashDeleted == 0);
} else { } else {
*existsptr = 0; *existsptr = 0;
} }
} else { } else {
int isHashDeleted;
valueptr = va_arg(ap,RedisModuleString**); valueptr = va_arg(ap,RedisModuleString**);
if (key->value) { if (key->value) {
*valueptr = hashTypeGetValueObject(key->db,key->value,field->ptr, &isHashDeleted); *valueptr = hashTypeGetValueObject(key->db, key->value, field->ptr,
hfeFlags, NULL);
   
/* Currently hash-field-expiration is not exposed to modules */
serverAssert(isHashDeleted == 0);
if (*valueptr) { if (*valueptr) {
robj *decoded = getDecodedObject(*valueptr); robj *decoded = getDecodedObject(*valueptr);
decrRefCount(*valueptr); decrRefCount(*valueptr);
...@@ -11080,6 +11087,11 @@ static void moduleScanKeyCallback(void *privdata, const dictEntry *de) { ...@@ -11080,6 +11087,11 @@ static void moduleScanKeyCallback(void *privdata, const dictEntry *de) {
value = NULL; value = NULL;
} else if (o->type == OBJ_HASH) { } else if (o->type == OBJ_HASH) {
sds val = dictGetVal(de); sds val = dictGetVal(de);
/* If field is expired, then ignore */
if (hfieldIsExpired(key))
return;
field = createStringObject(key, hfieldlen(key)); field = createStringObject(key, hfieldlen(key));
value = createStringObject(val, sdslen(val)); value = createStringObject(val, sdslen(val));
} else if (o->type == OBJ_ZSET) { } else if (o->type == OBJ_ZSET) {
...@@ -11189,9 +11201,8 @@ int RM_ScanKey(RedisModuleKey *key, RedisModuleScanCursor *cursor, RedisModuleSc ...@@ -11189,9 +11201,8 @@ int RM_ScanKey(RedisModuleKey *key, RedisModuleScanCursor *cursor, RedisModuleSc
ret = 0; ret = 0;
} else if (o->type == OBJ_ZSET || o->type == OBJ_HASH) { } else if (o->type == OBJ_ZSET || o->type == OBJ_HASH) {
unsigned char *lp, *p; unsigned char *lp, *p;
unsigned char *vstr; /* is hash with expiry on fields, then lp tuples are [field][value][expire] */
unsigned int vlen; int hfe = o->type == OBJ_HASH && o->encoding == OBJ_ENCODING_LISTPACK_EX;
long long vll;
   
if (o->type == OBJ_HASH) if (o->type == OBJ_HASH)
lp = hashTypeListpackGetLp(o); lp = hashTypeListpackGetLp(o);
...@@ -11200,19 +11211,32 @@ int RM_ScanKey(RedisModuleKey *key, RedisModuleScanCursor *cursor, RedisModuleSc ...@@ -11200,19 +11211,32 @@ int RM_ScanKey(RedisModuleKey *key, RedisModuleScanCursor *cursor, RedisModuleSc
   
p = lpSeek(lp,0); p = lpSeek(lp,0);
while(p) { while(p) {
vstr = lpGetValue(p,&vlen,&vll); long long vllField, vllValue, vllExpire;
robj *field = (vstr != NULL) ? unsigned int lenField, lenValue;
createStringObject((char*)vstr,vlen) : unsigned char *pField, *pValue;
createStringObjectFromLongLongWithSds(vll);
pField = lpGetValue(p,&lenField,&vllField);
p = lpNext(lp,p); p = lpNext(lp,p);
vstr = lpGetValue(p,&vlen,&vll); pValue = lpGetValue(p,&lenValue,&vllValue);
robj *value = (vstr != NULL) ?
createStringObject((char*)vstr,vlen) :
createStringObjectFromLongLongWithSds(vll);
fn(key, field, value, privdata);
p = lpNext(lp,p); p = lpNext(lp,p);
if (o->type == OBJ_HASH && o->encoding == OBJ_ENCODING_LISTPACK_EX)
p = lpNext(lp, p); /* Skip expire time */ if (hfe) {
serverAssert(lpGetIntegerValue(p, &vllExpire));
p = lpNext(lp, p);
/* Skip expired fields */
if (hashTypeIsExpired(o, vllExpire))
continue;
}
robj *value = (pValue != NULL) ?
createStringObject((char*)pValue,lenValue) :
createStringObjectFromLongLongWithSds(vllValue);
robj *field = (pField != NULL) ?
createStringObject((char*)pField,lenField) :
createStringObjectFromLongLongWithSds(vllField);
fn(key, field, value, privdata);
   
decrRefCount(field); decrRefCount(field);
decrRefCount(value); decrRefCount(value);
...@@ -11225,7 +11249,6 @@ int RM_ScanKey(RedisModuleKey *key, RedisModuleScanCursor *cursor, RedisModuleSc ...@@ -11225,7 +11249,6 @@ int RM_ScanKey(RedisModuleKey *key, RedisModuleScanCursor *cursor, RedisModuleSc
return ret; return ret;
} }
   
/* -------------------------------------------------------------------------- /* --------------------------------------------------------------------------
* ## Module fork API * ## Module fork API
* -------------------------------------------------------------------------- */ * -------------------------------------------------------------------------- */
......
...@@ -3191,9 +3191,14 @@ typedef struct dictExpireMetadata { ...@@ -3191,9 +3191,14 @@ typedef struct dictExpireMetadata {
#define HASH_SET_TAKE_VALUE (1<<1) #define HASH_SET_TAKE_VALUE (1<<1)
#define HASH_SET_COPY 0 #define HASH_SET_COPY 0
/* Hash field lazy expiration flags. Used by core hashTypeGetValue() and its callers */
#define HFE_LAZY_EXPIRE (0) /* Delete expired field, and if last field also the hash */
#define HFE_LAZY_AVOID_FIELD_DEL (1<<0) /* Avoid deleting expired field */
#define HFE_LAZY_AVOID_HASH_DEL (1<<1) /* Avoid deleting hash if the field is the last one */
void hashTypeConvert(robj *o, int enc, ebuckets *hexpires); void hashTypeConvert(robj *o, int enc, ebuckets *hexpires);
void hashTypeTryConversion(redisDb *db, robj *subject, robj **argv, int start, int end); void hashTypeTryConversion(redisDb *db, robj *subject, robj **argv, int start, int end);
int hashTypeExists(redisDb *db, robj *o, sds key, int *isHashDeleted); int hashTypeExists(redisDb *db, robj *o, sds key, int hfeFlags, int *isHashDeleted);
int hashTypeDelete(robj *o, void *key, int isSdsField); int hashTypeDelete(robj *o, void *key, int isSdsField);
unsigned long hashTypeLength(const robj *o, int subtractExpiredFields); unsigned long hashTypeLength(const robj *o, int subtractExpiredFields);
hashTypeIterator *hashTypeInitIterator(robj *subject); hashTypeIterator *hashTypeInitIterator(robj *subject);
...@@ -3210,7 +3215,7 @@ void hashTypeCurrentObject(hashTypeIterator *hi, int what, unsigned char **vstr, ...@@ -3210,7 +3215,7 @@ void hashTypeCurrentObject(hashTypeIterator *hi, int what, unsigned char **vstr,
unsigned int *vlen, long long *vll, uint64_t *expireTime); unsigned int *vlen, long long *vll, uint64_t *expireTime);
sds hashTypeCurrentObjectNewSds(hashTypeIterator *hi, int what); sds hashTypeCurrentObjectNewSds(hashTypeIterator *hi, int what);
hfield hashTypeCurrentObjectNewHfield(hashTypeIterator *hi); hfield hashTypeCurrentObjectNewHfield(hashTypeIterator *hi);
robj *hashTypeGetValueObject(redisDb *db, robj *o, sds field, int *isHashDeleted); robj *hashTypeGetValueObject(redisDb *db, robj *o, sds field, int hfeFlags, int *isHashDeleted);
int hashTypeSet(redisDb *db, robj *o, sds field, sds value, int flags); int hashTypeSet(redisDb *db, robj *o, sds field, sds value, int flags);
robj *hashTypeDup(robj *o, sds newkey, uint64_t *minHashExpire); robj *hashTypeDup(robj *o, sds newkey, uint64_t *minHashExpire);
uint64_t hashTypeRemoveFromExpires(ebuckets *hexpires, robj *o); uint64_t hashTypeRemoveFromExpires(ebuckets *hexpires, robj *o);
......
...@@ -95,7 +95,7 @@ robj *lookupKeyByPattern(redisDb *db, robj *pattern, robj *subst) { ...@@ -95,7 +95,7 @@ robj *lookupKeyByPattern(redisDb *db, robj *pattern, robj *subst) {
/* Retrieve value from hash by the field name. The returned object /* Retrieve value from hash by the field name. The returned object
* is a new object with refcount already incremented. */ * is a new object with refcount already incremented. */
int isHashDeleted; int isHashDeleted;
o = hashTypeGetValueObject(db, o, fieldobj->ptr, &isHashDeleted); o = hashTypeGetValueObject(db, o, fieldobj->ptr, HFE_LAZY_EXPIRE, &isHashDeleted);
if (isHashDeleted) if (isHashDeleted)
goto noobj; goto noobj;
......
...@@ -21,7 +21,7 @@ typedef enum GetFieldRes { ...@@ -21,7 +21,7 @@ typedef enum GetFieldRes {
GETF_NOT_FOUND, /* The field was not found. */ GETF_NOT_FOUND, /* The field was not found. */
/* used only by hashTypeGetValue() */ /* used only by hashTypeGetValue() */
GETF_EXPIRED, /* Logically expired but not yet deleted. */ GETF_EXPIRED, /* Logically expired (Might be lazy deleted or not) */
GETF_EXPIRED_HASH, /* Delete hash since retrieved field was expired and GETF_EXPIRED_HASH, /* Delete hash since retrieved field was expired and
* it was the last field in the hash. */ * it was the last field in the hash. */
} GetFieldRes; } GetFieldRes;
...@@ -152,21 +152,18 @@ static inline int isDictWithMetaHFE(dict *d) { ...@@ -152,21 +152,18 @@ static inline int isDictWithMetaHFE(dict *d) {
} }
/*----------------------------------------------------------------------------- /*-----------------------------------------------------------------------------
* setex* - Set field OR field's expiration * setex* - Set field's expiration
* *
* Whereas setting plain fields is rather straightforward, setting expiration * Setting expiration time to fields might be time-consuming and complex since
* time to fields might be time-consuming and complex since each update of * each update of expiration time, not only updates `ebuckets` of corresponding
* expiration time, not only updates `ebuckets` of corresponding hash, but also * hash, but also might update `ebuckets` of global HFE DS. It is required to opt
* might update `ebuckets` of global HFE DS. It is required to opt sequence of * sequence of field updates with expirartion for a given hash, such that only
* field updates with expirartion for a given hash, such that only once done, * once done, the global HFE DS will get updated.
* the global HFE DS will get updated.
* *
* To do so, follow the scheme: * To do so, follow the scheme:
* 1. Call hashTypeSetExInit() to initialize the HashTypeSetEx struct. * 1. Call hashTypeSetExInit() to initialize the HashTypeSetEx struct.
* 2. Call hashTypeSetEx() one time or more, for each field/expiration update. * 2. Call hashTypeSetEx() one time or more, for each field/expiration update.
* 3. Call hashTypeSetExDone() for notification and update of global HFE. * 3. Call hashTypeSetExDone() for notification and update of global HFE.
*
* If expiration is not required, then avoid this API and use instead hashTypeSet()
*----------------------------------------------------------------------------*/ *----------------------------------------------------------------------------*/
/* Returned value of hashTypeSetEx() */ /* Returned value of hashTypeSetEx() */
...@@ -178,10 +175,6 @@ typedef enum SetExRes { ...@@ -178,10 +175,6 @@ typedef enum SetExRes {
HSETEX_NO_FIELD = -2, /* No such hash-field */ HSETEX_NO_FIELD = -2, /* No such hash-field */
HSETEX_NO_CONDITION_MET = 0, /* Specified NX | XX | GT | LT condition not met */ HSETEX_NO_CONDITION_MET = 0, /* Specified NX | XX | GT | LT condition not met */
HSETEX_DELETED = 2, /* Field deleted because the specified time is in the past */ HSETEX_DELETED = 2, /* Field deleted because the specified time is in the past */
/* If not provided HashTypeSetEx struct to hashTypeSetEx() (plain HSET) */
HSET_UPDATE = 4, /* Update of the field without expiration time */
} SetExRes; } SetExRes;
/* Used by httlGenericCommand() */ /* Used by httlGenericCommand() */
...@@ -190,14 +183,6 @@ typedef enum GetExpireTimeRes { ...@@ -190,14 +183,6 @@ typedef enum GetExpireTimeRes {
HFE_GET_NO_TTL = -1, /* No TTL attached to the field */ HFE_GET_NO_TTL = -1, /* No TTL attached to the field */
} GetExpireTimeRes; } GetExpireTimeRes;
/* on fail return HSETEX_NO_CONDITION_MET */
typedef enum FieldSetCond {
FIELD_CREATE_OR_OVRWRT = 0,
FIELD_DONT_CREATE = 1,
FIELD_DONT_CREATE2 = 2, /* on fail return HSETEX_NO_FIELD */
FIELD_DONT_OVRWRT = 3
} FieldSetCond;
typedef enum FieldGet { /* TBD */ typedef enum FieldGet { /* TBD */
FIELD_GET_NONE = 0, FIELD_GET_NONE = 0,
FIELD_GET_NEW = 1, FIELD_GET_NEW = 1,
...@@ -220,7 +205,6 @@ typedef struct HashTypeSet { ...@@ -220,7 +205,6 @@ typedef struct HashTypeSet {
typedef struct HashTypeSetEx { typedef struct HashTypeSetEx {
/*** config ***/ /*** config ***/
FieldSetCond fieldSetCond; /* [DCF | DOF] */
ExpireSetCond expireSetCond; /* [XX | NX | GT | LT] */ ExpireSetCond expireSetCond; /* [XX | NX | GT | LT] */
/*** metadata ***/ /*** metadata ***/
...@@ -239,14 +223,10 @@ typedef struct HashTypeSetEx { ...@@ -239,14 +223,10 @@ typedef struct HashTypeSetEx {
const char *cmd; const char *cmd;
} HashTypeSetEx; } HashTypeSetEx;
static SetExRes hashTypeSetExListpack(redisDb *db, robj *o, sds field, HashTypeSet *setParams,
uint64_t expireAt, HashTypeSetEx *exParams);
int hashTypeSetExInit(robj *key, robj *o, client *c, redisDb *db, const char *cmd, int hashTypeSetExInit(robj *key, robj *o, client *c, redisDb *db, const char *cmd,
FieldSetCond fieldSetCond, ExpireSetCond expireSetCond, HashTypeSetEx *ex); ExpireSetCond expireSetCond, HashTypeSetEx *ex);
SetExRes hashTypeSetEx(redisDb *db, robj *o, sds field, HashTypeSet *setKeyVal, SetExRes hashTypeSetEx(robj *o, sds field, uint64_t expireAt, HashTypeSetEx *exInfo);
uint64_t expireAt, HashTypeSetEx *exInfo);
void hashTypeSetExDone(HashTypeSetEx *e); void hashTypeSetExDone(HashTypeSetEx *e);
...@@ -712,23 +692,21 @@ GetFieldRes hashTypeGetFromHashTable(robj *o, sds field, sds *value, uint64_t *e ...@@ -712,23 +692,21 @@ GetFieldRes hashTypeGetFromHashTable(robj *o, sds field, sds *value, uint64_t *e
/* Higher level function of hashTypeGet*() that returns the hash value /* Higher level function of hashTypeGet*() that returns the hash value
* associated with the specified field. * associated with the specified field.
* Arguments:
* hfeFlags - Lookup for HFE_LAZY_* flags
* *
* Returned: * Returned:
* - GetFieldRes: OK: Return Field's valid value * GetFieldRes - Result of get operation
* NOT_FOUND: Field was not found. * vstr, vlen - if string, ref in either *vstr and *vlen if it's
* EXPIRED: Field is expired and Lazy deleted
* EXPIRED_HASH: Returned only if the field is the last one in the
* hash and the hash is deleted.
* - vstr, vlen : if string, ref in either *vstr and *vlen if it's
* returned in string form, * returned in string form,
* - vll : or stored in *vll if it's returned as a number. * vll - or stored in *vll if it's returned as a number.
* If *vll is populated *vstr is set to NULL, so the caller can * If *vll is populated *vstr is set to NULL, so the caller can
* always check the function return by checking the return value * always check the function return by checking the return value
* for GETF_OK and checking if vll (or vstr) is NULL. * for GETF_OK and checking if vll (or vstr) is NULL.
* *
*/ */
GetFieldRes hashTypeGetValue(redisDb *db, robj *o, sds field, unsigned char **vstr, GetFieldRes hashTypeGetValue(redisDb *db, robj *o, sds field, unsigned char **vstr,
unsigned int *vlen, long long *vll) { unsigned int *vlen, long long *vll, int hfeFlags) {
uint64_t expiredAt; uint64_t expiredAt;
sds key; sds key;
GetFieldRes res; GetFieldRes res;
...@@ -760,7 +738,12 @@ GetFieldRes hashTypeGetValue(redisDb *db, robj *o, sds field, unsigned char **vs ...@@ -760,7 +738,12 @@ GetFieldRes hashTypeGetValue(redisDb *db, robj *o, sds field, unsigned char **vs
(expiredAt >= (uint64_t) commandTimeSnapshot()) ) (expiredAt >= (uint64_t) commandTimeSnapshot()) )
return GETF_OK; return GETF_OK;
/* Got expired. Extract attached key from LISTPACK_EX/HT */ /* Field is expired */
/* If indicated to avoid deleting expired field */
if (hfeFlags & HFE_LAZY_AVOID_FIELD_DEL)
return GETF_EXPIRED;
if (o->encoding == OBJ_ENCODING_LISTPACK_EX) if (o->encoding == OBJ_ENCODING_LISTPACK_EX)
key = ((listpackEx *) o->ptr)->key; key = ((listpackEx *) o->ptr)->key;
else else
...@@ -771,10 +754,11 @@ GetFieldRes hashTypeGetValue(redisDb *db, robj *o, sds field, unsigned char **vs ...@@ -771,10 +754,11 @@ GetFieldRes hashTypeGetValue(redisDb *db, robj *o, sds field, unsigned char **vs
propagateHashFieldDeletion(db, key, field, sdslen(field)); propagateHashFieldDeletion(db, key, field, sdslen(field));
/* If the field is the last one in the hash, then the hash will be deleted */ /* If the field is the last one in the hash, then the hash will be deleted */
if (hashTypeLength(o, 0) == 0) { if ((hashTypeLength(o, 0) == 0) && (!(hfeFlags & HFE_LAZY_AVOID_HASH_DEL))) {
robj *keyObj = createStringObject(key, sdslen(key)); robj *keyObj = createStringObject(key, sdslen(key));
notifyKeyspaceEvent(NOTIFY_GENERIC, "del", keyObj, db->id); notifyKeyspaceEvent(NOTIFY_GENERIC, "del", keyObj, db->id);
dbDelete(db,keyObj); dbDelete(db,keyObj);
signalModifiedKey(NULL, db, keyObj);
decrRefCount(keyObj); decrRefCount(keyObj);
return GETF_EXPIRED_HASH; return GETF_EXPIRED_HASH;
} }
...@@ -787,24 +771,25 @@ GetFieldRes hashTypeGetValue(redisDb *db, robj *o, sds field, unsigned char **vs ...@@ -787,24 +771,25 @@ GetFieldRes hashTypeGetValue(redisDb *db, robj *o, sds field, unsigned char **vs
* The function returns NULL if the field is not found in the hash. Otherwise * The function returns NULL if the field is not found in the hash. Otherwise
* a newly allocated string object with the value is returned. * a newly allocated string object with the value is returned.
* *
* hfeFlags - Lookup HFE_LAZY_* flags
* isHashDeleted - If attempted to access expired field and it's the last field * isHashDeleted - If attempted to access expired field and it's the last field
* in the hash, then the hash will as well be deleted. In this case, * in the hash, then the hash will as well be deleted. In this case,
* isHashDeleted will be set to 1. * isHashDeleted will be set to 1.
*/ */
robj *hashTypeGetValueObject(redisDb *db, robj *o, sds field, int *isHashDeleted) { robj *hashTypeGetValueObject(redisDb *db, robj *o, sds field, int hfeFlags, int *isHashDeleted) {
unsigned char *vstr; unsigned char *vstr;
unsigned int vlen; unsigned int vlen;
long long vll; long long vll;
*isHashDeleted = 0; /*default*/ if (isHashDeleted) *isHashDeleted = 0;
GetFieldRes res = hashTypeGetValue(db,o,field,&vstr,&vlen,&vll); GetFieldRes res = hashTypeGetValue(db,o,field,&vstr,&vlen,&vll, hfeFlags);
if (res == GETF_OK) { if (res == GETF_OK) {
if (vstr) return createStringObject((char*)vstr,vlen); if (vstr) return createStringObject((char*)vstr,vlen);
else return createStringObjectFromLongLong(vll); else return createStringObjectFromLongLong(vll);
} }
if (res == GETF_EXPIRED_HASH) if ((res == GETF_EXPIRED_HASH) && (isHashDeleted))
*isHashDeleted = 1; *isHashDeleted = 1;
/* GETF_EXPIRED_HASH, GETF_EXPIRED, GETF_NOT_FOUND */ /* GETF_EXPIRED_HASH, GETF_EXPIRED, GETF_NOT_FOUND */
...@@ -814,18 +799,20 @@ robj *hashTypeGetValueObject(redisDb *db, robj *o, sds field, int *isHashDeleted ...@@ -814,18 +799,20 @@ robj *hashTypeGetValueObject(redisDb *db, robj *o, sds field, int *isHashDeleted
/* Test if the specified field exists in the given hash. If the field is /* Test if the specified field exists in the given hash. If the field is
* expired (HFE), then it will be lazy deleted * expired (HFE), then it will be lazy deleted
* *
* Returns 1 if the field exists, and 0 when it doesn't. * hfeFlags - Lookup HFE_LAZY_* flags
*
* isHashDeleted - If attempted to access expired field and it is the last field * isHashDeleted - If attempted to access expired field and it is the last field
* in the hash, then the hash will as well be deleted. In this case, * in the hash, then the hash will as well be deleted. In this case,
* isHashDeleted will be set to 1. * isHashDeleted will be set to 1.
*
* Returns 1 if the field exists, and 0 when it doesn't.
*/ */
int hashTypeExists(redisDb *db, robj *o, sds field, int *isHashDeleted) { int hashTypeExists(redisDb *db, robj *o, sds field, int hfeFlags, int *isHashDeleted) {
unsigned char *vstr = NULL; unsigned char *vstr = NULL;
unsigned int vlen = UINT_MAX; unsigned int vlen = UINT_MAX;
long long vll = LLONG_MAX; long long vll = LLONG_MAX;
GetFieldRes res = hashTypeGetValue(db, o, field, &vstr, &vlen, &vll); GetFieldRes res = hashTypeGetValue(db, o, field, &vstr, &vlen, &vll, hfeFlags);
if (isHashDeleted)
*isHashDeleted = (res == GETF_EXPIRED_HASH) ? 1 : 0; *isHashDeleted = (res == GETF_EXPIRED_HASH) ? 1 : 0;
return (res == GETF_OK) ? 1 : 0; return (res == GETF_OK) ? 1 : 0;
} }
...@@ -839,7 +826,7 @@ int hashTypeExists(redisDb *db, robj *o, sds field, int *isHashDeleted) { ...@@ -839,7 +826,7 @@ int hashTypeExists(redisDb *db, robj *o, sds field, int *isHashDeleted) {
* *
* HASH_SET_TAKE_FIELD -- The SDS field ownership passes to the function. * HASH_SET_TAKE_FIELD -- The SDS field ownership passes to the function.
* HASH_SET_TAKE_VALUE -- The SDS value ownership passes to the function. * HASH_SET_TAKE_VALUE -- The SDS value ownership passes to the function.
* HASH_SET_KEEP_FIELD -- keep original field along with TTL if already exists * HASH_SET_KEEP_TTL -- keep original TTL if field already exists
* *
* When the flags are used the caller does not need to release the passed * When the flags are used the caller does not need to release the passed
* SDS string(s). It's up to the function to use the string to create a new * SDS string(s). It's up to the function to use the string to create a new
...@@ -851,45 +838,142 @@ int hashTypeExists(redisDb *db, robj *o, sds field, int *isHashDeleted) { ...@@ -851,45 +838,142 @@ int hashTypeExists(redisDb *db, robj *o, sds field, int *isHashDeleted) {
*/ */
#define HASH_SET_TAKE_FIELD (1<<0) #define HASH_SET_TAKE_FIELD (1<<0)
#define HASH_SET_TAKE_VALUE (1<<1) #define HASH_SET_TAKE_VALUE (1<<1)
#define HASH_SET_KEEP_FIELD (1<<2) #define HASH_SET_KEEP_TTL (1<<2)
#define HASH_SET_COPY 0 #define HASH_SET_COPY 0
int hashTypeSet(redisDb *db, robj *o, sds field, sds value, int flags) { int hashTypeSet(redisDb *db, robj *o, sds field, sds value, int flags) {
HashTypeSet set = {value, flags}; int update = 0;
return (hashTypeSetEx(db, o, field, &set, 0, NULL) == HSET_UPDATE) ? 1 : 0;
}
SetExRes hashTypeSetExpiry(HashTypeSetEx *ex, sds field, uint64_t expireAt, dictEntry **de) { /* Check if the field is too long for listpack, and convert before adding the item.
dict *ht = ex->hashObj->ptr; * This is needed for HINCRBY* case since in other commands this is handled early by
dictEntry *newEntry = NULL, *existingEntry = NULL; * hashTypeTryConversion, so this check will be a NOP. */
if (o->encoding == OBJ_ENCODING_LISTPACK ||
o->encoding == OBJ_ENCODING_LISTPACK_EX) {
if (sdslen(field) > server.hash_max_listpack_value || sdslen(value) > server.hash_max_listpack_value)
hashTypeConvert(o, OBJ_ENCODING_HT, &db->hexpires);
}
/* New field with expiration metadata */ if (o->encoding == OBJ_ENCODING_LISTPACK) {
hfield hfNew = hfieldNew(field, sdslen(field), 1 /*withExpireMeta*/); unsigned char *zl, *fptr, *vptr;
if ((ex->fieldSetCond == FIELD_DONT_CREATE) || (ex->fieldSetCond == FIELD_DONT_CREATE2)) { zl = o->ptr;
if ((existingEntry = dictFind(ht, field)) == NULL) { fptr = lpFirst(zl);
hfieldFree(hfNew); if (fptr != NULL) {
return (ex->fieldSetCond == FIELD_DONT_CREATE) ? fptr = lpFind(zl, fptr, (unsigned char*)field, sdslen(field), 1);
HSETEX_NO_CONDITION_MET : HSETEX_NO_FIELD; if (fptr != NULL) {
/* Grab pointer to the value (fptr points to the field) */
vptr = lpNext(zl, fptr);
serverAssert(vptr != NULL);
/* Replace value */
zl = lpReplace(zl, &vptr, (unsigned char*)value, sdslen(value));
update = 1;
} }
} else { }
if (!update) {
/* Push new field/value pair onto the tail of the listpack */
zl = lpAppend(zl, (unsigned char*)field, sdslen(field));
zl = lpAppend(zl, (unsigned char*)value, sdslen(value));
}
o->ptr = zl;
/* Check if the listpack needs to be converted to a hash table */
if (hashTypeLength(o, 0) > server.hash_max_listpack_entries)
hashTypeConvert(o, OBJ_ENCODING_HT, &db->hexpires);
} else if (o->encoding == OBJ_ENCODING_LISTPACK_EX) {
unsigned char *fptr = NULL, *vptr = NULL, *tptr = NULL;
listpackEx *lpt = o->ptr;
long long expireTime = HASH_LP_NO_TTL;
fptr = lpFirst(lpt->lp);
if (fptr != NULL) {
fptr = lpFind(lpt->lp, fptr, (unsigned char*)field, sdslen(field), 2);
if (fptr != NULL) {
/* Grab pointer to the value (fptr points to the field) */
vptr = lpNext(lpt->lp, fptr);
serverAssert(vptr != NULL);
/* Replace value */
lpt->lp = lpReplace(lpt->lp, &vptr, (unsigned char *) value, sdslen(value));
update = 1;
fptr = lpPrev(lpt->lp, vptr);
serverAssert(fptr != NULL);
tptr = lpNext(lpt->lp, vptr);
serverAssert(tptr && lpGetIntegerValue(tptr, &expireTime));
if (flags & HASH_SET_KEEP_TTL) {
/* keep old field along with TTL */
} else if (expireTime != HASH_LP_NO_TTL) {
/* re-insert field and override TTL */
listpackExUpdateExpiry(o, field, fptr, vptr, HASH_LP_NO_TTL);
}
}
}
if (!update)
listpackExAddNew(o, field, sdslen(field), value, sdslen(value),
HASH_LP_NO_TTL);
/* Check if the listpack needs to be converted to a hash table */
if (hashTypeLength(o, 0) > server.hash_max_listpack_entries)
hashTypeConvert(o, OBJ_ENCODING_HT, &db->hexpires);
} else if (o->encoding == OBJ_ENCODING_HT) {
hfield newField = hfieldNew(field, sdslen(field), 0);
dict *ht = o->ptr;
dictEntry *de, *existing;
/* stored key is different than lookup key */
dictUseStoredKeyApi(ht, 1); dictUseStoredKeyApi(ht, 1);
newEntry = dictAddRaw(ht, hfNew, &existingEntry); de = dictAddRaw(ht, newField, &existing);
dictUseStoredKeyApi(ht, 0); dictUseStoredKeyApi(ht, 0);
}
if (newEntry) { /* If field already exists, then update "field". "Value" will be set afterward */
*de = newEntry; if (de == NULL) {
if (flags & HASH_SET_KEEP_TTL) {
/* keep old field along with TTL */
hfieldFree(newField);
} else {
/* If attached TTL to the old field, then remove it from hash's private ebuckets */
hfield oldField = dictGetKey(existing);
hfieldPersist(o, oldField);
hfieldFree(oldField);
dictSetKey(ht, existing, newField);
}
sdsfree(dictGetVal(existing));
update = 1;
de = existing;
}
if (ex->expireSetCond & (HFE_XX | HFE_LT | HFE_GT)) { if (flags & HASH_SET_TAKE_VALUE) {
dictDelete(ht, field); dictSetVal(ht, de, value);
return HSETEX_NO_CONDITION_MET; flags &= ~HASH_SET_TAKE_VALUE;
} else {
dictSetVal(ht, de, sdsdup(value));
} }
} else { /* field exist */ } else {
*de = existingEntry; serverPanic("Unknown hash encoding");
}
/* Free SDS strings we did not referenced elsewhere if the flags
* want this function to be responsible. */
if (flags & HASH_SET_TAKE_FIELD && field) sdsfree(field);
if (flags & HASH_SET_TAKE_VALUE && value) sdsfree(value);
return update;
}
SetExRes hashTypeSetExpiryHT(HashTypeSetEx *exInfo, sds field, uint64_t expireAt) {
dict *ht = exInfo->hashObj->ptr;
dictEntry *existingEntry = NULL;
/* New field with expiration metadata */
hfield hfNew = hfieldNew(field, sdslen(field), 1 /*withExpireMeta*/);
if (ex->fieldSetCond == FIELD_DONT_OVRWRT) { if ((existingEntry = dictFind(ht, field)) == NULL) {
hfieldFree(hfNew); hfieldFree(hfNew);
return HSETEX_NO_CONDITION_MET; return HSETEX_NO_FIELD;
} }
hfield hfOld = dictGetKey(existingEntry); hfield hfOld = dictGetKey(existingEntry);
...@@ -898,7 +982,7 @@ SetExRes hashTypeSetExpiry(HashTypeSetEx *ex, sds field, uint64_t expireAt, dict ...@@ -898,7 +982,7 @@ SetExRes hashTypeSetExpiry(HashTypeSetEx *ex, sds field, uint64_t expireAt, dict
if (!hfieldIsExpireAttached(hfOld)) { if (!hfieldIsExpireAttached(hfOld)) {
/* For fields without expiry, LT condition is considered valid */ /* For fields without expiry, LT condition is considered valid */
if (ex->expireSetCond & (HFE_XX | HFE_GT)) { if (exInfo->expireSetCond & (HFE_XX | HFE_GT)) {
hfieldFree(hfNew); hfieldFree(hfNew);
return HSETEX_NO_CONDITION_MET; return HSETEX_NO_CONDITION_MET;
} }
...@@ -915,9 +999,9 @@ SetExRes hashTypeSetExpiry(HashTypeSetEx *ex, sds field, uint64_t expireAt, dict ...@@ -915,9 +999,9 @@ SetExRes hashTypeSetExpiry(HashTypeSetEx *ex, sds field, uint64_t expireAt, dict
/* If field has valid expiration time, then check GT|LT|NX */ /* If field has valid expiration time, then check GT|LT|NX */
if (prevExpire != EB_EXPIRE_TIME_INVALID) { if (prevExpire != EB_EXPIRE_TIME_INVALID) {
if (((ex->expireSetCond == HFE_GT) && (prevExpire >= expireAt)) || if (((exInfo->expireSetCond == HFE_GT) && (prevExpire >= expireAt)) ||
((ex->expireSetCond == HFE_LT) && (prevExpire <= expireAt)) || ((exInfo->expireSetCond == HFE_LT) && (prevExpire <= expireAt)) ||
(ex->expireSetCond == HFE_NX) ) (exInfo->expireSetCond == HFE_NX) )
return HSETEX_NO_CONDITION_MET; return HSETEX_NO_CONDITION_MET;
/* remove old expiry time from hash's private ebuckets */ /* remove old expiry time from hash's private ebuckets */
...@@ -925,14 +1009,14 @@ SetExRes hashTypeSetExpiry(HashTypeSetEx *ex, sds field, uint64_t expireAt, dict ...@@ -925,14 +1009,14 @@ SetExRes hashTypeSetExpiry(HashTypeSetEx *ex, sds field, uint64_t expireAt, dict
ebRemove(&dm->hfe, &hashFieldExpireBucketsType, hfOld); ebRemove(&dm->hfe, &hashFieldExpireBucketsType, hfOld);
/* Track of minimum expiration time (only later update global HFE DS) */ /* Track of minimum expiration time (only later update global HFE DS) */
if (ex->minExpireFields > prevExpire) if (exInfo->minExpireFields > prevExpire)
ex->minExpireFields = prevExpire; exInfo->minExpireFields = prevExpire;
} else { } else {
/* field has invalid expiry. No need to ebRemove() */ /* field has invalid expiry. No need to ebRemove() */
/* Check XX|LT|GT */ /* Check XX|LT|GT */
if (ex->expireSetCond & (HFE_XX | HFE_GT)) if (exInfo->expireSetCond & (HFE_XX | HFE_GT))
return HSETEX_NO_CONDITION_MET; return HSETEX_NO_CONDITION_MET;
} }
...@@ -941,114 +1025,64 @@ SetExRes hashTypeSetExpiry(HashTypeSetEx *ex, sds field, uint64_t expireAt, dict ...@@ -941,114 +1025,64 @@ SetExRes hashTypeSetExpiry(HashTypeSetEx *ex, sds field, uint64_t expireAt, dict
} }
dictSetKey(ht, existingEntry, hfNew); dictSetKey(ht, existingEntry, hfNew);
}
/* if expiration time is in the past */ /* if expiration time is in the past */
if (unlikely(checkAlreadyExpired(expireAt))) { if (unlikely(checkAlreadyExpired(expireAt))) {
hashTypeDelete(ex->hashObj, field, 1); hashTypeDelete(exInfo->hashObj, field, 1);
ex->fieldDeleted++; exInfo->fieldDeleted++;
return HSETEX_DELETED; return HSETEX_DELETED;
} }
if (ex->minExpireFields > expireAt) if (exInfo->minExpireFields > expireAt)
ex->minExpireFields = expireAt; exInfo->minExpireFields = expireAt;
dictExpireMetadata *dm = (dictExpireMetadata *) dictMetadata(ht); dictExpireMetadata *dm = (dictExpireMetadata *) dictMetadata(ht);
ebAdd(&dm->hfe, &hashFieldExpireBucketsType, hfNew, expireAt); ebAdd(&dm->hfe, &hashFieldExpireBucketsType, hfNew, expireAt);
ex->fieldUpdated++; exInfo->fieldUpdated++;
return HSETEX_OK; return HSETEX_OK;
} }
/* /*
* Set fields OR field's expiration (See also `setex*` comment above) * Set field expiration
* *
* Take care to call first hashTypeSetExInit() and then call this function. * Take care to call first hashTypeSetExInit() and then call this function.
* Finally, call hashTypeSetExDone() to notify and update global HFE DS. * Finally, call hashTypeSetExDone() to notify and update global HFE DS.
*
* NOTE: this functions is also called during RDB load to set dict-encoded
* fields with and without expiration.
*/ */
SetExRes hashTypeSetEx(redisDb *db, robj *o, sds field, HashTypeSet *setKeyVal, SetExRes hashTypeSetEx(robj *o, sds field, uint64_t expireAt, HashTypeSetEx *exInfo)
uint64_t expireAt, HashTypeSetEx *exInfo)
{ {
SetExRes res = HSETEX_OK; if (o->encoding == OBJ_ENCODING_LISTPACK_EX)
int isSetKeyValue = (setKeyVal) ? 1 : 0;
int isSetExpire = (exInfo) ? 1 : 0;
int flags = (setKeyVal) ? setKeyVal->flags : 0;
/* Check if the field is too long for listpack, and convert before adding the item.
* This is needed for HINCRBY* case since in other commands this is handled early by
* hashTypeTryConversion, so this check will be a NOP. */
if (o->encoding == OBJ_ENCODING_LISTPACK ||
o->encoding == OBJ_ENCODING_LISTPACK_EX)
{
if ( (isSetKeyValue) &&
(sdslen(field) > server.hash_max_listpack_value ||
sdslen(setKeyVal->value) > server.hash_max_listpack_value) )
{ {
hashTypeConvert(o, OBJ_ENCODING_HT, &db->hexpires); unsigned char *fptr = NULL, *vptr = NULL, *tptr = NULL;
} else {
res = hashTypeSetExListpack(db, o, field, setKeyVal, expireAt, exInfo);
goto SetExDone; /*done*/
}
}
if (o->encoding != OBJ_ENCODING_HT) listpackEx *lpt = o->ptr;
serverPanic("Unknown hash encoding"); long long expireTime = HASH_LP_NO_TTL;
/*** now deal with HT ***/ if ((fptr = lpFirst(lpt->lp)) == NULL)
hfield newField; return HSETEX_NO_FIELD;
dict *ht = o->ptr;
dictEntry *de;
/* If needed to set the field along with expiry */ fptr = lpFind(lpt->lp, fptr, (unsigned char*)field, sdslen(field), 2);
if (isSetExpire) {
res = hashTypeSetExpiry(exInfo, field, expireAt, &de);
if (res != HSETEX_OK) goto SetExDone;
} else {
dictEntry *existing;
/* Cannot leverage HASH_SET_TAKE_FIELD since hfield is not of type sds */
newField = hfieldNew(field, sdslen(field), 0);
/* stored key is different than lookup key */ if (!fptr)
dictUseStoredKeyApi(ht, 1); return HSETEX_NO_FIELD;
de = dictAddRaw(ht, newField, &existing);
dictUseStoredKeyApi(ht, 0);
/* If field already exists, then update "field". "Value" will be set afterward */ /* Grab pointer to the value (fptr points to the field) */
if (de == NULL) { vptr = lpNext(lpt->lp, fptr);
if (flags & HASH_SET_KEEP_FIELD) { serverAssert(vptr != NULL);
/* Not keep old field along with TTL */
hfieldFree(newField);
} else {
/* If attached TTL to the old field, then remove it from hash's private ebuckets */
hfield oldField = dictGetKey(existing);
hfieldPersist(o, oldField);
hfieldFree(oldField);
dictSetKey(ht, existing, newField);
}
sdsfree(dictGetVal(existing));
res = HSET_UPDATE;
de = existing;
}
}
/* If need to set value */ tptr = lpNext(lpt->lp, vptr);
if (isSetKeyValue) { serverAssert(tptr && lpGetIntegerValue(tptr, &expireTime));
if (flags & HASH_SET_TAKE_VALUE) {
dictSetVal(ht, de, setKeyVal->value); /* update TTL */
flags &= ~HASH_SET_TAKE_VALUE; return hashTypeSetExpiryListpack(exInfo, field, fptr, vptr, tptr, expireAt);
} else if (o->encoding == OBJ_ENCODING_HT) {
/* If needed to set the field along with expiry */
return hashTypeSetExpiryHT(exInfo, field, expireAt);
} else { } else {
dictSetVal(ht, de, sdsdup(setKeyVal->value)); serverPanic("Unknown hash encoding");
}
} }
SetExDone: return HSETEX_OK; /* never reach here */
/* Free SDS strings we did not referenced elsewhere if the flags
* want this function to be responsible. */
if (flags & HASH_SET_TAKE_FIELD && field) sdsfree(field);
if (flags & HASH_SET_TAKE_VALUE && setKeyVal->value) sdsfree(setKeyVal->value);
return res;
} }
void initDictExpireMetadata(sds key, robj *o) { void initDictExpireMetadata(sds key, robj *o) {
...@@ -1066,12 +1100,10 @@ void initDictExpireMetadata(sds key, robj *o) { ...@@ -1066,12 +1100,10 @@ void initDictExpireMetadata(sds key, robj *o) {
* Don't have to provide client and "cmd". If provided, then notification once * Don't have to provide client and "cmd". If provided, then notification once
* done by function hashTypeSetExDone(). * done by function hashTypeSetExDone().
*/ */
int hashTypeSetExInit(robj *key, robj *o, client *c, redisDb *db, const char *cmd, FieldSetCond fieldSetCond, int hashTypeSetExInit(robj *key, robj *o, client *c, redisDb *db, const char *cmd,
ExpireSetCond expireSetCond, HashTypeSetEx *ex) ExpireSetCond expireSetCond, HashTypeSetEx *ex)
{ {
dict *ht = o->ptr; dict *ht = o->ptr;
ex->fieldSetCond = fieldSetCond;
ex->expireSetCond = expireSetCond; ex->expireSetCond = expireSetCond;
ex->minExpire = EB_EXPIRE_TIME_INVALID; ex->minExpire = EB_EXPIRE_TIME_INVALID;
ex->c = c; ex->c = c;
...@@ -1123,15 +1155,15 @@ void hashTypeSetExDone(HashTypeSetEx *ex) { ...@@ -1123,15 +1155,15 @@ void hashTypeSetExDone(HashTypeSetEx *ex) {
/* Notify keyspace event, update dirty count and update global HFE DS */ /* Notify keyspace event, update dirty count and update global HFE DS */
if (ex->fieldDeleted + ex->fieldUpdated > 0) { if (ex->fieldDeleted + ex->fieldUpdated > 0) {
if (ex->c) {
server.dirty += ex->fieldDeleted + ex->fieldUpdated; server.dirty += ex->fieldDeleted + ex->fieldUpdated;
signalModifiedKey(ex->c, ex->db, ex->key);
notifyKeyspaceEvent(NOTIFY_HASH, "hexpire", ex->key, ex->db->id);
}
if (ex->fieldDeleted && hashTypeLength(ex->hashObj, 0) == 0) { if (ex->fieldDeleted && hashTypeLength(ex->hashObj, 0) == 0) {
dbDelete(ex->db,ex->key); dbDelete(ex->db,ex->key);
if (ex->c) notifyKeyspaceEvent(NOTIFY_GENERIC,"del",ex->key, ex->db->id); signalModifiedKey(ex->c, ex->db, ex->key);
notifyKeyspaceEvent(NOTIFY_GENERIC,"del",ex->key, ex->db->id);
} else { } else {
signalModifiedKey(ex->c, ex->db, ex->key);
notifyKeyspaceEvent(NOTIFY_HASH, "hexpire", ex->key, ex->db->id);
/* If minimum HFE of the hash is smaller than expiration time of the /* If minimum HFE of the hash is smaller than expiration time of the
* specified fields in the command as well as it is smaller or equal * specified fields in the command as well as it is smaller or equal
* than expiration time provided in the command, then the minimum * than expiration time provided in the command, then the minimum
...@@ -1161,99 +1193,6 @@ void hashTypeSetExDone(HashTypeSetEx *ex) { ...@@ -1161,99 +1193,6 @@ void hashTypeSetExDone(HashTypeSetEx *ex) {
} }
} }
/* Check if the field is too long for listpack, and convert before adding the item.
* This is needed for HINCRBY* case since in other commands this is handled early by
* hashTypeTryConversion, so this check will be a NOP. */
static SetExRes hashTypeSetExListpack(redisDb *db, robj *o, sds field, HashTypeSet *setParams,
uint64_t expireAt, HashTypeSetEx *exParams)
{
int res = HSETEX_OK;
unsigned char *fptr = NULL, *vptr = NULL, *tptr = NULL;
if (o->encoding == OBJ_ENCODING_LISTPACK) {
/* If reached here, then no need to set expiration. Otherwise, as precond
* listpack is converted to listpackex by hashTypeSetExInit() */
unsigned char *zl = o->ptr;
fptr = lpFirst(zl);
if (fptr != NULL) {
fptr = lpFind(zl, fptr, (unsigned char*)field, sdslen(field), 1);
if (fptr != NULL) {
/* Grab pointer to the value (fptr points to the field) */
vptr = lpNext(zl, fptr);
serverAssert(vptr != NULL);
res = HSET_UPDATE;
/* Replace value */
zl = lpReplace(zl, &vptr, (unsigned char *) setParams->value, sdslen(setParams->value));
}
}
if (res != HSET_UPDATE) {
/* Push new field/value pair onto the tail of the listpack */
zl = lpAppend(zl, (unsigned char*)field, sdslen(field));
zl = lpAppend(zl, (unsigned char*)setParams->value, sdslen(setParams->value));
}
o->ptr = zl;
goto out;
} else if (o->encoding == OBJ_ENCODING_LISTPACK_EX) {
listpackEx *lpt = o->ptr;
long long expireTime = HASH_LP_NO_TTL;
fptr = lpFirst(lpt->lp);
if (fptr != NULL) {
fptr = lpFind(lpt->lp, fptr, (unsigned char*)field, sdslen(field), 2);
if (fptr != NULL) {
/* Grab pointer to the value (fptr points to the field) */
vptr = lpNext(lpt->lp, fptr);
serverAssert(vptr != NULL);
if (setParams) {
/* Replace value */
lpt->lp = lpReplace(lpt->lp, &vptr,
(unsigned char *) setParams->value,
sdslen(setParams->value));
fptr = lpPrev(lpt->lp, vptr);
serverAssert(fptr != NULL);
res = HSET_UPDATE;
}
tptr = lpNext(lpt->lp, vptr);
serverAssert(tptr && lpGetIntegerValue(tptr, &expireTime));
/* Keep, update or clear TTL */
if (setParams && setParams->flags & HASH_SET_KEEP_FIELD) {
/* keep old field along with TTL */
} else if (exParams) {
res = hashTypeSetExpiryListpack(exParams, field, fptr, vptr, tptr,
expireAt);
if (res != HSETEX_OK)
goto out;
} else if (res == HSET_UPDATE && expireTime != HASH_LP_NO_TTL) {
/* Clear TTL */
listpackExUpdateExpiry(o, field, fptr, vptr, HASH_LP_NO_TTL);
}
}
}
if (!fptr) {
if (setParams) {
listpackExAddNew(o, field, sdslen(field),
setParams->value, sdslen(setParams->value),
exParams ? expireAt : HASH_LP_NO_TTL);
} else {
res = HSETEX_NO_FIELD;
}
}
}
out:
/* Check if the listpack needs to be converted to a hash table */
if (hashTypeLength(o, 0) > server.hash_max_listpack_entries)
hashTypeConvert(o, OBJ_ENCODING_HT, &db->hexpires);
return res;
}
/* Delete an element from a hash. /* Delete an element from a hash.
* *
* Return 1 on deleted and 0 on not found. * Return 1 on deleted and 0 on not found.
...@@ -2079,7 +2018,7 @@ void hsetnxCommand(client *c) { ...@@ -2079,7 +2018,7 @@ void hsetnxCommand(client *c) {
robj *o; robj *o;
if ((o = hashTypeLookupWriteOrCreate(c,c->argv[1])) == NULL) return; if ((o = hashTypeLookupWriteOrCreate(c,c->argv[1])) == NULL) return;
if (hashTypeExists(c->db, o, c->argv[2]->ptr, &isHashDeleted)) { if (hashTypeExists(c->db, o, c->argv[2]->ptr, HFE_LAZY_EXPIRE, &isHashDeleted)) {
addReply(c, shared.czero); addReply(c, shared.czero);
return; return;
} }
...@@ -2137,7 +2076,8 @@ void hincrbyCommand(client *c) { ...@@ -2137,7 +2076,8 @@ void hincrbyCommand(client *c) {
if (getLongLongFromObjectOrReply(c,c->argv[3],&incr,NULL) != C_OK) return; if (getLongLongFromObjectOrReply(c,c->argv[3],&incr,NULL) != C_OK) return;
if ((o = hashTypeLookupWriteOrCreate(c,c->argv[1])) == NULL) return; if ((o = hashTypeLookupWriteOrCreate(c,c->argv[1])) == NULL) return;
GetFieldRes res = hashTypeGetValue(c->db,o,c->argv[2]->ptr,&vstr,&vlen,&value); GetFieldRes res = hashTypeGetValue(c->db,o,c->argv[2]->ptr,&vstr,&vlen,&value,
HFE_LAZY_EXPIRE);
if (res == GETF_OK) { if (res == GETF_OK) {
if (vstr) { if (vstr) {
if (string2ll((char*)vstr,vlen,&value) == 0) { if (string2ll((char*)vstr,vlen,&value) == 0) {
...@@ -2162,7 +2102,7 @@ void hincrbyCommand(client *c) { ...@@ -2162,7 +2102,7 @@ void hincrbyCommand(client *c) {
} }
value += incr; value += incr;
new = sdsfromlonglong(value); new = sdsfromlonglong(value);
hashTypeSet(c->db, o,c->argv[2]->ptr,new,HASH_SET_TAKE_VALUE | HASH_SET_KEEP_FIELD); hashTypeSet(c->db, o,c->argv[2]->ptr,new,HASH_SET_TAKE_VALUE | HASH_SET_KEEP_TTL);
addReplyLongLong(c,value); addReplyLongLong(c,value);
signalModifiedKey(c,c->db,c->argv[1]); signalModifiedKey(c,c->db,c->argv[1]);
notifyKeyspaceEvent(NOTIFY_HASH,"hincrby",c->argv[1],c->db->id); notifyKeyspaceEvent(NOTIFY_HASH,"hincrby",c->argv[1],c->db->id);
...@@ -2183,7 +2123,8 @@ void hincrbyfloatCommand(client *c) { ...@@ -2183,7 +2123,8 @@ void hincrbyfloatCommand(client *c) {
return; return;
} }
if ((o = hashTypeLookupWriteOrCreate(c,c->argv[1])) == NULL) return; if ((o = hashTypeLookupWriteOrCreate(c,c->argv[1])) == NULL) return;
GetFieldRes res = hashTypeGetValue(c->db, o,c->argv[2]->ptr,&vstr,&vlen,&ll); GetFieldRes res = hashTypeGetValue(c->db, o,c->argv[2]->ptr,&vstr,&vlen,&ll,
HFE_LAZY_EXPIRE);
if (res == GETF_OK) { if (res == GETF_OK) {
if (vstr) { if (vstr) {
if (string2ld((char*)vstr,vlen,&value) == 0) { if (string2ld((char*)vstr,vlen,&value) == 0) {
...@@ -2211,7 +2152,7 @@ void hincrbyfloatCommand(client *c) { ...@@ -2211,7 +2152,7 @@ void hincrbyfloatCommand(client *c) {
char buf[MAX_LONG_DOUBLE_CHARS]; char buf[MAX_LONG_DOUBLE_CHARS];
int len = ld2string(buf,sizeof(buf),value,LD_STR_HUMAN); int len = ld2string(buf,sizeof(buf),value,LD_STR_HUMAN);
new = sdsnewlen(buf,len); new = sdsnewlen(buf,len);
hashTypeSet(c->db, o,c->argv[2]->ptr,new,HASH_SET_TAKE_VALUE | HASH_SET_KEEP_FIELD); hashTypeSet(c->db, o,c->argv[2]->ptr,new,HASH_SET_TAKE_VALUE | HASH_SET_KEEP_TTL);
addReplyBulkCBuffer(c,buf,len); addReplyBulkCBuffer(c,buf,len);
signalModifiedKey(c,c->db,c->argv[1]); signalModifiedKey(c,c->db,c->argv[1]);
notifyKeyspaceEvent(NOTIFY_HASH,"hincrbyfloat",c->argv[1],c->db->id); notifyKeyspaceEvent(NOTIFY_HASH,"hincrbyfloat",c->argv[1],c->db->id);
...@@ -2237,7 +2178,8 @@ static GetFieldRes addHashFieldToReply(client *c, robj *o, sds field) { ...@@ -2237,7 +2178,8 @@ static GetFieldRes addHashFieldToReply(client *c, robj *o, sds field) {
unsigned int vlen = UINT_MAX; unsigned int vlen = UINT_MAX;
long long vll = LLONG_MAX; long long vll = LLONG_MAX;
GetFieldRes res = hashTypeGetValue(c->db, o, field, &vstr, &vlen, &vll); GetFieldRes res = hashTypeGetValue(c->db, o, field, &vstr, &vlen, &vll,
HFE_LAZY_EXPIRE);
if (res == GETF_OK) { if (res == GETF_OK) {
if (vstr) { if (vstr) {
addReplyBulkCBuffer(c, vstr, vlen); addReplyBulkCBuffer(c, vstr, vlen);
...@@ -2330,7 +2272,8 @@ void hstrlenCommand(client *c) { ...@@ -2330,7 +2272,8 @@ void hstrlenCommand(client *c) {
if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.czero)) == NULL || if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.czero)) == NULL ||
checkType(c,o,OBJ_HASH)) return; checkType(c,o,OBJ_HASH)) return;
GetFieldRes res = hashTypeGetValue(c->db, o, c->argv[2]->ptr, &vstr, &vlen, &vll); GetFieldRes res = hashTypeGetValue(c->db, o, c->argv[2]->ptr, &vstr, &vlen, &vll,
HFE_LAZY_EXPIRE);
if (res == GETF_NOT_FOUND || res == GETF_EXPIRED || res == GETF_EXPIRED_HASH) { if (res == GETF_NOT_FOUND || res == GETF_EXPIRED || res == GETF_EXPIRED_HASH) {
addReply(c, shared.czero); addReply(c, shared.czero);
...@@ -2421,11 +2364,11 @@ void hgetallCommand(client *c) { ...@@ -2421,11 +2364,11 @@ void hgetallCommand(client *c) {
void hexistsCommand(client *c) { void hexistsCommand(client *c) {
robj *o; robj *o;
int isHashDeleted;
if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.czero)) == NULL || if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.czero)) == NULL ||
checkType(c,o,OBJ_HASH)) return; checkType(c,o,OBJ_HASH)) return;
addReply(c,hashTypeExists(c->db,o,c->argv[2]->ptr,&isHashDeleted) ? shared.cone : shared.czero); addReply(c,hashTypeExists(c->db,o,c->argv[2]->ptr,HFE_LAZY_EXPIRE, NULL) ?
shared.cone : shared.czero);
} }
void hscanCommand(client *c) { void hscanCommand(client *c) {
...@@ -2847,6 +2790,7 @@ static ExpireMeta *hashGetExpireMeta(const eItem hash) { ...@@ -2847,6 +2790,7 @@ static ExpireMeta *hashGetExpireMeta(const eItem hash) {
serverPanic("Unknown encoding: %d", hashObj->encoding); serverPanic("Unknown encoding: %d", hashObj->encoding);
} }
} }
/* HTTL key <FIELDS count field [field ...]> */ /* HTTL key <FIELDS count field [field ...]> */
static void httlGenericCommand(client *c, const char *cmd, long long basetime, int unit) { static void httlGenericCommand(client *c, const char *cmd, long long basetime, int unit) {
UNUSED(cmd); UNUSED(cmd);
...@@ -3032,16 +2976,12 @@ static void hexpireGenericCommand(client *c, const char *cmd, long long basetime ...@@ -3032,16 +2976,12 @@ static void hexpireGenericCommand(client *c, const char *cmd, long long basetime
} }
HashTypeSetEx exCtx; HashTypeSetEx exCtx;
hashTypeSetExInit(keyArg, hashObj, c, c->db, cmd, hashTypeSetExInit(keyArg, hashObj, c, c->db, cmd, expireSetCond, &exCtx);
FIELD_DONT_CREATE2,
expireSetCond,
&exCtx);
addReplyArrayLen(c, numFields); addReplyArrayLen(c, numFields);
for (int i = 0 ; i < numFields ; i++) { for (int i = 0 ; i < numFields ; i++) {
sds field = c->argv[numFieldsAt+i+1]->ptr; sds field = c->argv[numFieldsAt+i+1]->ptr;
SetExRes res = hashTypeSetEx(c->db, hashObj, field, NULL, expire, &exCtx); SetExRes res = hashTypeSetEx(hashObj, field, expire, &exCtx);
addReplyLongLong(c,res); addReplyLongLong(c,res);
} }
hashTypeSetExDone(&exCtx); hashTypeSetExDone(&exCtx);
...@@ -3216,5 +3156,9 @@ void hpersistCommand(client *c) { ...@@ -3216,5 +3156,9 @@ void hpersistCommand(client *c) {
/* Generates a hpersist event if the expiry time associated with any field /* Generates a hpersist event if the expiry time associated with any field
* has been successfully deleted. */ * has been successfully deleted. */
if (changed) notifyKeyspaceEvent(NOTIFY_HASH,"hpersist",c->argv[1],c->db->id); if (changed) {
notifyKeyspaceEvent(NOTIFY_HASH, "hpersist", c->argv[1], c->db->id);
signalModifiedKey(c, c->db, c->argv[1]);
server.dirty++;
}
} }
...@@ -21,6 +21,45 @@ start_server {tags {"modules"}} { ...@@ -21,6 +21,45 @@ start_server {tags {"modules"}} {
r hgetall k r hgetall k
} {squirrel ofcourse banana no what nothing something nice} } {squirrel ofcourse banana no what nothing something nice}
test {Module hash - set (override) NX expired field successfully} {
r debug set-active-expire 0
r del H1 H2
r hash.set H1 "n" f1 v1
r hpexpire H1 1 FIELDS 1 f1
r hash.set H2 "n" f1 v1 f2 v2
r hpexpire H2 1 FIELDS 1 f1
after 5
assert_equal 0 [r hash.set H1 "n" f1 xx]
assert_equal "f1 xx" [r hgetall H1]
assert_equal 0 [r hash.set H2 "n" f1 yy]
assert_equal "f1 f2 v2 yy" [lsort [r hgetall H2]]
r debug set-active-expire 1
} {OK} {needs:debug}
test {Module hash - set XX of expired field gets failed as expected} {
r debug set-active-expire 0
r del H1 H2
r hash.set H1 "n" f1 v1
r hpexpire H1 1 FIELDS 1 f1
r hash.set H2 "n" f1 v1 f2 v2
r hpexpire H2 1 FIELDS 1 f1
after 5
# expected to fail on condition XX. hgetall should return empty list
r hash.set H1 "x" f1 xx
assert_equal "" [lsort [r hgetall H1]]
# But expired field was not lazy deleted
assert_equal 1 [r hlen H1]
# expected to fail on condition XX. hgetall should return list without expired f1
r hash.set H2 "x" f1 yy
assert_equal "f2 v2" [lsort [r hgetall H2]]
# But expired field was not lazy deleted
assert_equal 2 [r hlen H2]
r debug set-active-expire 1
} {OK} {needs:debug}
test "Unload the module - hash" { test "Unload the module - hash" {
assert_equal {OK} [r module unload hash] assert_equal {OK} [r module unload hash]
} }
......
...@@ -25,12 +25,16 @@ start_server {tags {"modules"}} { ...@@ -25,12 +25,16 @@ start_server {tags {"modules"}} {
} {{f1 1}} } {{f1 1}}
test {Module scan hash listpack with hexpire} { test {Module scan hash listpack with hexpire} {
r hmset hh f1 v1 f2 v2 r debug set-active-expire 0
r hmset hh f1 v1 f2 v2 f3 v3
r hexpire hh 100000 fields 1 f1 r hexpire hh 100000 fields 1 f1
r hpexpire hh 1 fields 1 f3
after 10
assert_range [r httl hh fields 1 f1] 10000 100000 assert_range [r httl hh fields 1 f1] 10000 100000
assert_encoding listpackex hh assert_encoding listpackex hh
r debug set-active-expire 1
lsort [r scan.scan_key hh] lsort [r scan.scan_key hh]
} {{f1 v1} {f2 v2}} } {{f1 v1} {f2 v2}} {needs:debug}
test {Module scan hash dict} { test {Module scan hash dict} {
r config set hash-max-ziplist-entries 2 r config set hash-max-ziplist-entries 2
...@@ -44,10 +48,22 @@ start_server {tags {"modules"}} { ...@@ -44,10 +48,22 @@ start_server {tags {"modules"}} {
r del hh r del hh
r hmset hh f1 v1 f2 v2 f3 v3 r hmset hh f1 v1 f2 v2 f3 v3
r hexpire hh 100000 fields 1 f2 r hexpire hh 100000 fields 1 f2
r hpexpire hh 5 fields 1 f3
assert_range [r httl hh fields 1 f2] 10000 100000 assert_range [r httl hh fields 1 f2] 10000 100000
assert_encoding hashtable hh assert_encoding hashtable hh
after 10
lsort [r scan.scan_key hh] lsort [r scan.scan_key hh]
} {{f1 v1} {f2 v2} {f3 v3}} } {{f1 v1} {f2 v2}}
test {Module scan hash with hexpire can return no items} {
r del hh
r debug set-active-expire 0
r hmset hh f1 v1 f2 v2 f3 v3
r hpexpire hh 1 fields 3 f1 f2 f3
after 10
r debug set-active-expire 1
lsort [r scan.scan_key hh]
} {} {needs:debug}
test {Module scan zset listpack} { test {Module scan zset listpack} {
r zadd zz 1 f1 2 f2 r zadd zz 1 f1 2 f2
......
...@@ -1103,8 +1103,11 @@ start_server {tags {"external:skip needs:debug"}} { ...@@ -1103,8 +1103,11 @@ start_server {tags {"external:skip needs:debug"}} {
r hexpireat h1 [expr [clock seconds]+100] NX FIELDS 1 f1 r hexpireat h1 [expr [clock seconds]+100] NX FIELDS 1 f1
r hset h2 f2 v2 r hset h2 f2 v2
r hpexpireat h2 [expr [clock seconds]*1000+100000] NX FIELDS 1 f2 r hpexpireat h2 [expr [clock seconds]*1000+100000] NX FIELDS 1 f2
r hset h3 f3 v3 f4 v4 r hset h3 f3 v3 f4 v4 f5 v5
# hpersist does nothing here. Verify it is not propagated.
r hpersist h3 FIELDS 1 f5
r hexpire h3 100 FIELDS 3 f3 f4 non_exists_field r hexpire h3 100 FIELDS 3 f3 f4 non_exists_field
r hpersist h3 FIELDS 1 f3
assert_replication_stream $repl { assert_replication_stream $repl {
{select *} {select *}
...@@ -1112,8 +1115,9 @@ start_server {tags {"external:skip needs:debug"}} { ...@@ -1112,8 +1115,9 @@ start_server {tags {"external:skip needs:debug"}} {
{hpexpireat h1 * NX FIELDS 1 f1} {hpexpireat h1 * NX FIELDS 1 f1}
{hset h2 f2 v2} {hset h2 f2 v2}
{hpexpireat h2 * NX FIELDS 1 f2} {hpexpireat h2 * NX FIELDS 1 f2}
{hset h3 f3 v3 f4 v4} {hset h3 f3 v3 f4 v4 f5 v5}
{hpexpireat h3 * FIELDS 3 f3 f4 non_exists_field} {hpexpireat h3 * FIELDS 3 f3 f4 non_exists_field}
{hpersist h3 FIELDS 1 f3}
} }
close_replication_stream $repl close_replication_stream $repl
} {} {needs:repl} } {} {needs:repl}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment