Unverified Commit ca4ed48d authored by Ozan Tezcan's avatar Ozan Tezcan Committed by GitHub
Browse files

Add listpack support, hgetf and hsetf commands (#13209)

**Changes:**
- Adds listpack support to hash field expiration 
- Implements hgetf/hsetf commands

**Listpack support for hash field expiration**

We keep field name and value pairs in listpack for the hash type. With
this PR, if one of hash field expiration command is called on the key
for the first time, it converts listpack layout to triplets to hold
field name, value and ttl per field. If a field does not have a TTL, we
store zero as the ttl value. Zero is encoded as two bytes in the
listpack. So, once we convert listpack to hold triplets, for the fields
that don't have a TTL, it will be consuming those extra 2 bytes per
item. Fields are ordered by ttl in the listpack to find the field with
minimum expiry time efficiently.

**New command implementations as part of this PR:** 

- HGETF command

For each specified field get its value and optionally set the field's
expiration time in sec/msec /unix-sec/unix-msec:
  ```
  HGETF key 
    [NX | XX | GT | LT]
[EX seconds | PX milliseconds | EXAT unix-time-seconds | PXAT
unix-time-milliseconds | PERSIST]
    <FIELDS count field [field ...]>
  ```

- HSETF command

For each specified field value pair: set field to value and optionally
set the field's expiration time in sec/msec /unix-sec/unix-msec:
  ```
  HSETF key 
    [DC] 
    [DCF | DOF] 
    [NX | XX | GT | LT] 
    [GETNEW | GETOLD] 
[EX seconds | PX milliseconds | EXAT unix-time-seconds | PXAT
unix-time-milliseconds | KEEPTTL]
    <FVS count field value [field value …]>
  ```

Todo:
- Performance improvement.
- rdb load/save
- aof
- defrag
parent 13401f8b
......@@ -1944,7 +1944,7 @@ static int rioWriteHashIteratorCursor(rio *r, hashTypeIterator *hi, int what) {
unsigned int vlen = UINT_MAX;
long long vll = LLONG_MAX;
hashTypeCurrentFromListpack(hi, what, &vstr, &vlen, &vll);
hashTypeCurrentFromListpack(hi, what, &vstr, &vlen, &vll, NULL);
if (vstr)
return rioWriteBulkString(r, (char*)vstr, vlen);
else
......
......@@ -3452,6 +3452,52 @@ struct COMMAND_ARG HGETALL_Args[] = {
{MAKE_ARG("key",ARG_TYPE_KEY,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)},
};
/********** HGETF ********************/
#ifndef SKIP_CMD_HISTORY_TABLE
/* HGETF history */
#define HGETF_History NULL
#endif
#ifndef SKIP_CMD_TIPS_TABLE
/* HGETF tips */
#define HGETF_Tips NULL
#endif
#ifndef SKIP_CMD_KEY_SPECS_TABLE
/* HGETF key specs */
keySpec HGETF_Keyspecs[1] = {
{NULL,CMD_KEY_RW|CMD_KEY_UPDATE,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}}
};
#endif
/* HGETF condition argument table */
struct COMMAND_ARG HGETF_condition_Subargs[] = {
{MAKE_ARG("nx",ARG_TYPE_PURE_TOKEN,-1,"NX",NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("xx",ARG_TYPE_PURE_TOKEN,-1,"XX",NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("gt",ARG_TYPE_PURE_TOKEN,-1,"GT",NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("lt",ARG_TYPE_PURE_TOKEN,-1,"LT",NULL,NULL,CMD_ARG_NONE,0,NULL)},
};
/* HGETF expiration argument table */
struct COMMAND_ARG HGETF_expiration_Subargs[] = {
{MAKE_ARG("seconds",ARG_TYPE_INTEGER,-1,"EX",NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("milliseconds",ARG_TYPE_INTEGER,-1,"PX",NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("unix-time-seconds",ARG_TYPE_UNIX_TIME,-1,"EXAT",NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("unix-time-milliseconds",ARG_TYPE_UNIX_TIME,-1,"PXAT",NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("persist",ARG_TYPE_PURE_TOKEN,-1,"PERSIST",NULL,NULL,CMD_ARG_NONE,0,NULL)},
};
/* HGETF argument table */
struct COMMAND_ARG HGETF_Args[] = {
{MAKE_ARG("key",ARG_TYPE_KEY,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("condition",ARG_TYPE_ONEOF,-1,NULL,NULL,NULL,CMD_ARG_OPTIONAL,4,NULL),.subargs=HGETF_condition_Subargs},
{MAKE_ARG("expiration",ARG_TYPE_ONEOF,-1,NULL,NULL,NULL,CMD_ARG_OPTIONAL,5,NULL),.subargs=HGETF_expiration_Subargs},
{MAKE_ARG("fields",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("count",ARG_TYPE_INTEGER,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("field",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_MULTIPLE,0,NULL)},
};
/********** HINCRBY ********************/
#ifndef SKIP_CMD_HISTORY_TABLE
......@@ -3856,6 +3902,73 @@ struct COMMAND_ARG HSET_Args[] = {
{MAKE_ARG("data",ARG_TYPE_BLOCK,-1,NULL,NULL,NULL,CMD_ARG_MULTIPLE,2,NULL),.subargs=HSET_data_Subargs},
};
/********** HSETF ********************/
#ifndef SKIP_CMD_HISTORY_TABLE
/* HSETF history */
#define HSETF_History NULL
#endif
#ifndef SKIP_CMD_TIPS_TABLE
/* HSETF tips */
#define HSETF_Tips NULL
#endif
#ifndef SKIP_CMD_KEY_SPECS_TABLE
/* HSETF key specs */
keySpec HSETF_Keyspecs[1] = {
{NULL,CMD_KEY_RW|CMD_KEY_UPDATE,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}}
};
#endif
/* HSETF create option argument table */
struct COMMAND_ARG HSETF_create_option_Subargs[] = {
{MAKE_ARG("dcf",ARG_TYPE_PURE_TOKEN,-1,"DCF",NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("dof",ARG_TYPE_PURE_TOKEN,-1,"DOF",NULL,NULL,CMD_ARG_NONE,0,NULL)},
};
/* HSETF return option argument table */
struct COMMAND_ARG HSETF_return_option_Subargs[] = {
{MAKE_ARG("getnew",ARG_TYPE_PURE_TOKEN,-1,"GETNEW",NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("getold",ARG_TYPE_PURE_TOKEN,-1,"GETOLD",NULL,NULL,CMD_ARG_NONE,0,NULL)},
};
/* HSETF condition argument table */
struct COMMAND_ARG HSETF_condition_Subargs[] = {
{MAKE_ARG("nx",ARG_TYPE_PURE_TOKEN,-1,"NX",NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("xx",ARG_TYPE_PURE_TOKEN,-1,"XX",NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("gt",ARG_TYPE_PURE_TOKEN,-1,"GT",NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("lt",ARG_TYPE_PURE_TOKEN,-1,"LT",NULL,NULL,CMD_ARG_NONE,0,NULL)},
};
/* HSETF expiration argument table */
struct COMMAND_ARG HSETF_expiration_Subargs[] = {
{MAKE_ARG("seconds",ARG_TYPE_INTEGER,-1,"EX",NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("milliseconds",ARG_TYPE_INTEGER,-1,"PX",NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("unix-time-seconds",ARG_TYPE_UNIX_TIME,-1,"EXAT",NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("unix-time-milliseconds",ARG_TYPE_UNIX_TIME,-1,"PXAT",NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("keepttl",ARG_TYPE_PURE_TOKEN,-1,"KEEPTTL",NULL,NULL,CMD_ARG_NONE,0,NULL)},
};
/* HSETF data argument table */
struct COMMAND_ARG HSETF_data_Subargs[] = {
{MAKE_ARG("field",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("value",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)},
};
/* HSETF argument table */
struct COMMAND_ARG HSETF_Args[] = {
{MAKE_ARG("key",ARG_TYPE_KEY,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("create key option",ARG_TYPE_PURE_TOKEN,-1,"DC",NULL,NULL,CMD_ARG_OPTIONAL,0,NULL)},
{MAKE_ARG("create option",ARG_TYPE_ONEOF,-1,NULL,NULL,NULL,CMD_ARG_OPTIONAL,2,NULL),.subargs=HSETF_create_option_Subargs},
{MAKE_ARG("return option",ARG_TYPE_ONEOF,-1,NULL,NULL,NULL,CMD_ARG_OPTIONAL,2,NULL),.subargs=HSETF_return_option_Subargs},
{MAKE_ARG("condition",ARG_TYPE_ONEOF,-1,NULL,NULL,NULL,CMD_ARG_OPTIONAL,4,NULL),.subargs=HSETF_condition_Subargs},
{MAKE_ARG("expiration",ARG_TYPE_ONEOF,-1,NULL,NULL,NULL,CMD_ARG_OPTIONAL,5,NULL),.subargs=HSETF_expiration_Subargs},
{MAKE_ARG("fvs",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("count",ARG_TYPE_INTEGER,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("data",ARG_TYPE_BLOCK,-1,NULL,NULL,NULL,CMD_ARG_MULTIPLE,2,NULL),.subargs=HSETF_data_Subargs},
};
/********** HSETNX ********************/
#ifndef SKIP_CMD_HISTORY_TABLE
......@@ -10989,6 +11102,7 @@ struct COMMAND_STRUCT redisCommandTable[] = {
{MAKE_CMD("hexpiretime","Returns the expiration time of a hash field as a Unix timestamp, in seconds.","O(N) where N is the number of arguments to the command","8.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HEXPIRETIME_History,0,HEXPIRETIME_Tips,0,hexpiretimeCommand,-4,CMD_READONLY|CMD_FAST,ACL_CATEGORY_HASH,HEXPIRETIME_Keyspecs,1,NULL,3),.args=HEXPIRETIME_Args},
{MAKE_CMD("hget","Returns the value of a field in a hash.","O(1)","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HGET_History,0,HGET_Tips,0,hgetCommand,3,CMD_READONLY|CMD_FAST,ACL_CATEGORY_HASH,HGET_Keyspecs,1,NULL,2),.args=HGET_Args},
{MAKE_CMD("hgetall","Returns all fields and values in a hash.","O(N) where N is the size of the hash.","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HGETALL_History,0,HGETALL_Tips,1,hgetallCommand,2,CMD_READONLY,ACL_CATEGORY_HASH,HGETALL_Keyspecs,1,NULL,1),.args=HGETALL_Args},
{MAKE_CMD("hgetf","For each specified field, returns its value and optionally set the field's remaining expiration time in seconds / milliseconds","O(N) where N is the number of arguments to the command","8.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HGETF_History,0,HGETF_Tips,0,hgetfCommand,-5,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_HASH,HGETF_Keyspecs,1,NULL,6),.args=HGETF_Args},
{MAKE_CMD("hincrby","Increments the integer value of a field in a hash by a number. Uses 0 as initial value if the field doesn't exist.","O(1)","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HINCRBY_History,0,HINCRBY_Tips,0,hincrbyCommand,4,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_HASH,HINCRBY_Keyspecs,1,NULL,3),.args=HINCRBY_Args},
{MAKE_CMD("hincrbyfloat","Increments the floating point value of a field by a number. Uses 0 as initial value if the field doesn't exist.","O(1)","2.6.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HINCRBYFLOAT_History,0,HINCRBYFLOAT_Tips,0,hincrbyfloatCommand,4,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_HASH,HINCRBYFLOAT_Keyspecs,1,NULL,3),.args=HINCRBYFLOAT_Args},
{MAKE_CMD("hkeys","Returns all fields in a hash.","O(N) where N is the size of the hash.","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HKEYS_History,0,HKEYS_Tips,1,hkeysCommand,2,CMD_READONLY,ACL_CATEGORY_HASH,HKEYS_Keyspecs,1,NULL,1),.args=HKEYS_Args},
......@@ -11003,6 +11117,7 @@ struct COMMAND_STRUCT redisCommandTable[] = {
{MAKE_CMD("hrandfield","Returns one or more random fields from a hash.","O(N) where N is the number of fields returned","6.2.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HRANDFIELD_History,0,HRANDFIELD_Tips,1,hrandfieldCommand,-2,CMD_READONLY,ACL_CATEGORY_HASH,HRANDFIELD_Keyspecs,1,NULL,2),.args=HRANDFIELD_Args},
{MAKE_CMD("hscan","Iterates over fields and values of a hash.","O(1) for every call. O(N) for a complete iteration, including enough command calls for the cursor to return back to 0. N is the number of elements inside the collection.","2.8.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HSCAN_History,0,HSCAN_Tips,1,hscanCommand,-3,CMD_READONLY,ACL_CATEGORY_HASH,HSCAN_Keyspecs,1,NULL,5),.args=HSCAN_Args},
{MAKE_CMD("hset","Creates or modifies the value of a field in a hash.","O(1) for each field/value pair added, so O(N) to add N field/value pairs when the command is called with multiple field/value pairs.","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HSET_History,1,HSET_Tips,0,hsetCommand,-4,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_HASH,HSET_Keyspecs,1,NULL,2),.args=HSET_Args},
{MAKE_CMD("hsetf","For each specified field, returns its value and optionally set the field's remaining expiration time in seconds / milliseconds","O(N) where N is the number of arguments to the command","8.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HSETF_History,0,HSETF_Tips,0,hsetfCommand,-6,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_HASH,HSETF_Keyspecs,1,NULL,9),.args=HSETF_Args},
{MAKE_CMD("hsetnx","Sets the value of a field in a hash only when the field doesn't exist.","O(1)","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HSETNX_History,0,HSETNX_Tips,0,hsetnxCommand,4,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_HASH,HSETNX_Keyspecs,1,NULL,3),.args=HSETNX_Args},
{MAKE_CMD("hstrlen","Returns the length of the value of a field.","O(1)","3.2.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HSTRLEN_History,0,HSTRLEN_Tips,0,hstrlenCommand,3,CMD_READONLY|CMD_FAST,ACL_CATEGORY_HASH,HSTRLEN_Keyspecs,1,NULL,2),.args=HSTRLEN_Args},
{MAKE_CMD("httl","Returns the TTL in seconds of a hash field.","O(N) where N is the number of arguments to the command","8.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HTTL_History,0,HTTL_Tips,0,httlCommand,-4,CMD_READONLY|CMD_FAST,ACL_CATEGORY_HASH,HTTL_Keyspecs,1,NULL,3),.args=HTTL_Args},
......
{
"HGETF": {
"summary": "For each specified field, returns its value and optionally set the field's remaining expiration time in seconds / milliseconds",
"complexity": "O(N) where N is the number of arguments to the command",
"group": "hash",
"since": "8.0.0",
"arity": -5,
"function": "hgetfCommand",
"history": [],
"command_flags": [
"WRITE",
"DENYOOM",
"FAST"
],
"acl_categories": [
"HASH"
],
"key_specs": [
{
"flags": [
"RW",
"UPDATE"
],
"begin_search": {
"index": {
"pos": 1
}
},
"find_keys": {
"range": {
"lastkey": 0,
"step": 1,
"limit": 0
}
}
}
],
"reply_schema": {
"oneOf": [
{
"description": "Key does not exist.",
"type": "null"
},
{
"description": "Array of results",
"type": "array",
"minItems": 1,
"maxItems": 4294967295,
"items": {
"description": "Field value",
"type": "string"
}
}
]
},
"arguments": [
{
"name": "key",
"type": "key",
"key_spec_index": 0
},
{
"name": "condition",
"type": "oneof",
"optional": true,
"arguments": [
{
"name": "nx",
"type": "pure-token",
"token": "NX"
},
{
"name": "xx",
"type": "pure-token",
"token": "XX"
},
{
"name": "gt",
"type": "pure-token",
"token": "GT"
},
{
"name": "lt",
"type": "pure-token",
"token": "LT"
}
]
},
{
"name": "expiration",
"type": "oneof",
"optional": true,
"arguments": [
{
"name": "seconds",
"type": "integer",
"token": "EX"
},
{
"name": "milliseconds",
"type": "integer",
"token": "PX"
},
{
"name": "unix-time-seconds",
"type": "unix-time",
"token": "EXAT"
},
{
"name": "unix-time-milliseconds",
"type": "unix-time",
"token": "PXAT"
},
{
"name": "persist",
"type": "pure-token",
"token": "PERSIST"
}
]
},
{
"name": "FIELDS",
"type": "string"
},
{
"name": "count",
"type": "integer"
},
{
"name": "field",
"type": "string",
"multiple": true
}
]
}
}
{
"HSETF": {
"summary": "For each specified field, returns its value and optionally set the field's remaining expiration time in seconds / milliseconds",
"complexity": "O(N) where N is the number of arguments to the command",
"group": "hash",
"since": "8.0.0",
"arity": -6,
"function": "hsetfCommand",
"history": [],
"command_flags": [
"WRITE",
"DENYOOM",
"FAST"
],
"acl_categories": [
"HASH"
],
"key_specs": [
{
"flags": [
"RW",
"UPDATE"
],
"begin_search": {
"index": {
"pos": 1
}
},
"find_keys": {
"range": {
"lastkey": 0,
"step": 1,
"limit": 0
}
}
}
],
"reply_schema": {
"oneOf": [
{
"description": "Key does not exist and DC condition was given",
"type": "null"
},
{
"description": "Array of field values when GETNEW/GETOLD is used",
"type": "array",
"minItems": 1,
"maxItems": 4294967295,
"items": {
"oneOf": [
{
"description": "Field value",
"type": "string"
},
{
"description": "Field does not exist and couldn't create it (DCF not met)",
"type": "null"
}
]
}
},
{
"description": "Array of results",
"type": "array",
"minItems": 1,
"maxItems": 4294967295,
"items": {
"oneOf": [
{
"description": "Cannot set field value. DOC/DOF condition not met.",
"const": 0
},
{
"description": "Set field value without updating TTL",
"const": 1
},
{
"description": "Set field value and updated TTL",
"const": 3
}
]
}
}
]
},
"arguments": [
{
"name": "key",
"type": "key",
"key_spec_index": 0
},
{
"name": "create key option",
"optional": true,
"type": "pure-token",
"token": "DC"
},
{
"name": "create option",
"type": "oneof",
"optional": true,
"arguments": [
{
"name": "dcf",
"type": "pure-token",
"token": "DCF"
},
{
"name": "dof",
"type": "pure-token",
"token": "DOF"
}
]
},
{
"name": "return option",
"type": "oneof",
"optional": true,
"arguments": [
{
"name": "getnew",
"type": "pure-token",
"token": "GETNEW"
},
{
"name": "getold",
"type": "pure-token",
"token": "GETOLD"
}
]
},
{
"name": "condition",
"type": "oneof",
"optional": true,
"arguments": [
{
"name": "nx",
"type": "pure-token",
"token": "NX"
},
{
"name": "xx",
"type": "pure-token",
"token": "XX"
},
{
"name": "gt",
"type": "pure-token",
"token": "GT"
},
{
"name": "lt",
"type": "pure-token",
"token": "LT"
}
]
},
{
"name": "expiration",
"type": "oneof",
"optional": true,
"arguments": [
{
"name": "seconds",
"type": "integer",
"token": "EX"
},
{
"name": "milliseconds",
"type": "integer",
"token": "PX"
},
{
"name": "unix-time-seconds",
"type": "unix-time",
"token": "EXAT"
},
{
"name": "unix-time-milliseconds",
"type": "unix-time",
"token": "PXAT"
},
{
"name": "keepttl",
"type": "pure-token",
"token": "KEEPTTL"
}
]
},
{
"name": "FVS",
"type": "string"
},
{
"name": "count",
"type": "integer"
},
{
"name": "data",
"type": "block",
"multiple": true,
"arguments": [
{
"name": "field",
"type": "string"
},
{
"name": "value",
"type": "string"
}
]
}
]
}
}
......@@ -1233,6 +1233,40 @@ void scanGenericCommand(client *c, robj *o, unsigned long long cursor) {
p = lpNext(o->ptr, p);
}
cursor = 0;
} else if (o->type == OBJ_HASH && o->encoding == OBJ_ENCODING_LISTPACK_EX) {
int64_t len;
long long expire_at;
unsigned char *lp = hashTypeListpackGetLp(o);
unsigned char *p = lpFirst(lp);
unsigned char *str, *val;
unsigned char intbuf[LP_INTBUF_SIZE];
while (p) {
str = lpGet(p, &len, intbuf);
p = lpNext(lp, p);
val = p; /* Keep pointer to value */
p = lpNext(lp, p);
serverAssert(!lpGetValue(p, NULL, &expire_at));
if (hashTypeIsExpired(o, expire_at) ||
(use_pattern && !stringmatchlen(pat, sdslen(pat), (char *)str, len, 0)))
{
/* jump to the next key/val pair */
p = lpNext(lp, p);
continue;
}
/* add key object */
listAddNodeTail(keys, sdsnewlen(str, len));
/* add value object */
if (!no_values) {
str = lpGet(val, &len, intbuf);
listAddNodeTail(keys, sdsnewlen(str, len));
}
p = lpNext(lp, p);
}
cursor = 0;
} else {
serverPanic("Not handled encoding in SCAN.");
}
......@@ -1393,7 +1427,7 @@ void renameGenericCommand(client *c, int nx) {
/* If hash with expiration on fields then remove it from global HFE DS and
* keep next expiration time. Otherwise, dbDelete() will remove it from the
* global HFE DS and we will lose the expiration time. */
if (o->type == OBJ_HASH && o->encoding == OBJ_ENCODING_HT)
if (o->type == OBJ_HASH)
minHashExpireTime = hashTypeRemoveFromExpires(&c->db->hexpires, o);
dbDelete(c->db,c->argv[1]);
......@@ -1472,7 +1506,7 @@ void moveCommand(client *c) {
/* If hash with expiration on fields, remove it from global HFE DS and keep
* aside registered expiration time. Must be before deletion of the object.
* hexpires (ebuckets) embed in stored items its structure. */
if (o->type == OBJ_HASH && o->encoding == OBJ_ENCODING_HT)
if (o->type == OBJ_HASH)
hashExpireTime = hashTypeRemoveFromExpires(&src->hexpires, o);
incrRefCount(o);
......
......@@ -664,10 +664,14 @@ NULL
if ((o = objectCommandLookupOrReply(c,c->argv[2],shared.nokeyerr))
== NULL) return;
if (o->encoding != OBJ_ENCODING_LISTPACK) {
if (o->encoding != OBJ_ENCODING_LISTPACK && o->encoding != OBJ_ENCODING_LISTPACK_EX) {
addReplyError(c,"Not a listpack encoded object.");
} else {
lpRepr(o->ptr);
if (o->encoding == OBJ_ENCODING_LISTPACK)
lpRepr(o->ptr);
else if (o->encoding == OBJ_ENCODING_LISTPACK_EX)
lpRepr(((listpackEx*)o->ptr)->lp);
addReplyStatus(c,"Listpack structure printed on stdout");
}
} else if (!strcasecmp(c->argv[1]->ptr,"quicklist") && (c->argc == 3 || c->argc == 4)) {
......
......@@ -1846,7 +1846,7 @@ ExpireAction expireItemCb(eItem item, void *ctx) {
}
ExpireAction expireUpdateThirdItemCb(eItem item, void *ctx) {
uint64_t expTime = (uint64_t) ctx;
uint64_t expTime = (uint64_t) (uintptr_t) ctx;
static int calls = 0;
if ((calls++) == 3) {
ebSetMetaExpTime(&(((MyItem *)item)->mexpire), expTime );
......
......@@ -1408,15 +1408,20 @@ static inline void lpSaveValue(unsigned char *val, unsigned int len, int64_t lva
/* Randomly select a pair of key and value.
* total_count is a pre-computed length/2 of the listpack (to avoid calls to lpLength)
* 'key' and 'val' are used to store the result key value pair.
* 'val' can be NULL if the value is not needed. */
void lpRandomPair(unsigned char *lp, unsigned long total_count, listpackEntry *key, listpackEntry *val) {
* 'val' can be NULL if the value is not needed.
* 'tuple_len' indicates entry count of a single logical item. It should be 2
* if listpack was saved as key-value pair or more for key-value-...(n_entries). */
void lpRandomPair(unsigned char *lp, unsigned long total_count,
listpackEntry *key, listpackEntry *val, int tuple_len)
{
unsigned char *p;
assert(tuple_len >= 2);
/* Avoid div by zero on corrupt listpack */
assert(total_count);
/* Generate even numbers, because listpack saved K-V pair */
int r = (rand() % total_count) * 2;
int r = (rand() % total_count) * tuple_len;
assert((p = lpSeek(lp, r)));
key->sval = lpGetValue(p, &(key->slen), &(key->lval));
......@@ -1466,26 +1471,31 @@ void lpRandomEntries(unsigned char *lp, unsigned int count, listpackEntry *entri
/* Randomly select count of key value pairs and store into 'keys' and
* 'vals' args. The order of the picked entries is random, and the selections
* are non-unique (repetitions are possible).
* The 'vals' arg can be NULL in which case we skip these. */
void lpRandomPairs(unsigned char *lp, unsigned int count, listpackEntry *keys, listpackEntry *vals) {
* The 'vals' arg can be NULL in which case we skip these.
* 'tuple_len' indicates entry count of a single logical item. It should be 2
* if listpack was saved as key-value pair or more for key-value-...(n_entries). */
void lpRandomPairs(unsigned char *lp, unsigned int count, listpackEntry *keys, listpackEntry *vals, int tuple_len) {
unsigned char *p, *key, *value;
unsigned int klen = 0, vlen = 0;
long long klval = 0, vlval = 0;
assert(tuple_len >= 2);
/* Notice: the index member must be first due to the use in uintCompare */
typedef struct {
unsigned int index;
unsigned int order;
} rand_pick;
rand_pick *picks = lp_malloc(sizeof(rand_pick)*count);
unsigned int total_size = lpLength(lp)/2;
unsigned int total_size = lpLength(lp)/tuple_len;
/* Avoid div by zero on corrupt listpack */
assert(total_size);
/* create a pool of random indexes (some may be duplicate). */
for (unsigned int i = 0; i < count; i++) {
picks[i].index = (rand() % total_size) * 2; /* Generate even indexes */
/* Generate indexes that key exist at */
picks[i].index = (rand() % total_size) * tuple_len;
/* keep track of the order we picked them */
picks[i].order = i;
}
......@@ -1507,8 +1517,11 @@ void lpRandomPairs(unsigned char *lp, unsigned int count, listpackEntry *keys, l
lpSaveValue(value, vlen, vlval, &vals[storeorder]);
pickindex++;
}
lpindex += 2;
p = lpNext(lp, p);
lpindex += tuple_len;
for (int i = 0; i < tuple_len - 1; i++) {
p = lpNext(lp, p);
}
}
lp_free(picks);
......@@ -1518,13 +1531,20 @@ void lpRandomPairs(unsigned char *lp, unsigned int count, listpackEntry *keys, l
* 'vals' args. The selections are unique (no repetitions), and the order of
* the picked entries is NOT-random.
* The 'vals' arg can be NULL in which case we skip these.
* 'tuple_len' indicates entry count of a single logical item. It should be 2
* if listpack was saved as key-value pair or more for key-value-...(n_entries).
* The return value is the number of items picked which can be lower than the
* requested count if the listpack doesn't hold enough pairs. */
unsigned int lpRandomPairsUnique(unsigned char *lp, unsigned int count, listpackEntry *keys, listpackEntry *vals) {
unsigned int lpRandomPairsUnique(unsigned char *lp, unsigned int count,
listpackEntry *keys, listpackEntry *vals,
int tuple_len)
{
assert(tuple_len >= 2);
unsigned char *p, *key;
unsigned int klen = 0;
long long klval = 0;
unsigned int total_size = lpLength(lp)/2;
unsigned int total_size = lpLength(lp)/tuple_len;
unsigned int index = 0;
if (count > total_size)
count = total_size;
......@@ -1532,7 +1552,7 @@ unsigned int lpRandomPairsUnique(unsigned char *lp, unsigned int count, listpack
p = lpFirst(lp);
unsigned int picked = 0, remaining = count;
while (picked < count && p) {
assert((p = lpNextRandom(lp, p, &index, remaining, 1)));
assert((p = lpNextRandom(lp, p, &index, remaining, tuple_len)));
key = lpGetValue(p, &klen, &klval);
lpSaveValue(key, klen, klval, &keys[picked]);
assert((p = lpNext(lp, p)));
......@@ -1554,8 +1574,9 @@ unsigned int lpRandomPairsUnique(unsigned char *lp, unsigned int count, listpack
* the end of the list. The 'index' needs to be initialized according to the
* current zero-based index matching the position of the starting element 'p'
* and is updated to match the returned element's zero-based index. If
* 'even_only' is nonzero, an element with an even index is picked, which is
* useful if the listpack represents a key-value pair sequence.
* 'tuple_len' indicates entry count of a single logical item. e.g. This is
* useful if listpack represents key-value pairs. In this case, tuple_len should
* be two and even indexes will be picked.
*
* Note that this function can return p. In order to skip the previously
* returned element, you need to call lpNext() or lpDelete() after each call to
......@@ -1565,7 +1586,7 @@ unsigned int lpRandomPairsUnique(unsigned char *lp, unsigned int count, listpack
* p = lpFirst(lp);
* i = 0;
* while (remaining > 0) {
* p = lpNextRandom(lp, p, &i, remaining--, 0);
* p = lpNextRandom(lp, p, &i, remaining--, 1);
*
* // ... Do stuff with p ...
*
......@@ -1574,8 +1595,9 @@ unsigned int lpRandomPairsUnique(unsigned char *lp, unsigned int count, listpack
* }
*/
unsigned char *lpNextRandom(unsigned char *lp, unsigned char *p, unsigned int *index,
unsigned int remaining, int even_only)
unsigned int remaining, int tuple_len)
{
assert(tuple_len > 0);
/* To only iterate once, every time we try to pick a member, the probability
* we pick it is the quotient of the count left we want to pick and the
* count still we haven't visited. This way, we could make every member be
......@@ -1583,15 +1605,14 @@ unsigned char *lpNextRandom(unsigned char *lp, unsigned char *p, unsigned int *i
unsigned int i = *index;
unsigned int total_size = lpLength(lp);
while (i < total_size && p != NULL) {
if (even_only && i % 2 != 0) {
if (i % tuple_len != 0) {
p = lpNext(lp, p);
i++;
continue;
}
/* Do we pick this element? */
unsigned int available = total_size - i;
if (even_only) available /= 2;
unsigned int available = (total_size - i) / tuple_len;
double randomDouble = ((double)rand()) / RAND_MAX;
double threshold = ((double)remaining) / available;
if (randomDouble <= threshold) {
......@@ -2210,7 +2231,7 @@ int listpackTest(int argc, char *argv[], int flags) {
unsigned index = 0;
while (remaining > 0) {
assert(p != NULL);
p = lpNextRandom(lp, p, &index, remaining--, 0);
p = lpNextRandom(lp, p, &index, remaining--, 1);
assert(p != NULL);
assert(p != prev);
prev = p;
......@@ -2226,7 +2247,7 @@ int listpackTest(int argc, char *argv[], int flags) {
unsigned i = 0;
/* Pick from empty listpack returns NULL. */
assert(lpNextRandom(lp, NULL, &i, 2, 0) == NULL);
assert(lpNextRandom(lp, NULL, &i, 2, 1) == NULL);
/* Add some elements and find their pointers within the listpack. */
lp = lpAppend(lp, (unsigned char *)"abc", 3);
......@@ -2239,19 +2260,19 @@ int listpackTest(int argc, char *argv[], int flags) {
assert(lpNext(lp, p2) == NULL);
/* Pick zero elements returns NULL. */
i = 0; assert(lpNextRandom(lp, lpFirst(lp), &i, 0, 0) == NULL);
i = 0; assert(lpNextRandom(lp, lpFirst(lp), &i, 0, 1) == NULL);
/* Pick all returns all. */
i = 0; assert(lpNextRandom(lp, p0, &i, 3, 0) == p0 && i == 0);
i = 1; assert(lpNextRandom(lp, p1, &i, 2, 0) == p1 && i == 1);
i = 2; assert(lpNextRandom(lp, p2, &i, 1, 0) == p2 && i == 2);
i = 0; assert(lpNextRandom(lp, p0, &i, 3, 1) == p0 && i == 0);
i = 1; assert(lpNextRandom(lp, p1, &i, 2, 1) == p1 && i == 1);
i = 2; assert(lpNextRandom(lp, p2, &i, 1, 1) == p2 && i == 2);
/* Pick more than one when there's only one left returns the last one. */
i = 2; assert(lpNextRandom(lp, p2, &i, 42, 0) == p2 && i == 2);
i = 2; assert(lpNextRandom(lp, p2, &i, 42, 1) == p2 && i == 2);
/* Pick all even elements returns p0 and p2. */
i = 0; assert(lpNextRandom(lp, p0, &i, 10, 1) == p0 && i == 0);
i = 1; assert(lpNextRandom(lp, p1, &i, 10, 1) == p2 && i == 2);
i = 0; assert(lpNextRandom(lp, p0, &i, 10, 2) == p0 && i == 0);
i = 1; assert(lpNextRandom(lp, p1, &i, 10, 2) == p2 && i == 2);
/* Don't crash even for bad index. */
for (int j = 0; j < 100; j++) {
......@@ -2264,7 +2285,7 @@ int listpackTest(int argc, char *argv[], int flags) {
}
i = j % 7;
unsigned int remaining = j % 5;
p = lpNextRandom(lp, p, &i, remaining, 0);
p = lpNextRandom(lp, p, &i, remaining, 1);
assert(p == p0 || p == p1 || p == p2 || p == NULL);
}
lpFree(lp);
......@@ -2275,7 +2296,7 @@ int listpackTest(int argc, char *argv[], int flags) {
unsigned char *lp = lpNew(0);
lp = lpAppend(lp, (unsigned char*)"abc", 3);
lp = lpAppend(lp, (unsigned char*)"123", 3);
lpRandomPair(lp, 1, &key, &val);
lpRandomPair(lp, 1, &key, &val, 2);
assert(memcmp(key.sval, "abc", key.slen) == 0);
assert(val.lval == 123);
lpFree(lp);
......@@ -2288,7 +2309,7 @@ int listpackTest(int argc, char *argv[], int flags) {
lp = lpAppend(lp, (unsigned char*)"123", 3);
lp = lpAppend(lp, (unsigned char*)"456", 3);
lp = lpAppend(lp, (unsigned char*)"def", 3);
lpRandomPair(lp, 2, &key, &val);
lpRandomPair(lp, 2, &key, &val, 2);
if (key.sval) {
assert(!memcmp(key.sval, "abc", key.slen));
assert(key.slen == 3);
......@@ -2301,6 +2322,42 @@ int listpackTest(int argc, char *argv[], int flags) {
lpFree(lp);
}
TEST("Random pair with tuple_len 3") {
listpackEntry key, val;
unsigned char *lp = lpNew(0);
lp = lpAppend(lp, (unsigned char*)"abc", 3);
lp = lpAppend(lp, (unsigned char*)"123", 3);
lp = lpAppend(lp, (unsigned char*)"xxx", 3);
lp = lpAppend(lp, (unsigned char*)"456", 3);
lp = lpAppend(lp, (unsigned char*)"def", 3);
lp = lpAppend(lp, (unsigned char*)"xxx", 3);
lp = lpAppend(lp, (unsigned char*)"281474976710655", 15);
lp = lpAppend(lp, (unsigned char*)"789", 3);
lp = lpAppend(lp, (unsigned char*)"xxx", 3);
for (int i = 0; i < 5; i++) {
lpRandomPair(lp, 3, &key, &val, 3);
if (key.sval) {
if (!memcmp(key.sval, "abc", key.slen)) {
assert(key.slen == 3);
assert(val.lval == 123);
} else {
assert(0);
};
}
if (!key.sval) {
if (key.lval == 456)
assert(!memcmp(val.sval, "def", val.slen));
else if (key.lval == 281474976710655LL)
assert(val.lval == 789);
else
assert(0);
}
}
lpFree(lp);
}
TEST("Random pairs with one element") {
int count = 5;
unsigned char *lp = lpNew(0);
......@@ -2309,7 +2366,7 @@ int listpackTest(int argc, char *argv[], int flags) {
lp = lpAppend(lp, (unsigned char*)"abc", 3);
lp = lpAppend(lp, (unsigned char*)"123", 3);
lpRandomPairs(lp, count, keys, vals);
lpRandomPairs(lp, count, keys, vals, 2);
assert(memcmp(keys[4].sval, "abc", keys[4].slen) == 0);
assert(vals[4].lval == 123);
zfree(keys);
......@@ -2327,7 +2384,7 @@ int listpackTest(int argc, char *argv[], int flags) {
lp = lpAppend(lp, (unsigned char*)"123", 3);
lp = lpAppend(lp, (unsigned char*)"456", 3);
lp = lpAppend(lp, (unsigned char*)"def", 3);
lpRandomPairs(lp, count, keys, vals);
lpRandomPairs(lp, count, keys, vals, 2);
for (int i = 0; i < count; i++) {
if (keys[i].sval) {
assert(!memcmp(keys[i].sval, "abc", keys[i].slen));
......@@ -2344,6 +2401,47 @@ int listpackTest(int argc, char *argv[], int flags) {
lpFree(lp);
}
TEST("Random pairs with many elements and tuple_len 3") {
int count = 5;
lp = lpNew(0);
listpackEntry *keys = zcalloc(sizeof(listpackEntry) * count);
listpackEntry *vals = zcalloc(sizeof(listpackEntry) * count);
lp = lpAppend(lp, (unsigned char*)"abc", 3);
lp = lpAppend(lp, (unsigned char*)"123", 3);
lp = lpAppend(lp, (unsigned char*)"xxx", 3);
lp = lpAppend(lp, (unsigned char*)"456", 3);
lp = lpAppend(lp, (unsigned char*)"def", 3);
lp = lpAppend(lp, (unsigned char*)"xxx", 3);
lp = lpAppend(lp, (unsigned char*)"281474976710655", 15);
lp = lpAppend(lp, (unsigned char*)"789", 3);
lp = lpAppend(lp, (unsigned char*)"xxx", 3);
lpRandomPairs(lp, count, keys, vals, 3);
for (int i = 0; i < count; i++) {
if (keys[i].sval) {
if (!memcmp(keys[i].sval, "abc", keys[i].slen)) {
assert(keys[i].slen == 3);
assert(vals[i].lval == 123);
} else {
assert(0);
};
}
if (!keys[i].sval) {
if (keys[i].lval == 456)
assert(!memcmp(vals[i].sval, "def", vals[i].slen));
else if (keys[i].lval == 281474976710655LL)
assert(vals[i].lval == 789);
else
assert(0);
}
}
zfree(keys);
zfree(vals);
lpFree(lp);
}
TEST("Random pairs unique with one element") {
unsigned picked;
int count = 5;
......@@ -2353,7 +2451,7 @@ int listpackTest(int argc, char *argv[], int flags) {
lp = lpAppend(lp, (unsigned char*)"abc", 3);
lp = lpAppend(lp, (unsigned char*)"123", 3);
picked = lpRandomPairsUnique(lp, count, keys, vals);
picked = lpRandomPairsUnique(lp, count, keys, vals, 2);
assert(picked == 1);
assert(memcmp(keys[0].sval, "abc", keys[0].slen) == 0);
assert(vals[0].lval == 123);
......@@ -2373,7 +2471,7 @@ int listpackTest(int argc, char *argv[], int flags) {
lp = lpAppend(lp, (unsigned char*)"123", 3);
lp = lpAppend(lp, (unsigned char*)"456", 3);
lp = lpAppend(lp, (unsigned char*)"def", 3);
picked = lpRandomPairsUnique(lp, count, keys, vals);
picked = lpRandomPairsUnique(lp, count, keys, vals, 2);
assert(picked == 2);
for (int i = 0; i < 2; i++) {
if (keys[i].sval) {
......@@ -2391,6 +2489,47 @@ int listpackTest(int argc, char *argv[], int flags) {
lpFree(lp);
}
TEST("Random pairs unique with many elements and tuple_len 3") {
unsigned picked;
int count = 5;
lp = lpNew(0);
listpackEntry *keys = zmalloc(sizeof(listpackEntry) * count);
listpackEntry *vals = zmalloc(sizeof(listpackEntry) * count);
lp = lpAppend(lp, (unsigned char*)"abc", 3);
lp = lpAppend(lp, (unsigned char*)"123", 3);
lp = lpAppend(lp, (unsigned char*)"xxx", 3);
lp = lpAppend(lp, (unsigned char*)"456", 3);
lp = lpAppend(lp, (unsigned char*)"def", 3);
lp = lpAppend(lp, (unsigned char*)"xxx", 3);
lp = lpAppend(lp, (unsigned char*)"281474976710655", 15);
lp = lpAppend(lp, (unsigned char*)"789", 3);
lp = lpAppend(lp, (unsigned char*)"xxx", 3);
picked = lpRandomPairsUnique(lp, count, keys, vals, 3);
assert(picked == 3);
for (int i = 0; i < 3; i++) {
if (keys[i].sval) {
if (!memcmp(keys[i].sval, "abc", keys[i].slen)) {
assert(keys[i].slen == 3);
assert(vals[i].lval == 123);
} else {
assert(0);
};
}
if (!keys[i].sval) {
if (keys[i].lval == 456)
assert(!memcmp(vals[i].sval, "def", vals[i].slen));
else if (keys[i].lval == 281474976710655LL)
assert(vals[i].lval == 789);
else
assert(0);
}
}
zfree(keys);
zfree(vals);
lpFree(lp);
}
TEST("push various encodings") {
lp = lpNew(0);
......
......@@ -69,12 +69,15 @@ int lpValidateIntegrity(unsigned char *lp, size_t size, int deep,
unsigned char *lpValidateFirst(unsigned char *lp);
int lpValidateNext(unsigned char *lp, unsigned char **pp, size_t lpbytes);
unsigned int lpCompare(unsigned char *p, unsigned char *s, uint32_t slen);
void lpRandomPair(unsigned char *lp, unsigned long total_count, listpackEntry *key, listpackEntry *val);
void lpRandomPairs(unsigned char *lp, unsigned int count, listpackEntry *keys, listpackEntry *vals);
unsigned int lpRandomPairsUnique(unsigned char *lp, unsigned int count, listpackEntry *keys, listpackEntry *vals);
void lpRandomPair(unsigned char *lp, unsigned long total_count,
listpackEntry *key, listpackEntry *val, int tuple_len);
void lpRandomPairs(unsigned char *lp, unsigned int count,
listpackEntry *keys, listpackEntry *vals, int tuple_len);
unsigned int lpRandomPairsUnique(unsigned char *lp, unsigned int count,
listpackEntry *keys, listpackEntry *vals, int tuple_len);
void lpRandomEntries(unsigned char *lp, unsigned int count, listpackEntry *entries);
unsigned char *lpNextRandom(unsigned char *lp, unsigned char *p, unsigned int *index,
unsigned int remaining, int even_only);
unsigned int remaining, int tuple_len);
int lpSafeToAdd(unsigned char* lp, size_t add);
void lpRepr(unsigned char *lp);
......
......@@ -5295,7 +5295,7 @@ int RM_HashSet(RedisModuleKey *key, int flags, ...) {
low_flags |= HASH_SET_TAKE_FIELD;
 
robj *argv[2] = {field,value};
hashTypeTryConversion(key->value,argv,0,1);
hashTypeTryConversion(key->db,key->value,argv,0,1);
int updated = hashTypeSet(key->db, key->value, field->ptr, value->ptr, low_flags);
count += (flags & REDISMODULE_HASH_COUNT_ALL) ? 1 : updated;
 
......
......@@ -333,17 +333,7 @@ void freeZsetObject(robj *o) {
}
void freeHashObject(robj *o) {
switch (o->encoding) {
case OBJ_ENCODING_HT:
dictRelease((dict*) o->ptr);
break;
case OBJ_ENCODING_LISTPACK:
lpFree(o->ptr);
break;
default:
serverPanic("Unknown hash encoding type");
break;
}
hashTypeFree(o);
}
void freeModuleObject(robj *o) {
......@@ -939,6 +929,7 @@ char *strEncoding(int encoding) {
case OBJ_ENCODING_HT: return "hashtable";
case OBJ_ENCODING_QUICKLIST: return "quicklist";
case OBJ_ENCODING_LISTPACK: return "listpack";
case OBJ_ENCODING_LISTPACK_EX: return "listpackex";
case OBJ_ENCODING_INTSET: return "intset";
case OBJ_ENCODING_SKIPLIST: return "skiplist";
case OBJ_ENCODING_EMBSTR: return "embstr";
......@@ -1051,6 +1042,9 @@ size_t objectComputeSize(robj *key, robj *o, size_t sample_size, int dbid) {
} else if (o->type == OBJ_HASH) {
if (o->encoding == OBJ_ENCODING_LISTPACK) {
asize = sizeof(*o)+zmalloc_size(o->ptr);
} else if (o->encoding == OBJ_ENCODING_LISTPACK_EX) {
listpackEx *lpt = o->ptr;
asize = sizeof(*o) + zmalloc_size(lpt) + zmalloc_size(lpt->lp);
} else if (o->encoding == OBJ_ENCODING_HT) {
d = o->ptr;
di = dictGetIterator(d);
......
......@@ -694,6 +694,8 @@ int rdbSaveObjectType(rio *rdb, robj *o) {
case OBJ_HASH:
if (o->encoding == OBJ_ENCODING_LISTPACK)
return rdbSaveType(rdb,RDB_TYPE_HASH_LISTPACK);
else if (o->encoding == OBJ_ENCODING_LISTPACK_EX)
return -1;
else if (o->encoding == OBJ_ENCODING_HT)
return rdbSaveType(rdb,RDB_TYPE_HASH);
else
......@@ -2070,7 +2072,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) {
/* Too many entries? Use a hash table right from the start. */
if (len > server.hash_max_listpack_entries)
hashTypeConvert(o, OBJ_ENCODING_HT);
hashTypeConvert(o, OBJ_ENCODING_HT, NULL);
else if (deep_integrity_validation) {
/* In this mode, we need to guarantee that the server won't crash
* later when the ziplist is converted to a dict.
......@@ -2115,7 +2117,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) {
sdslen(value) > server.hash_max_listpack_value ||
!lpSafeToAdd(o->ptr, hfieldlen(field) + sdslen(value)))
{
hashTypeConvert(o, OBJ_ENCODING_HT);
hashTypeConvert(o, OBJ_ENCODING_HT, NULL);
dictUseStoredKeyApi((dict *)o->ptr, 1);
ret = dictAdd((dict*)o->ptr, field, value);
dictUseStoredKeyApi((dict *)o->ptr, 0);
......@@ -2331,7 +2333,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) {
if (hashTypeLength(o, 0) > server.hash_max_listpack_entries ||
maxlen > server.hash_max_listpack_value)
{
hashTypeConvert(o, OBJ_ENCODING_HT);
hashTypeConvert(o, OBJ_ENCODING_HT, NULL);
}
}
break;
......@@ -2468,7 +2470,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) {
}
if (hashTypeLength(o, 0) > server.hash_max_listpack_entries)
hashTypeConvert(o, OBJ_ENCODING_HT);
hashTypeConvert(o, OBJ_ENCODING_HT, NULL);
else
o->ptr = lpShrinkToFit(o->ptr);
break;
......@@ -2490,7 +2492,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) {
}
if (hashTypeLength(o, 0) > server.hash_max_listpack_entries)
hashTypeConvert(o, OBJ_ENCODING_HT);
hashTypeConvert(o, OBJ_ENCODING_HT, NULL);
break;
default:
/* totally unreachable */
......
......@@ -886,6 +886,7 @@ struct RedisModuleDigest {
#define OBJ_ENCODING_QUICKLIST 9 /* Encoded as linked list of listpacks */
#define OBJ_ENCODING_STREAM 10 /* Encoded as a radix tree of listpacks */
#define OBJ_ENCODING_LISTPACK 11 /* Encoded as a listpack */
#define OBJ_ENCODING_LISTPACK_EX 12 /* Encoded as listpack, extended with metadata */
#define LRU_BITS 24
#define LRU_CLOCK_MAX ((1<<LRU_BITS)-1) /* Max value of obj->lru */
......@@ -2432,7 +2433,8 @@ typedef struct {
robj *subject;
int encoding;
unsigned char *fptr, *vptr;
unsigned char *fptr, *vptr, *tptr;
uint64_t expire_time; /* Only used with OBJ_ENCODING_LISTPACK_EX */
dictIterator *di;
dictEntry *de;
......@@ -3149,13 +3151,27 @@ void setTypeConvert(robj *subject, int enc);
int setTypeConvertAndExpand(robj *setobj, int enc, unsigned long cap, int panic);
robj *setTypeDup(robj *o);
/* Data structure for OBJ_ENCODING_LISTPACK_EX for hash. It contains listpack
* and metadata fields for hash field expiration.*/
typedef struct listpackEx {
ExpireMeta meta; /* To be used in order to register the hash in the
global ebuckets (i.e. db->hexpires) with next,
minimum, hash-field to expire. */
sds key; /* reference to the key, same one that stored in
db->dict. Will be used from active-expiration flow
for notification and deletion of the object, if
needed. */
void *lp; /* listpack that contains 'key-value-ttl' tuples which
are ordered by ttl. */
} listpackEx;
/* Hash data type */
#define HASH_SET_TAKE_FIELD (1<<0)
#define HASH_SET_TAKE_VALUE (1<<1)
#define HASH_SET_COPY 0
void hashTypeConvert(robj *o, int enc);
void hashTypeTryConversion(robj *subject, robj **argv, int start, int end);
void hashTypeConvert(robj *o, int enc, ebuckets *hexpires);
void hashTypeTryConversion(redisDb *db, robj *subject, robj **argv, int start, int end);
int hashTypeExists(robj *o, sds key);
int hashTypeDelete(robj *o, sds key);
unsigned long hashTypeLength(const robj *o, int subtractExpiredFields);
......@@ -3165,7 +3181,8 @@ int hashTypeNext(hashTypeIterator *hi, int skipExpiredFields);
void hashTypeCurrentFromListpack(hashTypeIterator *hi, int what,
unsigned char **vstr,
unsigned int *vlen,
long long *vll);
long long *vll,
uint64_t *expireTime);
void hashTypeCurrentFromHashTable(hashTypeIterator *hi, int what, char **str,
size_t *len, uint64_t *expireTime);
void hashTypeCurrentObject(hashTypeIterator *hi, int what, unsigned char **vstr,
......@@ -3177,7 +3194,9 @@ int hashTypeSet(redisDb *db, robj *o, sds field, sds value, int flags);
robj *hashTypeDup(robj *o, sds newkey, uint64_t *minHashExpire);
uint64_t hashTypeRemoveFromExpires(ebuckets *hexpires, robj *o);
void hashTypeAddToExpires(redisDb *db, sds key, robj *hashObj, uint64_t expireTime);
int64_t hashTypeGetMinExpire(robj *keyObj);
void hashTypeFree(robj *o);
int hashTypeIsExpired(const robj *o, uint64_t expireAt);
unsigned char *hashTypeListpackGetLp(robj *o);
/* Hash-Field data type (of t_hash.c) */
hfield hfieldNew(const void *field, size_t fieldlen, int withExpireMeta);
......@@ -3637,6 +3656,7 @@ void strlenCommand(client *c);
void zrankCommand(client *c);
void zrevrankCommand(client *c);
void hsetCommand(client *c);
void hsetfCommand(client *c);
void hpexpireCommand(client *c);
void hexpireCommand(client *c);
void hpexpireatCommand(client *c);
......@@ -3648,6 +3668,7 @@ void hpexpiretimeCommand(client *c);
void hpersistCommand(client *c);
void hsetnxCommand(client *c);
void hgetCommand(client *c);
void hgetfCommand(client *c);
void hmgetCommand(client *c);
void hdelCommand(client *c);
void hlenCommand(client *c);
......
This diff is collapsed.
......@@ -432,7 +432,7 @@ robj *setTypePopRandom(robj *set) {
if (set->encoding == OBJ_ENCODING_LISTPACK) {
/* Find random and delete it without re-seeking the listpack. */
unsigned int i = 0;
unsigned char *p = lpNextRandom(set->ptr, lpFirst(set->ptr), &i, 1, 0);
unsigned char *p = lpNextRandom(set->ptr, lpFirst(set->ptr), &i, 1, 1);
unsigned int len = 0; /* initialize to silence warning */
long long llele = 0; /* initialize to silence warning */
char *str = (char *)lpGetValue(p, &len, &llele);
......@@ -815,7 +815,7 @@ void spopWithCountCommand(client *c) {
unsigned int index = 0;
unsigned char **ps = zmalloc(sizeof(char *) * count);
for (unsigned long i = 0; i < count; i++) {
p = lpNextRandom(lp, p, &index, count - i, 0);
p = lpNextRandom(lp, p, &index, count - i, 1);
unsigned int len;
str = (char *)lpGetValue(p, &len, (long long *)&llele);
......@@ -877,7 +877,7 @@ void spopWithCountCommand(client *c) {
unsigned int index = 0;
unsigned char **ps = zmalloc(sizeof(char *) * remaining);
for (unsigned long i = 0; i < remaining; i++) {
p = lpNextRandom(lp, p, &index, remaining - i, 0);
p = lpNextRandom(lp, p, &index, remaining - i, 1);
unsigned int len;
str = (char *)lpGetValue(p, &len, (long long *)&llele);
setTypeAddAux(newset, str, len, llele, 0);
......@@ -1103,7 +1103,7 @@ void srandmemberWithCountCommand(client *c) {
unsigned int i = 0;
addReplyArrayLen(c, count);
while (count) {
p = lpNextRandom(lp, p, &i, count--, 0);
p = lpNextRandom(lp, p, &i, count--, 1);
unsigned int len;
str = (char *)lpGetValue(p, &len, (long long *)&llele);
if (str == NULL) {
......
......@@ -1754,7 +1754,7 @@ void zsetTypeRandomElement(robj *zsetobj, unsigned long zsetsize, listpackEntry
*score = *(double*)dictGetVal(de);
} else if (zsetobj->encoding == OBJ_ENCODING_LISTPACK) {
listpackEntry val;
lpRandomPair(zsetobj->ptr, zsetsize, key, &val);
lpRandomPair(zsetobj->ptr, zsetsize, key, &val, 2);
if (score) {
if (val.sval) {
*score = zzlStrtod(val.sval,val.slen);
......@@ -4263,7 +4263,7 @@ void zrandmemberWithCountCommand(client *c, long l, int withscores) {
while (count) {
sample_count = count > limit ? limit : count;
count -= sample_count;
lpRandomPairs(zsetobj->ptr, sample_count, keys, vals);
lpRandomPairs(zsetobj->ptr, sample_count, keys, vals, 2);
zrandmemberReplyWithListpack(c, sample_count, keys, vals);
if (c->flags & CLIENT_CLOSE_ASAP)
break;
......@@ -4317,7 +4317,7 @@ void zrandmemberWithCountCommand(client *c, long l, int withscores) {
keys = zmalloc(sizeof(listpackEntry)*count);
if (withscores)
vals = zmalloc(sizeof(listpackEntry)*count);
serverAssert(lpRandomPairsUnique(zsetobj->ptr, count, keys, vals) == count);
serverAssert(lpRandomPairsUnique(zsetobj->ptr, count, keys, vals, 2) == count);
zrandmemberReplyWithListpack(c, count, keys, vals);
zfree(keys);
zfree(vals);
......
This diff is collapsed.
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment