Unverified Commit 7b9e9606 authored by debing.sun's avatar debing.sun Committed by GitHub
Browse files

Hash Field Expiration (#13303)

## Background

This PR introduces support for field-level expiration in Redis hashes. Previously, Redis supported expiration only at the key level, but this enhancement allows setting expiration times for individual fields within a hash.

## New commands
* HEXPIRE
* HEXPIREAT
* HEXPIRETIME
* HPERSIST
* HPEXPIRE
* HPEXPIREAT
* HPEXPIRETIME
* HPTTL
* HTTL

## Short example
from @moticless
```sh
127.0.0.1:6379>  hset myhash f1 v1 f2 v2 f3 v3                                                   
(integer) 3
127.0.0.1:6379>  hpexpire myhash 10000 NX fields 2 f2 f3                                         
1) (integer) 1
2) (integer) 1
127.0.0.1:6379>  hpttl myhash fields 3 f1 f2 f3                                                                                                                                                                         
1) (integer) -1
2) (integer) 9997
3) (integer) 9997
127.0.0.1:6379>  hgetall myhash  
1) "f3"
2) "v3"
3) "f2"
4) "v2"
5) "f1"
6) "v1"

... after 10 seconds ...

127.0.0.1:6379>  hgetall myhash  
1) "f1"
2) "v1"
127.0.0.1:6379>
```

## Expiration strategy
1. Integrate active
    Redis periodically performs active expiration and deletion of hash keys that contain expired fields, with a maximum attempt limit.
3. Lazy expiration
    When a client touches fields within a hash, Redis checks if the fields are expired. If a field is expired, it will be deleted. However, we do not delete expired fields during a traversal, we implicitly skip over them.

## RDB changes
Add two new rdb type s`RDB_TYPE_HASH_METADATA` and `RDB_TYPE_HASH_LISTPACK_EX`.

## Notification
1. Add `hpersist` notification for `HPERSIST` command.
5. Add `hexpire` notification for `HEXPIRE`, `HEXPIREAT`, `HPEXPIRE` and `HPEXPIREAT` commands.

## Internal
1. Add new data structure `ebuckets`, which is used to store TTL and keys, enabling quick retrieval of keys based on TTL.
2. Add new data structure `mstr` like sds, which is used to store a string with TTL.

This work was done by @moticless, @tezc, @ronen-kalish, @sundb, I just release it.
parents 2d1bb42c f0389f28
......@@ -354,7 +354,7 @@ endif
REDIS_SERVER_NAME=redis-server$(PROG_SUFFIX)
REDIS_SENTINEL_NAME=redis-sentinel$(PROG_SUFFIX)
REDIS_SERVER_OBJ=threads_mngr.o adlist.o quicklist.o ae.o anet.o dict.o kvstore.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o cluster_legacy.o crc16.o endianconv.o slowlog.o eval.o bio.o rio.o rand.o memtest.o syscheck.o crcspeed.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o tracking.o socket.o tls.o sha256.o timeout.o setcpuaffinity.o monotonic.o mt19937-64.o resp_parser.o call_reply.o script_lua.o script.o functions.o function_lua.o commands.o strl.o connection.o unix.o logreqres.o
REDIS_SERVER_OBJ=threads_mngr.o adlist.o quicklist.o ae.o anet.o dict.o ebuckets.o mstr.o kvstore.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o cluster_legacy.o crc16.o endianconv.o slowlog.o eval.o bio.o rio.o rand.o memtest.o syscheck.o crcspeed.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o tracking.o socket.o tls.o sha256.o timeout.o setcpuaffinity.o monotonic.o mt19937-64.o resp_parser.o call_reply.o script_lua.o script.o functions.o function_lua.o commands.o strl.o connection.o unix.o logreqres.o
REDIS_CLI_NAME=redis-cli$(PROG_SUFFIX)
REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o ae.o redisassert.o crcspeed.o crc64.o siphash.o crc16.o monotonic.o cli_common.o mt19937-64.o strl.o cli_commands.o
REDIS_BENCHMARK_NAME=redis-benchmark$(PROG_SUFFIX)
......
......@@ -1939,19 +1939,21 @@ int rewriteSortedSetObject(rio *r, robj *key, robj *o) {
*
* The function returns 0 on error, non-zero on success. */
static int rioWriteHashIteratorCursor(rio *r, hashTypeIterator *hi, int what) {
if (hi->encoding == OBJ_ENCODING_LISTPACK) {
if ((hi->encoding == OBJ_ENCODING_LISTPACK) || (hi->encoding == OBJ_ENCODING_LISTPACK_EX)) {
unsigned char *vstr = NULL;
unsigned int vlen = UINT_MAX;
long long vll = LLONG_MAX;
hashTypeCurrentFromListpack(hi, what, &vstr, &vlen, &vll);
hashTypeCurrentFromListpack(hi, what, &vstr, &vlen, &vll, NULL);
if (vstr)
return rioWriteBulkString(r, (char*)vstr, vlen);
else
return rioWriteBulkLongLong(r, vll);
} else if (hi->encoding == OBJ_ENCODING_HT) {
sds value = hashTypeCurrentFromHashTable(hi, what);
return rioWriteBulkString(r, value, sdslen(value));
char *str;
size_t len;
hashTypeCurrentFromHashTable(hi, what, &str, &len, NULL);
return rioWriteBulkString(r, str, len);
}
serverPanic("Unknown hash encoding");
......@@ -1961,37 +1963,60 @@ static int rioWriteHashIteratorCursor(rio *r, hashTypeIterator *hi, int what) {
/* Emit the commands needed to rebuild a hash object.
* The function returns 0 on error, 1 on success. */
int rewriteHashObject(rio *r, robj *key, robj *o) {
int res = 0; /*fail*/
hashTypeIterator *hi;
long long count = 0, items = hashTypeLength(o);
long long count = 0, items = hashTypeLength(o, 0);
int isHFE = hashTypeGetMinExpire(o) != EB_EXPIRE_TIME_INVALID;
hi = hashTypeInitIterator(o);
while (hashTypeNext(hi) != C_ERR) {
if (count == 0) {
int cmd_items = (items > AOF_REWRITE_ITEMS_PER_CMD) ?
AOF_REWRITE_ITEMS_PER_CMD : items;
if (!rioWriteBulkCount(r,'*',2+cmd_items*2) ||
!rioWriteBulkString(r,"HMSET",5) ||
!rioWriteBulkObject(r,key))
{
hashTypeReleaseIterator(hi);
return 0;
if (!isHFE) {
while (hashTypeNext(hi, 0) != C_ERR) {
if (count == 0) {
int cmd_items = (items > AOF_REWRITE_ITEMS_PER_CMD) ?
AOF_REWRITE_ITEMS_PER_CMD : items;
if (!rioWriteBulkCount(r, '*', 2 + cmd_items * 2) ||
!rioWriteBulkString(r, "HMSET", 5) ||
!rioWriteBulkObject(r, key))
goto reHashEnd;
}
}
if (!rioWriteHashIteratorCursor(r, hi, OBJ_HASH_KEY) ||
!rioWriteHashIteratorCursor(r, hi, OBJ_HASH_VALUE))
{
hashTypeReleaseIterator(hi);
return 0;
if (!rioWriteHashIteratorCursor(r, hi, OBJ_HASH_KEY) ||
!rioWriteHashIteratorCursor(r, hi, OBJ_HASH_VALUE))
goto reHashEnd;
if (++count == AOF_REWRITE_ITEMS_PER_CMD) count = 0;
items--;
}
} else {
while (hashTypeNext(hi, 0) != C_ERR) {
char hmsetCmd[] = "*4\r\n$5\r\nHMSET\r\n";
if ( (!rioWrite(r, hmsetCmd, sizeof(hmsetCmd) - 1)) ||
(!rioWriteBulkObject(r, key)) ||
(!rioWriteHashIteratorCursor(r, hi, OBJ_HASH_KEY)) ||
(!rioWriteHashIteratorCursor(r, hi, OBJ_HASH_VALUE)) )
goto reHashEnd;
if (hi->expire_time != EB_EXPIRE_TIME_INVALID) {
char cmd[] = "*6\r\n$10\r\nHPEXPIREAT\r\n";
if ( (!rioWrite(r, cmd, sizeof(cmd) - 1)) ||
(!rioWriteBulkObject(r, key)) ||
(!rioWriteBulkLongLong(r, hi->expire_time)) ||
(!rioWriteBulkString(r, "FIELDS", 6)) ||
(!rioWriteBulkString(r, "1", 1)) ||
(!rioWriteHashIteratorCursor(r, hi, OBJ_HASH_KEY)) )
goto reHashEnd;
}
}
if (++count == AOF_REWRITE_ITEMS_PER_CMD) count = 0;
items--;
}
hashTypeReleaseIterator(hi);
res = 1; /* success */
return 1;
reHashEnd:
hashTypeReleaseIterator(hi);
return res;
}
/* Helper for rewriteStreamObject() that generates a bulk string into the
......
......@@ -176,6 +176,7 @@ void dumpCommand(client *c) {
/* RESTORE key ttl serialized-value [REPLACE] [ABSTTL] [IDLETIME seconds] [FREQ frequency] */
void restoreCommand(client *c) {
uint64_t minExpiredField = EB_EXPIRE_TIME_INVALID;
long long ttl, lfu_freq = -1, lru_idle = -1, lru_clock = -1;
rio payload;
int j, type, replace = 0, absttl = 0;
......@@ -239,7 +240,7 @@ void restoreCommand(client *c) {
rioInitWithBuffer(&payload,c->argv[3]->ptr);
if (((type = rdbLoadObjectType(&payload)) == -1) ||
((obj = rdbLoadObject(type,&payload,key->ptr,c->db->id,NULL)) == NULL))
((obj = rdbLoadObject(type,&payload,key->ptr,c->db,NULL, &minExpiredField)) == NULL))
{
addReplyError(c,"Bad data format");
return;
......@@ -265,7 +266,13 @@ void restoreCommand(client *c) {
}
/* Create the key and set the TTL if any */
dbAdd(c->db,key,obj);
dictEntry *de = dbAdd(c->db,key,obj);
/* If minExpiredField was set, then the object is hash with expiration
* on fields and need to register it in global HFE DS */
if (minExpiredField != EB_EXPIRE_TIME_INVALID)
hashTypeAddToExpires(c->db, dictGetKey(de), obj, minExpiredField);
if (ttl) {
setExpire(c,c->db,key,ttl);
if (!absttl) {
......
......@@ -3303,6 +3303,107 @@ struct COMMAND_ARG HEXISTS_Args[] = {
{MAKE_ARG("field",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)},
};
/********** HEXPIRE ********************/
#ifndef SKIP_CMD_HISTORY_TABLE
/* HEXPIRE history */
#define HEXPIRE_History NULL
#endif
#ifndef SKIP_CMD_TIPS_TABLE
/* HEXPIRE tips */
#define HEXPIRE_Tips NULL
#endif
#ifndef SKIP_CMD_KEY_SPECS_TABLE
/* HEXPIRE key specs */
keySpec HEXPIRE_Keyspecs[1] = {
{NULL,CMD_KEY_RW|CMD_KEY_UPDATE,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}}
};
#endif
/* HEXPIRE condition argument table */
struct COMMAND_ARG HEXPIRE_condition_Subargs[] = {
{MAKE_ARG("nx",ARG_TYPE_PURE_TOKEN,-1,"NX",NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("xx",ARG_TYPE_PURE_TOKEN,-1,"XX",NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("gt",ARG_TYPE_PURE_TOKEN,-1,"GT",NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("lt",ARG_TYPE_PURE_TOKEN,-1,"LT",NULL,NULL,CMD_ARG_NONE,0,NULL)},
};
/* HEXPIRE argument table */
struct COMMAND_ARG HEXPIRE_Args[] = {
{MAKE_ARG("key",ARG_TYPE_KEY,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("seconds",ARG_TYPE_INTEGER,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("condition",ARG_TYPE_ONEOF,-1,NULL,NULL,NULL,CMD_ARG_OPTIONAL,4,NULL),.subargs=HEXPIRE_condition_Subargs},
{MAKE_ARG("fields",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("numfields",ARG_TYPE_INTEGER,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("field",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_MULTIPLE,0,NULL)},
};
/********** HEXPIREAT ********************/
#ifndef SKIP_CMD_HISTORY_TABLE
/* HEXPIREAT history */
#define HEXPIREAT_History NULL
#endif
#ifndef SKIP_CMD_TIPS_TABLE
/* HEXPIREAT tips */
#define HEXPIREAT_Tips NULL
#endif
#ifndef SKIP_CMD_KEY_SPECS_TABLE
/* HEXPIREAT key specs */
keySpec HEXPIREAT_Keyspecs[1] = {
{NULL,CMD_KEY_RW|CMD_KEY_UPDATE,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}}
};
#endif
/* HEXPIREAT condition argument table */
struct COMMAND_ARG HEXPIREAT_condition_Subargs[] = {
{MAKE_ARG("nx",ARG_TYPE_PURE_TOKEN,-1,"NX",NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("xx",ARG_TYPE_PURE_TOKEN,-1,"XX",NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("gt",ARG_TYPE_PURE_TOKEN,-1,"GT",NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("lt",ARG_TYPE_PURE_TOKEN,-1,"LT",NULL,NULL,CMD_ARG_NONE,0,NULL)},
};
/* HEXPIREAT argument table */
struct COMMAND_ARG HEXPIREAT_Args[] = {
{MAKE_ARG("key",ARG_TYPE_KEY,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("unix-time-seconds",ARG_TYPE_UNIX_TIME,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("condition",ARG_TYPE_ONEOF,-1,NULL,NULL,NULL,CMD_ARG_OPTIONAL,4,NULL),.subargs=HEXPIREAT_condition_Subargs},
{MAKE_ARG("fields",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("numfields",ARG_TYPE_INTEGER,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("field",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_MULTIPLE,0,NULL)},
};
/********** HEXPIRETIME ********************/
#ifndef SKIP_CMD_HISTORY_TABLE
/* HEXPIRETIME history */
#define HEXPIRETIME_History NULL
#endif
#ifndef SKIP_CMD_TIPS_TABLE
/* HEXPIRETIME tips */
#define HEXPIRETIME_Tips NULL
#endif
#ifndef SKIP_CMD_KEY_SPECS_TABLE
/* HEXPIRETIME key specs */
keySpec HEXPIRETIME_Keyspecs[1] = {
{NULL,CMD_KEY_RO|CMD_KEY_ACCESS,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}}
};
#endif
/* HEXPIRETIME argument table */
struct COMMAND_ARG HEXPIRETIME_Args[] = {
{MAKE_ARG("key",ARG_TYPE_KEY,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("fields",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("numfields",ARG_TYPE_INTEGER,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("field",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_MULTIPLE,0,NULL)},
};
/********** HGET ********************/
#ifndef SKIP_CMD_HISTORY_TABLE
......@@ -3512,6 +3613,161 @@ struct COMMAND_ARG HMSET_Args[] = {
{MAKE_ARG("data",ARG_TYPE_BLOCK,-1,NULL,NULL,NULL,CMD_ARG_MULTIPLE,2,NULL),.subargs=HMSET_data_Subargs},
};
/********** HPERSIST ********************/
#ifndef SKIP_CMD_HISTORY_TABLE
/* HPERSIST history */
#define HPERSIST_History NULL
#endif
#ifndef SKIP_CMD_TIPS_TABLE
/* HPERSIST tips */
#define HPERSIST_Tips NULL
#endif
#ifndef SKIP_CMD_KEY_SPECS_TABLE
/* HPERSIST key specs */
keySpec HPERSIST_Keyspecs[1] = {
{NULL,CMD_KEY_RW|CMD_KEY_UPDATE,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}}
};
#endif
/* HPERSIST argument table */
struct COMMAND_ARG HPERSIST_Args[] = {
{MAKE_ARG("key",ARG_TYPE_KEY,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("fields",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("numfields",ARG_TYPE_INTEGER,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("field",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_MULTIPLE,0,NULL)},
};
/********** HPEXPIRE ********************/
#ifndef SKIP_CMD_HISTORY_TABLE
/* HPEXPIRE history */
#define HPEXPIRE_History NULL
#endif
#ifndef SKIP_CMD_TIPS_TABLE
/* HPEXPIRE tips */
#define HPEXPIRE_Tips NULL
#endif
#ifndef SKIP_CMD_KEY_SPECS_TABLE
/* HPEXPIRE key specs */
keySpec HPEXPIRE_Keyspecs[1] = {
{NULL,CMD_KEY_RW|CMD_KEY_UPDATE,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}}
};
#endif
/* HPEXPIRE condition argument table */
struct COMMAND_ARG HPEXPIRE_condition_Subargs[] = {
{MAKE_ARG("nx",ARG_TYPE_PURE_TOKEN,-1,"NX",NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("xx",ARG_TYPE_PURE_TOKEN,-1,"XX",NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("gt",ARG_TYPE_PURE_TOKEN,-1,"GT",NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("lt",ARG_TYPE_PURE_TOKEN,-1,"LT",NULL,NULL,CMD_ARG_NONE,0,NULL)},
};
/* HPEXPIRE argument table */
struct COMMAND_ARG HPEXPIRE_Args[] = {
{MAKE_ARG("key",ARG_TYPE_KEY,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("milliseconds",ARG_TYPE_INTEGER,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("condition",ARG_TYPE_ONEOF,-1,NULL,NULL,NULL,CMD_ARG_OPTIONAL,4,NULL),.subargs=HPEXPIRE_condition_Subargs},
{MAKE_ARG("fields",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("numfields",ARG_TYPE_INTEGER,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("field",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_MULTIPLE,0,NULL)},
};
/********** HPEXPIREAT ********************/
#ifndef SKIP_CMD_HISTORY_TABLE
/* HPEXPIREAT history */
#define HPEXPIREAT_History NULL
#endif
#ifndef SKIP_CMD_TIPS_TABLE
/* HPEXPIREAT tips */
#define HPEXPIREAT_Tips NULL
#endif
#ifndef SKIP_CMD_KEY_SPECS_TABLE
/* HPEXPIREAT key specs */
keySpec HPEXPIREAT_Keyspecs[1] = {
{NULL,CMD_KEY_RW|CMD_KEY_UPDATE,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}}
};
#endif
/* HPEXPIREAT condition argument table */
struct COMMAND_ARG HPEXPIREAT_condition_Subargs[] = {
{MAKE_ARG("nx",ARG_TYPE_PURE_TOKEN,-1,"NX",NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("xx",ARG_TYPE_PURE_TOKEN,-1,"XX",NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("gt",ARG_TYPE_PURE_TOKEN,-1,"GT",NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("lt",ARG_TYPE_PURE_TOKEN,-1,"LT",NULL,NULL,CMD_ARG_NONE,0,NULL)},
};
/* HPEXPIREAT argument table */
struct COMMAND_ARG HPEXPIREAT_Args[] = {
{MAKE_ARG("key",ARG_TYPE_KEY,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("unix-time-milliseconds",ARG_TYPE_UNIX_TIME,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("condition",ARG_TYPE_ONEOF,-1,NULL,NULL,NULL,CMD_ARG_OPTIONAL,4,NULL),.subargs=HPEXPIREAT_condition_Subargs},
{MAKE_ARG("fields",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("numfields",ARG_TYPE_INTEGER,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("field",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_MULTIPLE,0,NULL)},
};
/********** HPEXPIRETIME ********************/
#ifndef SKIP_CMD_HISTORY_TABLE
/* HPEXPIRETIME history */
#define HPEXPIRETIME_History NULL
#endif
#ifndef SKIP_CMD_TIPS_TABLE
/* HPEXPIRETIME tips */
#define HPEXPIRETIME_Tips NULL
#endif
#ifndef SKIP_CMD_KEY_SPECS_TABLE
/* HPEXPIRETIME key specs */
keySpec HPEXPIRETIME_Keyspecs[1] = {
{NULL,CMD_KEY_RO|CMD_KEY_ACCESS,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}}
};
#endif
/* HPEXPIRETIME argument table */
struct COMMAND_ARG HPEXPIRETIME_Args[] = {
{MAKE_ARG("key",ARG_TYPE_KEY,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("fields",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("numfields",ARG_TYPE_INTEGER,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("field",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_MULTIPLE,0,NULL)},
};
/********** HPTTL ********************/
#ifndef SKIP_CMD_HISTORY_TABLE
/* HPTTL history */
#define HPTTL_History NULL
#endif
#ifndef SKIP_CMD_TIPS_TABLE
/* HPTTL tips */
#define HPTTL_Tips NULL
#endif
#ifndef SKIP_CMD_KEY_SPECS_TABLE
/* HPTTL key specs */
keySpec HPTTL_Keyspecs[1] = {
{NULL,CMD_KEY_RO|CMD_KEY_ACCESS,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}}
};
#endif
/* HPTTL argument table */
struct COMMAND_ARG HPTTL_Args[] = {
{MAKE_ARG("key",ARG_TYPE_KEY,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("fields",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("numfields",ARG_TYPE_INTEGER,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("field",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_MULTIPLE,0,NULL)},
};
/********** HRANDFIELD ********************/
#ifndef SKIP_CMD_HISTORY_TABLE
......@@ -3659,6 +3915,33 @@ struct COMMAND_ARG HSTRLEN_Args[] = {
{MAKE_ARG("field",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)},
};
/********** HTTL ********************/
#ifndef SKIP_CMD_HISTORY_TABLE
/* HTTL history */
#define HTTL_History NULL
#endif
#ifndef SKIP_CMD_TIPS_TABLE
/* HTTL tips */
#define HTTL_Tips NULL
#endif
#ifndef SKIP_CMD_KEY_SPECS_TABLE
/* HTTL key specs */
keySpec HTTL_Keyspecs[1] = {
{NULL,CMD_KEY_RO|CMD_KEY_ACCESS,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}}
};
#endif
/* HTTL argument table */
struct COMMAND_ARG HTTL_Args[] = {
{MAKE_ARG("key",ARG_TYPE_KEY,0,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("fields",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("numfields",ARG_TYPE_INTEGER,-1,NULL,NULL,NULL,CMD_ARG_NONE,0,NULL)},
{MAKE_ARG("field",ARG_TYPE_STRING,-1,NULL,NULL,NULL,CMD_ARG_MULTIPLE,0,NULL)},
};
/********** HVALS ********************/
#ifndef SKIP_CMD_HISTORY_TABLE
......@@ -10710,6 +10993,9 @@ struct COMMAND_STRUCT redisCommandTable[] = {
/* hash */
{MAKE_CMD("hdel","Deletes one or more fields and their values from a hash. Deletes the hash if no fields remain.","O(N) where N is the number of fields to be removed.","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HDEL_History,1,HDEL_Tips,0,hdelCommand,-3,CMD_WRITE|CMD_FAST,ACL_CATEGORY_HASH,HDEL_Keyspecs,1,NULL,2),.args=HDEL_Args},
{MAKE_CMD("hexists","Determines whether a field exists in a hash.","O(1)","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HEXISTS_History,0,HEXISTS_Tips,0,hexistsCommand,3,CMD_READONLY|CMD_FAST,ACL_CATEGORY_HASH,HEXISTS_Keyspecs,1,NULL,2),.args=HEXISTS_Args},
{MAKE_CMD("hexpire","Set expiry for hash field using relative time to expire (seconds)","O(N) where N is the number of specified fields","7.4.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HEXPIRE_History,0,HEXPIRE_Tips,0,hexpireCommand,-6,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_HASH,HEXPIRE_Keyspecs,1,NULL,6),.args=HEXPIRE_Args},
{MAKE_CMD("hexpireat","Set expiry for hash field using an absolute Unix timestamp (seconds)","O(N) where N is the number of specified fields","7.4.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HEXPIREAT_History,0,HEXPIREAT_Tips,0,hexpireatCommand,-6,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_HASH,HEXPIREAT_Keyspecs,1,NULL,6),.args=HEXPIREAT_Args},
{MAKE_CMD("hexpiretime","Returns the expiration time of a hash field as a Unix timestamp, in seconds.","O(N) where N is the number of specified fields","7.4.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HEXPIRETIME_History,0,HEXPIRETIME_Tips,0,hexpiretimeCommand,-5,CMD_READONLY|CMD_FAST,ACL_CATEGORY_HASH,HEXPIRETIME_Keyspecs,1,NULL,4),.args=HEXPIRETIME_Args},
{MAKE_CMD("hget","Returns the value of a field in a hash.","O(1)","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HGET_History,0,HGET_Tips,0,hgetCommand,3,CMD_READONLY|CMD_FAST,ACL_CATEGORY_HASH,HGET_Keyspecs,1,NULL,2),.args=HGET_Args},
{MAKE_CMD("hgetall","Returns all fields and values in a hash.","O(N) where N is the size of the hash.","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HGETALL_History,0,HGETALL_Tips,1,hgetallCommand,2,CMD_READONLY,ACL_CATEGORY_HASH,HGETALL_Keyspecs,1,NULL,1),.args=HGETALL_Args},
{MAKE_CMD("hincrby","Increments the integer value of a field in a hash by a number. Uses 0 as initial value if the field doesn't exist.","O(1)","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HINCRBY_History,0,HINCRBY_Tips,0,hincrbyCommand,4,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_HASH,HINCRBY_Keyspecs,1,NULL,3),.args=HINCRBY_Args},
......@@ -10718,11 +11004,17 @@ struct COMMAND_STRUCT redisCommandTable[] = {
{MAKE_CMD("hlen","Returns the number of fields in a hash.","O(1)","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HLEN_History,0,HLEN_Tips,0,hlenCommand,2,CMD_READONLY|CMD_FAST,ACL_CATEGORY_HASH,HLEN_Keyspecs,1,NULL,1),.args=HLEN_Args},
{MAKE_CMD("hmget","Returns the values of all fields in a hash.","O(N) where N is the number of fields being requested.","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HMGET_History,0,HMGET_Tips,0,hmgetCommand,-3,CMD_READONLY|CMD_FAST,ACL_CATEGORY_HASH,HMGET_Keyspecs,1,NULL,2),.args=HMGET_Args},
{MAKE_CMD("hmset","Sets the values of multiple fields.","O(N) where N is the number of fields being set.","2.0.0",CMD_DOC_DEPRECATED,"`HSET` with multiple field-value pairs","4.0.0","hash",COMMAND_GROUP_HASH,HMSET_History,0,HMSET_Tips,0,hsetCommand,-4,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_HASH,HMSET_Keyspecs,1,NULL,2),.args=HMSET_Args},
{MAKE_CMD("hpersist","Removes the expiration time for each specified field","O(N) where N is the number of specified fields","7.4.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HPERSIST_History,0,HPERSIST_Tips,0,hpersistCommand,-5,CMD_WRITE|CMD_FAST,ACL_CATEGORY_HASH,HPERSIST_Keyspecs,1,NULL,4),.args=HPERSIST_Args},
{MAKE_CMD("hpexpire","Set expiry for hash field using relative time to expire (milliseconds)","O(N) where N is the number of specified fields","7.4.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HPEXPIRE_History,0,HPEXPIRE_Tips,0,hpexpireCommand,-6,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_HASH,HPEXPIRE_Keyspecs,1,NULL,6),.args=HPEXPIRE_Args},
{MAKE_CMD("hpexpireat","Set expiry for hash field using an absolute Unix timestamp (milliseconds)","O(N) where N is the number of specified fields","7.4.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HPEXPIREAT_History,0,HPEXPIREAT_Tips,0,hpexpireatCommand,-6,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_HASH,HPEXPIREAT_Keyspecs,1,NULL,6),.args=HPEXPIREAT_Args},
{MAKE_CMD("hpexpiretime","Returns the expiration time of a hash field as a Unix timestamp, in msec.","O(N) where N is the number of specified fields","7.4.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HPEXPIRETIME_History,0,HPEXPIRETIME_Tips,0,hpexpiretimeCommand,-5,CMD_READONLY|CMD_FAST,ACL_CATEGORY_HASH,HPEXPIRETIME_Keyspecs,1,NULL,4),.args=HPEXPIRETIME_Args},
{MAKE_CMD("hpttl","Returns the TTL in milliseconds of a hash field.","O(N) where N is the number of specified fields","7.4.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HPTTL_History,0,HPTTL_Tips,0,hpttlCommand,-5,CMD_READONLY|CMD_FAST,ACL_CATEGORY_HASH,HPTTL_Keyspecs,1,NULL,4),.args=HPTTL_Args},
{MAKE_CMD("hrandfield","Returns one or more random fields from a hash.","O(N) where N is the number of fields returned","6.2.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HRANDFIELD_History,0,HRANDFIELD_Tips,1,hrandfieldCommand,-2,CMD_READONLY,ACL_CATEGORY_HASH,HRANDFIELD_Keyspecs,1,NULL,2),.args=HRANDFIELD_Args},
{MAKE_CMD("hscan","Iterates over fields and values of a hash.","O(1) for every call. O(N) for a complete iteration, including enough command calls for the cursor to return back to 0. N is the number of elements inside the collection.","2.8.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HSCAN_History,0,HSCAN_Tips,1,hscanCommand,-3,CMD_READONLY,ACL_CATEGORY_HASH,HSCAN_Keyspecs,1,NULL,5),.args=HSCAN_Args},
{MAKE_CMD("hset","Creates or modifies the value of a field in a hash.","O(1) for each field/value pair added, so O(N) to add N field/value pairs when the command is called with multiple field/value pairs.","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HSET_History,1,HSET_Tips,0,hsetCommand,-4,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_HASH,HSET_Keyspecs,1,NULL,2),.args=HSET_Args},
{MAKE_CMD("hsetnx","Sets the value of a field in a hash only when the field doesn't exist.","O(1)","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HSETNX_History,0,HSETNX_Tips,0,hsetnxCommand,4,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_HASH,HSETNX_Keyspecs,1,NULL,3),.args=HSETNX_Args},
{MAKE_CMD("hstrlen","Returns the length of the value of a field.","O(1)","3.2.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HSTRLEN_History,0,HSTRLEN_Tips,0,hstrlenCommand,3,CMD_READONLY|CMD_FAST,ACL_CATEGORY_HASH,HSTRLEN_Keyspecs,1,NULL,2),.args=HSTRLEN_Args},
{MAKE_CMD("httl","Returns the TTL in seconds of a hash field.","O(N) where N is the number of specified fields","7.4.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HTTL_History,0,HTTL_Tips,0,httlCommand,-5,CMD_READONLY|CMD_FAST,ACL_CATEGORY_HASH,HTTL_Keyspecs,1,NULL,4),.args=HTTL_Args},
{MAKE_CMD("hvals","Returns all values in a hash.","O(N) where N is the size of the hash.","2.0.0",CMD_DOC_NONE,NULL,NULL,"hash",COMMAND_GROUP_HASH,HVALS_History,0,HVALS_Tips,1,hvalsCommand,2,CMD_READONLY,ACL_CATEGORY_HASH,HVALS_Keyspecs,1,NULL,1),.args=HVALS_Args},
/* hyperloglog */
{MAKE_CMD("pfadd","Adds elements to a HyperLogLog key. Creates the key if it doesn't exist.","O(1) to add every element.","2.8.9",CMD_DOC_NONE,NULL,NULL,"hyperloglog",COMMAND_GROUP_HYPERLOGLOG,PFADD_History,0,PFADD_Tips,0,pfaddCommand,-2,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_HYPERLOGLOG,PFADD_Keyspecs,1,NULL,2),.args=PFADD_Args},
......
{
"HEXPIRE": {
"summary": "Set expiry for hash field using relative time to expire (seconds)",
"complexity": "O(N) where N is the number of specified fields",
"group": "hash",
"since": "7.4.0",
"arity": -6,
"function": "hexpireCommand",
"history": [],
"command_flags": [
"WRITE",
"DENYOOM",
"FAST"
],
"acl_categories": [
"HASH"
],
"key_specs": [
{
"flags": [
"RW",
"UPDATE"
],
"begin_search": {
"index": {
"pos": 1
}
},
"find_keys": {
"range": {
"lastkey": 0,
"step": 1,
"limit": 0
}
}
}
],
"reply_schema": {
"description": "Array of results. Returns empty array if the key does not exist.",
"type": "array",
"minItems": 0,
"maxItems": 4294967295,
"items": {
"oneOf": [
{
"description": "The field does not exist.",
"const": -2
},
{
"description": "Specified NX | XX | GT | LT condition not met",
"const": 0
},
{
"description": "Expiration time was set or updated.",
"const": 1
},
{
"description": "Field deleted because the specified expiration time is in the past.",
"const": 2
}
]
}
},
"arguments": [
{
"name": "key",
"type": "key",
"key_spec_index": 0
},
{
"name": "seconds",
"type": "integer"
},
{
"name": "condition",
"type": "oneof",
"optional": true,
"arguments": [
{
"name": "nx",
"type": "pure-token",
"token": "NX"
},
{
"name": "xx",
"type": "pure-token",
"token": "XX"
},
{
"name": "gt",
"type": "pure-token",
"token": "GT"
},
{
"name": "lt",
"type": "pure-token",
"token": "LT"
}
]
},
{
"name": "FIELDS",
"type": "string"
},
{
"name": "numfields",
"type": "integer"
},
{
"name": "field",
"type": "string",
"multiple": true
}
]
}
}
{
"HEXPIREAT": {
"summary": "Set expiry for hash field using an absolute Unix timestamp (seconds)",
"complexity": "O(N) where N is the number of specified fields",
"group": "hash",
"since": "7.4.0",
"arity": -6,
"function": "hexpireatCommand",
"history": [],
"command_flags": [
"WRITE",
"DENYOOM",
"FAST"
],
"acl_categories": [
"HASH"
],
"key_specs": [
{
"flags": [
"RW",
"UPDATE"
],
"begin_search": {
"index": {
"pos": 1
}
},
"find_keys": {
"range": {
"lastkey": 0,
"step": 1,
"limit": 0
}
}
}
],
"reply_schema": {
"description": "Array of results. Returns empty array if the key does not exist.",
"type": "array",
"minItems": 0,
"maxItems": 4294967295,
"items": {
"oneOf": [
{
"description": "The field does not exist.",
"const": -2
},
{
"description": "Specified NX | XX | GT | LT condition not met",
"const": 0
},
{
"description": "Expiration time was set or updated.",
"const": 1
},
{
"description": "Field deleted because the specified expiration time is in the past.",
"const": 2
}
]
}
},
"arguments": [
{
"name": "key",
"type": "key",
"key_spec_index": 0
},
{
"name": "unix-time-seconds",
"type": "unix-time"
},
{
"name": "condition",
"type": "oneof",
"optional": true,
"arguments": [
{
"name": "nx",
"type": "pure-token",
"token": "NX"
},
{
"name": "xx",
"type": "pure-token",
"token": "XX"
},
{
"name": "gt",
"type": "pure-token",
"token": "GT"
},
{
"name": "lt",
"type": "pure-token",
"token": "LT"
}
]
},
{
"name": "FIELDS",
"type": "string"
},
{
"name": "numfields",
"type": "integer"
},
{
"name": "field",
"type": "string",
"multiple": true
}
]
}
}
\ No newline at end of file
{
"HEXPIRETIME": {
"summary": "Returns the expiration time of a hash field as a Unix timestamp, in seconds.",
"complexity": "O(N) where N is the number of specified fields",
"group": "hash",
"since": "7.4.0",
"arity": -5,
"function": "hexpiretimeCommand",
"history": [],
"command_flags": [
"READONLY",
"FAST"
],
"acl_categories": [
"HASH"
],
"key_specs": [
{
"flags": [
"RO",
"ACCESS"
],
"begin_search": {
"index": {
"pos": 1
}
},
"find_keys": {
"range": {
"lastkey": 0,
"step": 1,
"limit": 0
}
}
}
],
"reply_schema": {
"description": "Array of results. Returns empty array if the key does not exist.",
"type": "array",
"minItems": 0,
"maxItems": 4294967295,
"items": {
"oneOf": [
{
"description": "The field does not exist.",
"const": -2
},
{
"description": "The field exists but has no associated expire.",
"const": -1
},
{
"description": "Expiration Unix timestamp in seconds.",
"type": "integer",
"minimum": 1
}
]
}
},
"arguments": [
{
"name": "key",
"type": "key",
"key_spec_index": 0
},
{
"name": "FIELDS",
"type": "string"
},
{
"name": "numfields",
"type": "integer"
},
{
"name": "field",
"type": "string",
"multiple": true
}
]
}
}
{
"HPERSIST": {
"summary": "Removes the expiration time for each specified field",
"complexity": "O(N) where N is the number of specified fields",
"group": "hash",
"since": "7.4.0",
"arity": -5,
"function": "hpersistCommand",
"history": [],
"command_flags": [
"WRITE",
"FAST"
],
"acl_categories": [
"HASH"
],
"key_specs": [
{
"flags": [
"RW",
"UPDATE"
],
"begin_search": {
"index": {
"pos": 1
}
},
"find_keys": {
"range": {
"lastkey": 0,
"step": 1,
"limit": 0
}
}
}
],
"reply_schema": {
"description": "Array of results. Returns empty array if the key does not exist.",
"type": "array",
"minItems": 0,
"maxItems": 4294967295,
"items": {
"oneOf": [
{
"description": "The field does not exist.",
"const": -2
},
{
"description": "The field exists but has no associated expire.",
"const": -1
},
{
"description": "Expiration time was removed",
"const": 1
}
]
}
},
"arguments": [
{
"name": "key",
"type": "key",
"key_spec_index": 0
},
{
"name": "FIELDS",
"type": "string"
},
{
"name": "numfields",
"type": "integer"
},
{
"name": "field",
"type": "string",
"multiple": true
}
]
}
}
{
"HPEXPIRE": {
"summary": "Set expiry for hash field using relative time to expire (milliseconds)",
"complexity": "O(N) where N is the number of specified fields",
"group": "hash",
"since": "7.4.0",
"arity": -6,
"function": "hpexpireCommand",
"history": [],
"command_flags": [
"WRITE",
"DENYOOM",
"FAST"
],
"acl_categories": [
"HASH"
],
"key_specs": [
{
"flags": [
"RW",
"UPDATE"
],
"begin_search": {
"index": {
"pos": 1
}
},
"find_keys": {
"range": {
"lastkey": 0,
"step": 1,
"limit": 0
}
}
}
],
"reply_schema": {
"description": "Array of results. Returns empty array if the key does not exist.",
"type": "array",
"minItems": 0,
"maxItems": 4294967295,
"items": {
"oneOf": [
{
"description": "The field does not exist.",
"const": -2
},
{
"description": "Specified NX | XX | GT | LT condition not met",
"const": 0
},
{
"description": "Expiration time was set or updated.",
"const": 1
},
{
"description": "Field deleted because the specified expiration time is in the past.",
"const": 2
}
]
}
},
"arguments": [
{
"name": "key",
"type": "key",
"key_spec_index": 0
},
{
"name": "milliseconds",
"type": "integer"
},
{
"name": "condition",
"type": "oneof",
"optional": true,
"arguments": [
{
"name": "nx",
"type": "pure-token",
"token": "NX"
},
{
"name": "xx",
"type": "pure-token",
"token": "XX"
},
{
"name": "gt",
"type": "pure-token",
"token": "GT"
},
{
"name": "lt",
"type": "pure-token",
"token": "LT"
}
]
},
{
"name": "FIELDS",
"type": "string"
},
{
"name": "numfields",
"type": "integer"
},
{
"name": "field",
"type": "string",
"multiple": true
}
]
}
}
\ No newline at end of file
{
"HPEXPIREAT": {
"summary": "Set expiry for hash field using an absolute Unix timestamp (milliseconds)",
"complexity": "O(N) where N is the number of specified fields",
"group": "hash",
"since": "7.4.0",
"arity": -6,
"function": "hpexpireatCommand",
"history": [],
"command_flags": [
"WRITE",
"DENYOOM",
"FAST"
],
"acl_categories": [
"HASH"
],
"key_specs": [
{
"flags": [
"RW",
"UPDATE"
],
"begin_search": {
"index": {
"pos": 1
}
},
"find_keys": {
"range": {
"lastkey": 0,
"step": 1,
"limit": 0
}
}
}
],
"reply_schema": {
"description": "Array of results. Returns empty array if the key does not exist.",
"type": "array",
"minItems": 0,
"maxItems": 4294967295,
"items": {
"oneOf": [
{
"description": "The field does not exist.",
"const": -2
},
{
"description": "Specified NX | XX | GT | LT condition not met",
"const": 0
},
{
"description": "Expiration time was set or updated.",
"const": 1
},
{
"description": "Field deleted because the specified expiration time is in the past.",
"const": 2
}
]
}
},
"arguments": [
{
"name": "key",
"type": "key",
"key_spec_index": 0
},
{
"name": "unix-time-milliseconds",
"type": "unix-time"
},
{
"name": "condition",
"type": "oneof",
"optional": true,
"arguments": [
{
"name": "nx",
"type": "pure-token",
"token": "NX"
},
{
"name": "xx",
"type": "pure-token",
"token": "XX"
},
{
"name": "gt",
"type": "pure-token",
"token": "GT"
},
{
"name": "lt",
"type": "pure-token",
"token": "LT"
}
]
},
{
"name": "FIELDS",
"type": "string"
},
{
"name": "numfields",
"type": "integer"
},
{
"name": "field",
"type": "string",
"multiple": true
}
]
}
}
\ No newline at end of file
{
"HPEXPIRETIME": {
"summary": "Returns the expiration time of a hash field as a Unix timestamp, in msec.",
"complexity": "O(N) where N is the number of specified fields",
"group": "hash",
"since": "7.4.0",
"arity": -5,
"function": "hpexpiretimeCommand",
"history": [],
"command_flags": [
"READONLY",
"FAST"
],
"acl_categories": [
"HASH"
],
"key_specs": [
{
"flags": [
"RO",
"ACCESS"
],
"begin_search": {
"index": {
"pos": 1
}
},
"find_keys": {
"range": {
"lastkey": 0,
"step": 1,
"limit": 0
}
}
}
],
"reply_schema": {
"description": "Array of results. Returns empty array if the key does not exist.",
"type": "array",
"minItems": 0,
"maxItems": 4294967295,
"items": {
"oneOf": [
{
"description": "The field does not exist.",
"const": -2
},
{
"description": "The field exists but has no associated expire.",
"const": -1
},
{
"description": "Expiration Unix timestamp in milliseconds.",
"type": "integer",
"minimum": 1
}
]
}
},
"arguments": [
{
"name": "key",
"type": "key",
"key_spec_index": 0
},
{
"name": "FIELDS",
"type": "string"
},
{
"name": "numfields",
"type": "integer"
},
{
"name": "field",
"type": "string",
"multiple": true
}
]
}
}
{
"HPTTL": {
"summary": "Returns the TTL in milliseconds of a hash field.",
"complexity": "O(N) where N is the number of specified fields",
"group": "hash",
"since": "7.4.0",
"arity": -5,
"function": "hpttlCommand",
"history": [],
"command_flags": [
"READONLY",
"FAST"
],
"acl_categories": [
"HASH"
],
"key_specs": [
{
"flags": [
"RO",
"ACCESS"
],
"begin_search": {
"index": {
"pos": 1
}
},
"find_keys": {
"range": {
"lastkey": 0,
"step": 1,
"limit": 0
}
}
}
],
"reply_schema": {
"description": "Array of results. Returns empty array if the key does not exist.",
"type": "array",
"minItems": 0,
"maxItems": 4294967295,
"items": {
"oneOf": [
{
"description": "The field does not exist.",
"const": -2
},
{
"description": "The field exists but has no associated expire.",
"const": -1
},
{
"description": "TTL in milliseconds.",
"type": "integer",
"minimum": 1
}
]
}
},
"arguments": [
{
"name": "key",
"type": "key",
"key_spec_index": 0
},
{
"name": "FIELDS",
"type": "string"
},
{
"name": "numfields",
"type": "integer"
},
{
"name": "field",
"type": "string",
"multiple": true
}
]
}
}
{
"HTTL": {
"summary": "Returns the TTL in seconds of a hash field.",
"complexity": "O(N) where N is the number of specified fields",
"group": "hash",
"since": "7.4.0",
"arity": -5,
"function": "httlCommand",
"history": [],
"command_flags": [
"READONLY",
"FAST"
],
"acl_categories": [
"HASH"
],
"key_specs": [
{
"flags": [
"RO",
"ACCESS"
],
"begin_search": {
"index": {
"pos": 1
}
},
"find_keys": {
"range": {
"lastkey": 0,
"step": 1,
"limit": 0
}
}
}
],
"reply_schema": {
"description": "Array of results. Returns empty array if the key does not exist.",
"type": "array",
"minItems": 0,
"maxItems": 4294967295,
"items": {
"oneOf": [
{
"description": "The field does not exist.",
"const": -2
},
{
"description": "The field exists but has no associated expire.",
"const": -1
},
{
"description": "TTL in seconds.",
"type": "integer",
"minimum": 1
}
]
}
},
"arguments": [
{
"name": "key",
"type": "key",
"key_spec_index": 0
},
{
"name": "FIELDS",
"type": "string"
},
{
"name": "numfields",
"type": "integer"
},
{
"name": "field",
"type": "string",
"multiple": true
}
]
}
}
......@@ -177,13 +177,13 @@ robj *lookupKeyWriteOrReply(client *c, robj *key, robj *reply) {
*
* If the update_if_existing argument is false, the program is aborted
* if the key already exists, otherwise, it can fall back to dbOverwrite. */
static void dbAddInternal(redisDb *db, robj *key, robj *val, int update_if_existing) {
static dictEntry *dbAddInternal(redisDb *db, robj *key, robj *val, int update_if_existing) {
dictEntry *existing;
int slot = getKeySlot(key->ptr);
dictEntry *de = kvstoreDictAddRaw(db->keys, slot, key->ptr, &existing);
if (update_if_existing && existing) {
dbSetValue(db, key, val, 1, existing);
return;
return existing;
}
serverAssertWithInfo(NULL, key, de != NULL);
kvstoreDictSetKey(db->keys, slot, de, sdsdup(key->ptr));
......@@ -191,10 +191,11 @@ static void dbAddInternal(redisDb *db, robj *key, robj *val, int update_if_exist
kvstoreDictSetVal(db->keys, slot, de, val);
signalKeyAsReady(db, key, val->type);
notifyKeyspaceEvent(NOTIFY_NEW,"new",key,db->id);
return de;
}
void dbAdd(redisDb *db, robj *key, robj *val) {
dbAddInternal(db, key, val, 0);
dictEntry *dbAdd(redisDb *db, robj *key, robj *val) {
return dbAddInternal(db, key, val, 0);
}
/* Returns key's hash slot when cluster mode is enabled, or 0 when disabled.
......@@ -275,6 +276,11 @@ static void dbSetValue(redisDb *db, robj *key, robj *val, int overwrite, dictEnt
old = dictGetVal(de);
}
kvstoreDictSetVal(db->keys, slot, de, val);
/* if hash with HFEs, take care to remove from global HFE DS */
if (old->type == OBJ_HASH)
hashTypeRemoveFromExpires(&db->hexpires, old);
if (server.lazyfree_lazy_server_del) {
freeObjAsync(key,old,db->id);
} else {
......@@ -370,6 +376,11 @@ int dbGenericDelete(redisDb *db, robj *key, int async, int flags) {
dictEntry *de = kvstoreDictTwoPhaseUnlinkFind(db->keys, slot, key->ptr, &plink, &table);
if (de) {
robj *val = dictGetVal(de);
/* If hash object with expiry on fields, remove it from HFE DS of DB */
if (val->type == OBJ_HASH)
hashTypeRemoveFromExpires(&db->hexpires, val);
/* RM_StringDMA may call dbUnshareStringValue which may free val, so we
* need to incr to retain val */
incrRefCount(val);
......@@ -475,6 +486,9 @@ long long emptyDbStructure(redisDb *dbarray, int dbnum, int async,
if (async) {
emptyDbAsync(&dbarray[j]);
} else {
/* Destroy global HFE DS before deleting the hashes since ebuckets
* DS is embedded in the stored objects. */
ebDestroy(&dbarray[j].hexpires, &hashExpireBucketsType, NULL);
kvstoreEmpty(dbarray[j].keys, callback);
kvstoreEmpty(dbarray[j].expires, callback);
}
......@@ -554,6 +568,7 @@ redisDb *initTempDb(void) {
tempDb[i].id = i;
tempDb[i].keys = kvstoreCreate(&dbDictType, slot_count_bits, flags);
tempDb[i].expires = kvstoreCreate(&dbExpiresDictType, slot_count_bits, flags);
tempDb[i].hexpires = ebCreate();
}
return tempDb;
......@@ -566,6 +581,9 @@ void discardTempDb(redisDb *tempDb, void(callback)(dict*)) {
/* Release temp DBs. */
emptyDbStructure(tempDb, -1, async, callback);
for (int i=0; i<server.dbnum; i++) {
/* Destroy global HFE DS before deleting the hashes since ebuckets DS is
* embedded in the stored objects. */
ebDestroy(&tempDb[i].hexpires, &hashExpireBucketsType, NULL);
kvstoreRelease(tempDb[i].keys);
kvstoreRelease(tempDb[i].expires);
}
......@@ -894,6 +912,7 @@ typedef struct {
sds pattern; /* pattern string, NULL means no pattern */
long sampled; /* cumulative number of keys sampled */
int no_values; /* set to 1 means to return keys only */
size_t (*strlen)(char *s); /* (o->type == OBJ_HASH) ? hfieldlen : sdslen */
} scanData;
/* Helper function to compare key type in scan commands */
......@@ -918,7 +937,7 @@ void scanCallback(void *privdata, const dictEntry *de) {
list *keys = data->keys;
robj *o = data->o;
sds val = NULL;
sds key = NULL;
void *key = NULL; /* if OBJ_HASH then key is of type `hfield`. Otherwise, `sds` */
data->sampled++;
/* o and typename can not have values at the same time. */
......@@ -932,24 +951,29 @@ void scanCallback(void *privdata, const dictEntry *de) {
}*/
/* Filter element if it does not match the pattern. */
sds keysds = dictGetKey(de);
void *keyStr = dictGetKey(de);
if (data->pattern) {
if (!stringmatchlen(data->pattern, sdslen(data->pattern), keysds, sdslen(keysds), 0)) {
if (!stringmatchlen(data->pattern, sdslen(data->pattern), keyStr, data->strlen(keyStr), 0)) {
return;
}
}
if (o == NULL) {
key = keysds;
key = keyStr;
} else if (o->type == OBJ_SET) {
key = keysds;
key = keyStr;
} else if (o->type == OBJ_HASH) {
key = keysds;
key = keyStr;
val = dictGetVal(de);
/* If field is expired, then ignore */
if (hfieldIsExpired(key))
return;
} else if (o->type == OBJ_ZSET) {
char buf[MAX_LONG_DOUBLE_CHARS];
int len = ld2string(buf, sizeof(buf), *(double *)dictGetVal(de), LD_STR_AUTO);
key = sdsdup(keysds);
key = sdsdup(keyStr);
val = sdsnewlen(buf, len);
} else {
serverPanic("Type not handled in SCAN callback.");
......@@ -1023,6 +1047,7 @@ char *getObjectTypeName(robj *o) {
* In the case of a Hash object the function returns both the field and value
* of every element on the Hash. */
void scanGenericCommand(client *c, robj *o, unsigned long long cursor) {
int isKeysHfield = 0;
int i, j;
listNode *node;
long count = 10;
......@@ -1103,6 +1128,7 @@ void scanGenericCommand(client *c, robj *o, unsigned long long cursor) {
} else if (o->type == OBJ_SET && o->encoding == OBJ_ENCODING_HT) {
ht = o->ptr;
} else if (o->type == OBJ_HASH && o->encoding == OBJ_ENCODING_HT) {
isKeysHfield = 1;
ht = o->ptr;
} else if (o->type == OBJ_ZSET && o->encoding == OBJ_ENCODING_SKIPLIST) {
zset *zs = o->ptr;
......@@ -1141,7 +1167,7 @@ void scanGenericCommand(client *c, robj *o, unsigned long long cursor) {
* working on an empty dict, one with a lot of empty buckets, and
* for the buckets are not empty, we need to limit the spampled number
* to prevent a long hang time caused by filtering too many keys;
* 6. data.no_values: to control whether values will be returned or
* 6. data.no_values: to control whether values will be returned or
* only keys are returned. */
scanData data = {
.keys = keys,
......@@ -1150,6 +1176,7 @@ void scanGenericCommand(client *c, robj *o, unsigned long long cursor) {
.pattern = use_pattern ? pat : NULL,
.sampled = 0,
.no_values = no_values,
.strlen = (isKeysHfield) ? hfieldlen : sdslen,
};
/* A pattern may restrict all matching keys to one cluster slot. */
......@@ -1211,6 +1238,40 @@ void scanGenericCommand(client *c, robj *o, unsigned long long cursor) {
p = lpNext(o->ptr, p);
}
cursor = 0;
} else if (o->type == OBJ_HASH && o->encoding == OBJ_ENCODING_LISTPACK_EX) {
int64_t len;
long long expire_at;
unsigned char *lp = hashTypeListpackGetLp(o);
unsigned char *p = lpFirst(lp);
unsigned char *str, *val;
unsigned char intbuf[LP_INTBUF_SIZE];
while (p) {
str = lpGet(p, &len, intbuf);
p = lpNext(lp, p);
val = p; /* Keep pointer to value */
p = lpNext(lp, p);
serverAssert(p && lpGetIntegerValue(p, &expire_at));
if (hashTypeIsExpired(o, expire_at) ||
(use_pattern && !stringmatchlen(pat, sdslen(pat), (char *)str, len, 0)))
{
/* jump to the next key/val pair */
p = lpNext(lp, p);
continue;
}
/* add key object */
listAddNodeTail(keys, sdsnewlen(str, len));
/* add value object */
if (!no_values) {
str = lpGet(val, &len, intbuf);
listAddNodeTail(keys, sdsnewlen(str, len));
}
p = lpNext(lp, p);
}
cursor = 0;
} else {
serverPanic("Not handled encoding in SCAN.");
}
......@@ -1243,10 +1304,14 @@ void scanGenericCommand(client *c, robj *o, unsigned long long cursor) {
addReplyArrayLen(c, 2);
addReplyBulkLongLong(c,cursor);
unsigned long long idx = 0;
addReplyArrayLen(c, listLength(keys));
while ((node = listFirst(keys)) != NULL) {
sds key = listNodeValue(node);
addReplyBulkCBuffer(c, key, sdslen(key));
void *key = listNodeValue(node);
/* For HSCAN, list will contain keys value pairs unless no_values arg
* was given. We should call mstrlen for the keys only. */
int hfieldkey = isKeysHfield && (no_values || (idx++ % 2 == 0));
addReplyBulkCBuffer(c, key, hfieldkey ? mstrlen(key) : sdslen(key));
listDelNode(keys, node);
}
......@@ -1339,6 +1404,7 @@ void renameGenericCommand(client *c, int nx) {
robj *o;
long long expire;
int samekey = 0;
uint64_t minHashExpireTime = EB_EXPIRE_TIME_INVALID;
/* When source and dest key is the same, no operation is performed,
* if the key exists, however we still return an error on unexisting key. */
......@@ -1364,9 +1430,21 @@ void renameGenericCommand(client *c, int nx) {
* with the same name. */
dbDelete(c->db,c->argv[2]);
}
dbAdd(c->db,c->argv[2],o);
dictEntry *de = dbAdd(c->db, c->argv[2], o);
if (expire != -1) setExpire(c,c->db,c->argv[2],expire);
/* If hash with expiration on fields then remove it from global HFE DS and
* keep next expiration time. Otherwise, dbDelete() will remove it from the
* global HFE DS and we will lose the expiration time. */
if (o->type == OBJ_HASH)
minHashExpireTime = hashTypeRemoveFromExpires(&c->db->hexpires, o);
dbDelete(c->db,c->argv[1]);
/* If hash with HFEs, register in db->hexpires */
if (minHashExpireTime != EB_EXPIRE_TIME_INVALID)
hashTypeAddToExpires(c->db, dictGetKey(de), o, minHashExpireTime);
signalModifiedKey(c,c->db,c->argv[1]);
signalModifiedKey(c,c->db,c->argv[2]);
notifyKeyspaceEvent(NOTIFY_GENERIC,"rename_from",
......@@ -1390,6 +1468,7 @@ void moveCommand(client *c) {
redisDb *src, *dst;
int srcid, dbid;
long long expire;
uint64_t hashExpireTime = EB_EXPIRE_TIME_INVALID;
if (server.cluster_enabled) {
addReplyError(c,"MOVE is not allowed in cluster mode");
......@@ -1430,12 +1509,25 @@ void moveCommand(client *c) {
addReply(c,shared.czero);
return;
}
dbAdd(dst,c->argv[1],o);
dictEntry *dstDictEntry = dbAdd(dst,c->argv[1],o);
if (expire != -1) setExpire(c,dst,c->argv[1],expire);
/* If hash with expiration on fields, remove it from global HFE DS and keep
* aside registered expiration time. Must be before deletion of the object.
* hexpires (ebuckets) embed in stored items its structure. */
if (o->type == OBJ_HASH)
hashExpireTime = hashTypeRemoveFromExpires(&src->hexpires, o);
incrRefCount(o);
/* OK! key moved, free the entry in the source DB */
dbDelete(src,c->argv[1]);
/* If object of type hash with expiration on fields. Taken care to add the
* hash to hexpires of `dst` only after dbDelete(). */
if (hashExpireTime != EB_EXPIRE_TIME_INVALID)
hashTypeAddToExpires(dst, dictGetKey(dstDictEntry), o, hashExpireTime);
signalModifiedKey(c,src,c->argv[1]);
signalModifiedKey(c,dst,c->argv[1]);
notifyKeyspaceEvent(NOTIFY_GENERIC,
......@@ -1518,12 +1610,13 @@ void copyCommand(client *c) {
/* Duplicate object according to object's type. */
robj *newobj;
uint64_t minHashExpire = EB_EXPIRE_TIME_INVALID; /* HFE feature */
switch(o->type) {
case OBJ_STRING: newobj = dupStringObject(o); break;
case OBJ_LIST: newobj = listTypeDup(o); break;
case OBJ_SET: newobj = setTypeDup(o); break;
case OBJ_ZSET: newobj = zsetDup(o); break;
case OBJ_HASH: newobj = hashTypeDup(o); break;
case OBJ_HASH: newobj = hashTypeDup(o, newkey->ptr, &minHashExpire); break;
case OBJ_STREAM: newobj = streamDup(o); break;
case OBJ_MODULE:
newobj = moduleTypeDupOrReply(c, key, newkey, dst->id, o);
......@@ -1538,8 +1631,16 @@ void copyCommand(client *c) {
dbDelete(dst,newkey);
}
dbAdd(dst,newkey,newobj);
if (expire != -1) setExpire(c, dst, newkey, expire);
dictEntry *deCopy = dbAdd(dst,newkey,newobj);
/* if key with expiration then set it */
if (expire != -1)
setExpire(c, dst, newkey, expire);
/* If minExpiredField was set, then the object is hash with expiration
* on fields and need to register it in global HFE DS */
if (minHashExpire != EB_EXPIRE_TIME_INVALID)
hashTypeAddToExpires(dst, dictGetKey(deCopy), newobj, minHashExpire);
/* OK! key copied */
signalModifiedKey(c,dst,c->argv[2]);
......@@ -1629,11 +1730,13 @@ int dbSwapDatabases(int id1, int id2) {
* remain in the same DB they were. */
db1->keys = db2->keys;
db1->expires = db2->expires;
db1->hexpires = db2->hexpires;
db1->avg_ttl = db2->avg_ttl;
db1->expires_cursor = db2->expires_cursor;
db2->keys = aux.keys;
db2->expires = aux.expires;
db2->hexpires = aux.hexpires;
db2->avg_ttl = aux.avg_ttl;
db2->expires_cursor = aux.expires_cursor;
......@@ -1671,11 +1774,13 @@ void swapMainDbWithTempDb(redisDb *tempDb) {
* remain in the same DB they were. */
activedb->keys = newdb->keys;
activedb->expires = newdb->expires;
activedb->hexpires = newdb->hexpires;
activedb->avg_ttl = newdb->avg_ttl;
activedb->expires_cursor = newdb->expires_cursor;
newdb->keys = aux.keys;
newdb->expires = aux.expires;
newdb->hexpires = aux.hexpires;
newdb->avg_ttl = aux.avg_ttl;
newdb->expires_cursor = aux.expires_cursor;
......@@ -1864,7 +1969,7 @@ int keyIsExpired(redisDb *db, robj *key) {
* EXPIRE_AVOID_DELETE_EXPIRED flag.
*
* The return value of the function is KEY_VALID if the key is still valid.
* The function returns KEY_EXPIRED if the key is expired BUT not deleted,
* The function returns KEY_EXPIRED if the key is expired BUT not deleted,
* or returns KEY_DELETED if the key is expired and deleted. */
keyStatus expireIfNeeded(redisDb *db, robj *key, int flags) {
if (server.lazy_expire_disabled) return KEY_VALID;
......@@ -1878,7 +1983,7 @@ keyStatus expireIfNeeded(redisDb *db, robj *key, int flags) {
* replicas.
*
* Still we try to return the right information to the caller,
* that is, KEY_VALID if we think the key should still be valid,
* that is, KEY_VALID if we think the key should still be valid,
* KEY_EXPIRED if we think the key is expired but don't want to delete it at this time.
*
* When replicating commands from the master, keys are never considered
......
......@@ -200,17 +200,22 @@ void xorObjectDigest(redisDb *db, robj *keyobj, unsigned char *digest, robj *o)
}
} else if (o->type == OBJ_HASH) {
hashTypeIterator *hi = hashTypeInitIterator(o);
while (hashTypeNext(hi) != C_ERR) {
while (hashTypeNext(hi, 0) != C_ERR) {
unsigned char eledigest[20];
sds sdsele;
/* field */
memset(eledigest,0,20);
sdsele = hashTypeCurrentObjectNewSds(hi,OBJ_HASH_KEY);
mixDigest(eledigest,sdsele,sdslen(sdsele));
sdsfree(sdsele);
/* val */
sdsele = hashTypeCurrentObjectNewSds(hi,OBJ_HASH_VALUE);
mixDigest(eledigest,sdsele,sdslen(sdsele));
sdsfree(sdsele);
/* hash-field expiration (HFE) */
if (hi->expire_time != EB_EXPIRE_TIME_INVALID)
xorDigest(eledigest,"!!hexpire!!",11);
xorDigest(digest,eledigest,20);
}
hashTypeReleaseIterator(hi);
......@@ -445,9 +450,9 @@ void debugCommand(client *c) {
"SEGFAULT",
" Crash the server with sigsegv.",
"SET-ACTIVE-EXPIRE <0|1>",
" Setting it to 0 disables expiring keys in background when they are not",
" accessed (otherwise the Redis behavior). Setting it to 1 reenables back the",
" default.",
" Setting it to 0 disables expiring keys (and hash-fields) in background ",
" when they are not accessed (otherwise the Redis behavior). Setting it",
" to 1 reenables back the default.",
"QUICKLIST-PACKED-THRESHOLD <size>",
" Sets the threshold for elements to be inserted as plain vs packed nodes",
" Default value is 1GB, allows values up to 4GB. Setting to 0 restores to default.",
......@@ -664,10 +669,14 @@ NULL
if ((o = objectCommandLookupOrReply(c,c->argv[2],shared.nokeyerr))
== NULL) return;
if (o->encoding != OBJ_ENCODING_LISTPACK) {
if (o->encoding != OBJ_ENCODING_LISTPACK && o->encoding != OBJ_ENCODING_LISTPACK_EX) {
addReplyError(c,"Not a listpack encoded object.");
} else {
lpRepr(o->ptr);
if (o->encoding == OBJ_ENCODING_LISTPACK)
lpRepr(o->ptr);
else if (o->encoding == OBJ_ENCODING_LISTPACK_EX)
lpRepr(((listpackEx*)o->ptr)->lp);
addReplyStatus(c,"Listpack structure printed on stdout");
}
} else if (!strcasecmp(c->argv[1]->ptr,"quicklist") && (c->argc == 3 || c->argc == 4)) {
......@@ -1081,7 +1090,7 @@ void serverLogObjectDebugInfo(const robj *o) {
} else if (o->type == OBJ_SET) {
serverLog(LL_WARNING,"Set size: %d", (int) setTypeSize(o));
} else if (o->type == OBJ_HASH) {
serverLog(LL_WARNING,"Hash size: %d", (int) hashTypeLength(o));
serverLog(LL_WARNING,"Hash size: %d", (int) hashTypeLength(o, 0));
} else if (o->type == OBJ_ZSET) {
serverLog(LL_WARNING,"Sorted set size: %d", (int) zsetLength(o));
if (o->encoding == OBJ_ENCODING_SKIPLIST)
......
......@@ -70,6 +70,22 @@ sds activeDefragSds(sds sdsptr) {
return NULL;
}
/* Defrag helper for hfield strings
*
* returns NULL in case the allocation wasn't moved.
* when it returns a non-null value, the old pointer was already released
* and should NOT be accessed. */
hfield activeDefragHfield(hfield hf) {
void *ptr = hfieldGetAllocPtr(hf);
void *newptr = activeDefragAlloc(ptr);
if (newptr) {
size_t offset = hf - (char*)ptr;
hf = (char*)newptr + offset;
return hf;
}
return NULL;
}
/* Defrag helper for robj and/or string objects with expected refcount.
*
* Like activeDefragStringOb, but it requires the caller to pass in the expected
......@@ -250,6 +266,31 @@ void activeDefragSdsDictCallback(void *privdata, const dictEntry *de) {
UNUSED(de);
}
void activeDefragHfieldDictCallback(void *privdata, const dictEntry *de) {
dict *d = privdata;
hfield newhf, hf = dictGetKey(de);
if (hfieldGetExpireTime(hf) == EB_EXPIRE_TIME_INVALID) {
/* If the hfield does not have TTL, we directly defrag it. */
newhf = activeDefragHfield(hf);
} else {
/* Update its reference in the ebucket while defragging it. */
ebuckets *eb = hashTypeGetDictMetaHFE(d);
newhf = ebDefragItem(eb, &hashFieldExpireBucketsType, hf, (ebDefragFunction *)activeDefragHfield);
}
if (newhf) {
/* We can't search in dict for that key after we've released
* the pointer it holds, since it won't be able to do the string
* compare, but we can find the entry using key hash and pointer. */
dictUseStoredKeyApi(d, 1);
uint64_t hash = dictGetHash(d, newhf);
dictUseStoredKeyApi(d, 0);
dictEntry *de = dictFindEntryByPtrAndHash(d, hf, hash);
serverAssert(de);
dictSetKey(d, de, newhf);
}
}
/* Defrag a dict with sds key and optional value (either ptr, sds or robj string) */
void activeDefragSdsDict(dict* d, int val_type) {
unsigned long cursor = 0;
......@@ -268,6 +309,20 @@ void activeDefragSdsDict(dict* d, int val_type) {
} while (cursor != 0);
}
/* Defrag a dict with hfield key and sds value. */
void activeDefragHfieldDict(dict *d) {
unsigned long cursor = 0;
dictDefragFunctions defragfns = {
.defragAlloc = activeDefragAlloc,
.defragKey = NULL, /* Will be defragmented in activeDefragHfieldDictCallback. */
.defragVal = (dictDefragAllocFunction *)activeDefragSds
};
do {
cursor = dictScanDefrag(d, cursor, activeDefragHfieldDictCallback,
&defragfns, d);
} while (cursor != 0);
}
/* Defrag a list of ptr, sds or robj string values */
void activeDefragList(list *l, int val_type) {
listNode *ln, *newln;
......@@ -422,10 +477,10 @@ void scanLaterHash(robj *ob, unsigned long *cursor) {
dict *d = ob->ptr;
dictDefragFunctions defragfns = {
.defragAlloc = activeDefragAlloc,
.defragKey = (dictDefragAllocFunction *)activeDefragSds,
.defragKey = NULL, /* Will be defragmented in activeDefragHfieldDictCallback. */
.defragVal = (dictDefragAllocFunction *)activeDefragSds
};
*cursor = dictScanDefrag(d, *cursor, scanCallbackCountScanned, &defragfns, NULL);
*cursor = dictScanDefrag(d, *cursor, activeDefragHfieldDictCallback, &defragfns, d);
}
void defragQuicklist(redisDb *db, dictEntry *kde) {
......@@ -477,7 +532,7 @@ void defragHash(redisDb *db, dictEntry *kde) {
if (dictSize(d) > server.active_defrag_max_scan_fields)
defragLater(db, kde);
else
activeDefragSdsDict(d, DEFRAG_SDS_DICT_VAL_IS_SDS);
activeDefragHfieldDict(d);
/* defrag the dict struct and tables */
if ((newd = dictDefragTables(ob->ptr)))
ob->ptr = newd;
......@@ -672,7 +727,7 @@ void defragModule(redisDb *db, dictEntry *kde) {
* all the various pointers it has. */
void defragKey(defragCtx *ctx, dictEntry *de) {
sds keysds = dictGetKey(de);
robj *newob, *ob;
robj *newob, *ob = dictGetVal(de);
unsigned char *newzl;
sds newsds;
redisDb *db = ctx->privdata;
......@@ -689,11 +744,22 @@ void defragKey(defragCtx *ctx, dictEntry *de) {
dictEntry *expire_de = kvstoreDictFindEntryByPtrAndHash(db->expires, slot, keysds, hash);
if (expire_de) kvstoreDictSetKey(db->expires, slot, expire_de, newsds);
}
/* Update the key's reference in the dict's metadata or the listpackEx. */
if (unlikely(ob->type == OBJ_HASH))
hashTypeUpdateKeyRef(ob, newsds);
}
/* Try to defrag robj and / or string value. */
ob = dictGetVal(de);
if ((newob = activeDefragStringOb(ob))) {
if (unlikely(ob->type == OBJ_HASH && hashTypeGetMinExpire(ob) != EB_EXPIRE_TIME_INVALID)) {
/* Update its reference in the ebucket while defragging it. */
newob = ebDefragItem(&db->hexpires, &hashExpireBucketsType, ob,
(ebDefragFunction *)activeDefragStringOb);
} else {
/* If the dict doesn't have metadata, we directly defrag it. */
newob = activeDefragStringOb(ob);
}
if (newob) {
kvstoreDictSetVal(db->keys, slot, de, newob);
ob = newob;
}
......@@ -734,6 +800,12 @@ void defragKey(defragCtx *ctx, dictEntry *de) {
if (ob->encoding == OBJ_ENCODING_LISTPACK) {
if ((newzl = activeDefragAlloc(ob->ptr)))
ob->ptr = newzl;
} else if (ob->encoding == OBJ_ENCODING_LISTPACK_EX) {
listpackEx *newlpt, *lpt = (listpackEx*)ob->ptr;
if ((newlpt = activeDefragAlloc(lpt)))
ob->ptr = lpt = newlpt;
if ((newzl = activeDefragAlloc(lpt->lp)))
lpt->lp = newzl;
} else if (ob->encoding == OBJ_ENCODING_HT) {
defragHash(db, de);
} else {
......
......@@ -67,6 +67,25 @@ static int _dictInit(dict *d, dictType *type);
static dictEntry *dictGetNext(const dictEntry *de);
static dictEntry **dictGetNextRef(dictEntry *de);
static void dictSetNext(dictEntry *de, dictEntry *next);
static int dictDefaultCompare(dict *d, const void *key1, const void *key2);
/* -------------------------- misc inline functions -------------------------------- */
typedef int (*keyCmpFunc)(dict *d, const void *key1, const void *key2);
static inline keyCmpFunc dictGetKeyCmpFunc(dict *d) {
if (d->useStoredKeyApi && d->type->storedKeyCompare)
return d->type->storedKeyCompare;
if (d->type->keyCompare)
return d->type->keyCompare;
return dictDefaultCompare;
}
static inline uint64_t dictHashKey(dict *d, const void *key, int isStoredKey) {
if (isStoredKey && d->type->storedHashFunction)
return d->type->storedHashFunction(key);
else
return d->type->hashFunction(key);
}
/* -------------------------- hash functions -------------------------------- */
......@@ -173,6 +192,19 @@ dict *dictCreate(dictType *type)
return d;
}
/* Change dictType of dict to another one with metadata support
* Rest of dictType's values must stay the same */
void dictTypeAddMeta(dict **d, dictType *typeWithMeta) {
/* Verify new dictType is compatible with the old one */
dictType toCmp = *typeWithMeta;
toCmp.dictMetadataBytes = NULL; /* Expected old one not to have metadata */
toCmp.onDictRelease = (*d)->type->onDictRelease; /* Ignore 'onDictRelease' in comparison */
assert(memcmp((*d)->type, &toCmp, sizeof(dictType)) == 0); /* The rest of the dictType fields must be the same */
*d = zrealloc(*d, sizeof(dict) + typeWithMeta->dictMetadataBytes(*d));
(*d)->type = typeWithMeta;
}
/* Initialize the hash table */
int _dictInit(dict *d, dictType *type)
{
......@@ -182,6 +214,7 @@ int _dictInit(dict *d, dictType *type)
d->rehashidx = -1;
d->pauserehash = 0;
d->pauseAutoResize = 0;
d->useStoredKeyApi = 0;
return DICT_OK;
}
......@@ -285,7 +318,7 @@ static void rehashEntriesInBucketAtIndex(dict *d, uint64_t idx) {
void *key = dictGetKey(de);
/* Get the index in the new hash table */
if (d->ht_size_exp[1] > d->ht_size_exp[0]) {
h = dictHashKey(d, key) & DICTHT_SIZE_MASK(d->ht_size_exp[1]);
h = dictHashKey(d, key, 1) & DICTHT_SIZE_MASK(d->ht_size_exp[1]);
} else {
/* We're shrinking the table. The tables sizes are powers of
* two, so we simply mask the bucket index in the larger table
......@@ -572,7 +605,7 @@ static dictEntry *dictGenericDelete(dict *d, const void *key, int nofree) {
/* dict is empty */
if (dictSize(d) == 0) return NULL;
h = dictHashKey(d, key);
h = dictHashKey(d, key, d->useStoredKeyApi);
idx = h & DICTHT_SIZE_MASK(d->ht_size_exp[0]);
if (dictIsRehashing(d)) {
......@@ -587,6 +620,8 @@ static dictEntry *dictGenericDelete(dict *d, const void *key, int nofree) {
}
}
keyCmpFunc cmpFunc = dictGetKeyCmpFunc(d);
for (table = 0; table <= 1; table++) {
if (table == 0 && (long)idx < d->rehashidx) continue;
idx = h & DICTHT_SIZE_MASK(d->ht_size_exp[table]);
......@@ -594,7 +629,7 @@ static dictEntry *dictGenericDelete(dict *d, const void *key, int nofree) {
prevHe = NULL;
while(he) {
void *he_key = dictGetKey(he);
if (key == he_key || dictCompareKeys(d, key, he_key)) {
if (key == he_key || cmpFunc(d, key, he_key)) {
/* Unlink the element from the list */
if (prevHe)
dictSetNext(prevHe, dictGetNext(he));
......@@ -689,6 +724,10 @@ void dictRelease(dict *d)
* destroying the dict fake completion. */
if (dictIsRehashing(d) && d->type->rehashingCompleted)
d->type->rehashingCompleted(d);
if (d->type->onDictRelease)
d->type->onDictRelease(d);
_dictClear(d,0,NULL);
_dictClear(d,1,NULL);
zfree(d);
......@@ -701,8 +740,9 @@ dictEntry *dictFind(dict *d, const void *key)
if (dictSize(d) == 0) return NULL; /* dict is empty */
h = dictHashKey(d, key);
h = dictHashKey(d, key, d->useStoredKeyApi);
idx = h & DICTHT_SIZE_MASK(d->ht_size_exp[0]);
keyCmpFunc cmpFunc = dictGetKeyCmpFunc(d);
if (dictIsRehashing(d)) {
if ((long)idx >= d->rehashidx && d->ht_table[0][idx]) {
......@@ -722,7 +762,7 @@ dictEntry *dictFind(dict *d, const void *key)
he = d->ht_table[table][idx];
while(he) {
void *he_key = dictGetKey(he);
if (key == he_key || dictCompareKeys(d, key, he_key))
if (key == he_key || cmpFunc(d, key, he_key))
return he;
he = dictGetNext(he);
}
......@@ -759,7 +799,9 @@ dictEntry *dictTwoPhaseUnlinkFind(dict *d, const void *key, dictEntry ***plink,
if (dictSize(d) == 0) return NULL; /* dict is empty */
if (dictIsRehashing(d)) _dictRehashStep(d);
h = dictHashKey(d, key);
h = dictHashKey(d, key, d->useStoredKeyApi);
keyCmpFunc cmpFunc = dictGetKeyCmpFunc(d);
for (table = 0; table <= 1; table++) {
idx = h & DICTHT_SIZE_MASK(d->ht_size_exp[table]);
......@@ -767,7 +809,7 @@ dictEntry *dictTwoPhaseUnlinkFind(dict *d, const void *key, dictEntry ***plink,
dictEntry **ref = &d->ht_table[table][idx];
while (ref && *ref) {
void *de_key = dictGetKey(*ref);
if (key == de_key || dictCompareKeys(d, key, de_key)) {
if (key == de_key || cmpFunc(d, key, de_key)) {
*table_index = table;
*plink = ref;
dictPauseRehashing(d);
......@@ -1530,8 +1572,8 @@ static signed char _dictNextExp(unsigned long size)
void *dictFindPositionForInsert(dict *d, const void *key, dictEntry **existing) {
unsigned long idx, table;
dictEntry *he;
uint64_t hash = dictHashKey(d, key, d->useStoredKeyApi);
if (existing) *existing = NULL;
uint64_t hash = dictHashKey(d, key);
idx = hash & DICTHT_SIZE_MASK(d->ht_size_exp[0]);
if (dictIsRehashing(d)) {
......@@ -1548,6 +1590,8 @@ void *dictFindPositionForInsert(dict *d, const void *key, dictEntry **existing)
/* Expand the hash table if needed */
_dictExpandIfNeeded(d);
keyCmpFunc cmpFunc = dictGetKeyCmpFunc(d);
for (table = 0; table <= 1; table++) {
if (table == 0 && (long)idx < d->rehashidx) continue;
idx = hash & DICTHT_SIZE_MASK(d->ht_size_exp[table]);
......@@ -1555,7 +1599,7 @@ void *dictFindPositionForInsert(dict *d, const void *key, dictEntry **existing)
he = d->ht_table[table][idx];
while(he) {
void *he_key = dictGetKey(he);
if (key == he_key || dictCompareKeys(d, key, he_key)) {
if (key == he_key || cmpFunc(d, key, he_key)) {
if (existing) *existing = he;
return NULL;
}
......@@ -1587,7 +1631,7 @@ void dictSetResizeEnabled(dictResizeEnable enable) {
}
uint64_t dictGetHash(dict *d, const void *key) {
return dictHashKey(d, key);
return dictHashKey(d, key, d->useStoredKeyApi);
}
/* Finds the dictEntry using pointer and pre-calculated hash.
......@@ -1732,6 +1776,11 @@ void dictGetStats(char *buf, size_t bufsize, dict *d, int full) {
orig_buf[orig_bufsize-1] = '\0';
}
static int dictDefaultCompare(dict *d, const void *key1, const void *key2) {
(void)(d); /*unused*/
return key1 == key2;
}
/* ------------------------------- Benchmark ---------------------------------*/
#ifdef REDIS_TEST
......
......@@ -62,6 +62,32 @@ typedef struct dictType {
unsigned int keys_are_odd:1;
/* TODO: Add a 'keys_are_even' flag and use a similar optimization if that
* flag is set. */
/* Sometimes we want the ability to store a key in a given way inside the hash
* function, and lookup it in some other way without resorting to any kind of
* conversion. For instance the key may be stored as a structure also
* representing other things, but the lookup happens via just a pointer to a
* null terminated string. Optionally providing additional hash/cmp functions,
* dict supports such usage. In that case we'll have a hashFunction() that will
* expect a null terminated C string, and a storedHashFunction() that will
* instead expect the structure. Similarly, the two comparison functions will
* work differently. The keyCompare() will treat the first argument as a pointer
* to a C string and the other as a structure (this way we can directly lookup
* the structure key using the C string). While the storedKeyCompare() will
* check if two pointers to the key in structure form are the same.
*
* However, functions of dict that gets key as argument (void *key) don't get
* any indication whether it is a lookup or stored key. To indicate that
* you intend to use key of type stored-key, and, consequently, use
* dedicated compare and hash functions of stored-key, is by calling
* dictUseStoredKeyApi(1) before using any of the dict functions that gets
* key as a parameter and then call again dictUseStoredKeyApi(0) once done.
*
* Set to NULL both functions, if you don't want to support this feature. */
uint64_t (*storedHashFunction)(const void *key);
int (*storedKeyCompare)(dict *d, const void *key1, const void *key2);
/* Optional callback called when the dict is destroyed. */
void (*onDictRelease)(dict *d);
} dictType;
#define DICTHT_SIZE(exp) ((exp) == -1 ? 0 : (unsigned long)1<<(exp))
......@@ -76,7 +102,9 @@ struct dict {
long rehashidx; /* rehashing not in progress if rehashidx == -1 */
/* Keep small vars at end for optimal (minimal) struct padding */
int16_t pauserehash; /* If >0 rehashing is paused (<0 indicates coding error) */
unsigned pauserehash : 15; /* If >0 rehashing is paused */
unsigned useStoredKeyApi : 1; /* See comment of storedHashFunction above */
signed char ht_size_exp[2]; /* exponent of size. (size = 1<<exp) */
int16_t pauseAutoResize; /* If >0 automatic resizing is disallowed (<0 indicates coding error) */
void *metadata[];
......@@ -136,7 +164,6 @@ typedef struct {
#define dictMetadataSize(d) ((d)->type->dictMetadataBytes \
? (d)->type->dictMetadataBytes(d) : 0)
#define dictHashKey(d, key) ((d)->type->hashFunction(key))
#define dictBuckets(d) (DICTHT_SIZE((d)->ht_size_exp[0])+DICTHT_SIZE((d)->ht_size_exp[1]))
#define dictSize(d) ((d)->ht_used[0]+(d)->ht_used[1])
#define dictIsEmpty(d) ((d)->ht_used[0] == 0 && (d)->ht_used[1] == 0)
......@@ -146,6 +173,7 @@ typedef struct {
#define dictIsRehashingPaused(d) ((d)->pauserehash > 0)
#define dictPauseAutoResize(d) ((d)->pauseAutoResize++)
#define dictResumeAutoResize(d) ((d)->pauseAutoResize--)
#define dictUseStoredKeyApi(d, flag) ((d)->useStoredKeyApi = (flag))
/* If our unsigned long type can store a 64 bit number, use a 64 bit PRNG. */
#if ULONG_MAX >= 0xffffffffffffffff
......@@ -162,6 +190,7 @@ typedef enum {
/* API */
dict *dictCreate(dictType *type);
void dictTypeAddMeta(dict **d, dictType *typeWithMeta);
int dictExpand(dict *d, unsigned long size);
int dictTryExpand(dict *d, unsigned long size);
int dictShrink(dict *d, unsigned long size);
......
This diff is collapsed.
/*
* Copyright Redis Ltd. 2024 - present
*
* Licensed under your choice of the Redis Source Available License 2.0 (RSALv2)
* or the Server Side Public License v1 (SSPLv1).
*
*
* WHAT IS EBUCKETS?
* -----------------
* ebuckets is being used to store items that are set with expiration-time. It
* supports the basic API of add, remove and active expiration. The implementation
* of it is based on rax-tree, or plain linked-list when small. The expiration time
* of the items are used as the key to traverse rax-tree.
*
* Instead of holding a distinct item in each leaf of the rax-tree we can aggregate
* items into small segments and hold it in each leaf. This way we can avoid
* frequent modification of the rax-tree, since many of the modifications
* will be done only at the segment level. It will also save memory because
* rax-tree can be costly, around 40 bytes per leaf (with rax-key limited to 6
* bytes). Whereas each additional item in the segment will cost the size of the
* 'next' pointer in a list (8 bytes) and few more bytes for maintenance of the
* segment.
*
* EBUCKETS STRUCTURE
* ------------------
* The ebuckets data structure is organized in a hierarchical manner as follows:
*
* 1. ebuckets: This is the top-level data structure. It can be either a rax tree
* or a plain linked list. It contains one or more buckets, each representing
* an interval in time.
*
* 2. bucket: Each bucket represents an interval in time and contains one or more
* segments. The key in the rax-tree for each bucket represents low
* bound expiration-time for the items within this bucket. The key of the
* following bucket represents the upper bound expiration-time.
*
* 3. segment: Each segment within a bucket can hold up to `EB_SEG_MAX_ITEMS`
* items as a linked list. If there are more, the segment will try to
* split the bucket. To avoid wasting memory, it is a singly linked list (only
* next-item pointer). It is a cyclic linked-list to allow efficient removal of
* items from the middle of the segment without traversing the rax tree.
*
* 4. item: Each item that is stored in ebuckets should embed the ExpireMeta
* struct and supply getter function (see EbucketsType.getExpireMeta). This
* struct holds the expire-time of the item and few more fields that are used
* to maintain the segments data-structure.
*
* SPLITTING BUCKET
* ----------------
* Each segment can hold up-to `EB_SEG_MAX_ITEMS` items. On insertion of new
* item, it will try to split the segment. Here is an example For adding item
* with expiration of 42 to a segment that already reached its maximum capacity
* which will cause to split of the segment and in turn split of the bucket as
* well to a finer grained ranges:
*
* BUCKETS BUCKETS
* [ 00-10 ] -> size(Seg0) = 11 ==> [ 00-10 ] -> size(Seg0) = 11
* [ 11-76 ] -> size(Seg1) = 16 [ 11-36 ] -> size(Seg1) = 9
* [ 37-76 ] -> size(Seg2) = 7
*
* EXTENDING BUCKET
* ----------------
* In the example above, the reason it wasn't split evenly is that Seg1 must have
* been holding items with same TTL and they must reside together in the same
* bucket after the split. Which brings us to another important point. If there
* is a segment that reached its maximum capacity and all the items have same
* expiration-time key, then we cannot split the bucket but aggregate all the
* items, with same expiration time key, by allocating an extended-segment and
* chain it to the first segment in visited bucket. In that sense, extended
* segments will only hold items with same expiration-time key.
*
* BUCKETS BUCKETS
* [ 00-10 ] -> size(Seg0)=11 ==> [ 00-10 ] -> size(Seg0)=11
* [ 11-12 ] -> size(Seg1)=16 [ 11-12 ] -> size(Seg1)=1 -> size(Seg2)=16
*
* LIMITING RAX TREE DEPTH
* -----------------------
* The rax tree is basically a B-tree and its depth is bounded by the sizeof of
* the key. Holding 6 bytes for expiration-time key is more than enough to represent
* unix-time in msec, and in turn the depth of the tree is limited to 6 levels.
* At a first glance it might look sufficient but we need take into consideration
* the heavyweight maintenance and traversal of each node in the B-tree.
*
* And so, we can further prune the tree such that holding keys with msec precision
* in the tree doesn't bring with it much value. The active-expiration operation can
* live with deletion of expired items, say, older than 1 sec, which means the size
* of time-expiration keys to the rax tree become no more than ~4.5 bytes and we
* also get rid of the "noisy" bits which most probably will cause to yet another
* branching and modification of the rax tree in case of items with time-expiration
* difference of less than 1 second. The lazy expiration will still be precise and
* without compromise on accuracy because the exact expiration-time is kept
* attached as well to each item, in `ExpireMeta`, and each traversal of item with
* expiration will behave as expected down to the msec. Take care to configure
* `EB_BUCKET_KEY_PRECISION` according to your needs.
*
* EBUCKET KEY
* -----------
* Taking into account configured value of `EB_BUCKET_KEY_PRECISION`, two items
* with expiration-time t1 and t2 will be considered to have the same key in the
* rax-tree/buckets if and only if:
*
* EB_BUCKET_KEY(t1) == EB_BUCKET_KEY(t2)
*
* EBUCKETS CREATION
* -----------------
* To avoid the cost of allocating rax data-structure for only few elements,
* ebuckets will start as a simple linked-list and only when it reaches some
* threshold, it will be converted to rax.
*
* TODO
* ----
* - ebRemove() optimize to merge small segments into one segment.
* - ebAdd() Fix pathological case of cascade addition of items into rax such
* that their values are smaller/bigger than visited extended-segment which ends
* up with multiple segments with a single item in each segment.
*/
#ifndef __EBUCKETS_H
#define __EBUCKETS_H
#include <stdlib.h>
#include <sys/types.h>
#include <stdarg.h>
#include <stdint.h>
#include "rax.h"
/*
* EB_BUCKET_KEY_PRECISION - Defines the number of bits to ignore from the
* expiration-time when mapping to buckets. The higher the value, the more items
* with similar expiration-time will be aggregated into the same bucket. The lower
* the value, the more "accurate" the active expiration of buckets will be.
*
* Note that the accurate time expiration of each item is preserved anyway and
* enforced by lazy expiration. It only impacts the active expiration that will
* be able to work on buckets older than (1<<EB_BUCKET_KEY_PRECISION) msec ago.
* For example if EB_BUCKET_KEY_PRECISION is 10, then active expiration
* will work only on buckets that already got expired at least 1sec ago.
*
* The idea of it is to trim the rax tree depth, avoid having too many branches,
* and reduce frequent modifications of the tree to the minimum.
*/
#define EB_BUCKET_KEY_PRECISION 0 /* TBD: modify to 10 */
/* From expiration time to bucket-key */
#define EB_BUCKET_KEY(exptime) ((exptime) >> EB_BUCKET_KEY_PRECISION)
#define EB_EXPIRE_TIME_MAX ((uint64_t)0x0000FFFFFFFFFFFF) /* Maximum expire-time. */
#define EB_EXPIRE_TIME_INVALID (EB_EXPIRE_TIME_MAX+1) /* assumed bigger than max */
/* Handler to ebuckets DS. Pointer to a list, rax or NULL (empty DS). See also ebIsList(). */
typedef void *ebuckets;
/* Users of ebuckets will store `eItem` which is just a void pointer to their
* element. In addition, eItem should embed the ExpireMeta struct and supply
* getter function (see EbucketsType.getExpireMeta).
*/
typedef void *eItem;
/* This struct Should be embedded inside `eItem` and must be aligned in memory. */
typedef struct ExpireMeta {
/* 48bits of unix-time in msec. This value is sufficient to represent, in
* unix-time, until the date of 02 August, 10889
*/
uint32_t expireTimeLo; /* Low bits of expireTime. */
uint16_t expireTimeHi; /* High bits of expireTime. */
unsigned int lastInSegment : 1; /* Last item in segment. If set, then 'next' will
point to the NextSegHdr, unless lastItemBucket=1
then it will point to segment header of the
current segment. */
unsigned int firstItemBucket : 1; /* First item in bucket. This flag assist
to manipulate segments directly without
the need to traverse from start the
rax tree */
unsigned int lastItemBucket : 1; /* Last item in bucket. This flag assist
to manipulate segments directly without
the need to traverse from start the
rax tree */
unsigned int numItems : 5; /* Only first item in segment will maintain
this value. */
unsigned int trash : 1; /* This flag indicates whether the ExpireMeta
associated with the item is leftover.
There is always a potential to reuse the
item after removal/deletion. Note that,
the user can still safely O(1) TTL lookup
a given item and verify whether attached
TTL is valid or leftover. See function
ebGetExpireTime(). */
unsigned int userData : 3; /* ebuckets can be used to store in same
instance few different types of items,
such as, listpack and hash. This field
is reserved to store such identification
associated with the item and can help
to distinct on delete or expire callback.
It is not used by ebuckets internally and
should be maintained by the user */
unsigned int reserved : 4;
void *next; /* - If not last item in segment then next
points to next eItem (lastInSegment=0).
- If last in segment but not last in
bucket (lastItemBucket=0) then it
points to next segment header.
- If last in bucket then it points to
current segment header (Can be either
of type FirstSegHdr or NextSegHdr). */
} ExpireMeta;
/* Each instance of ebuckets need to have corresponding EbucketsType that holds
* the necessary callbacks and configuration to operate correctly on the type
* of items that are stored in it. Conceptually it should have hold reference
* from ebuckets instance to this type, but to save memory we will pass it as
* an argument to each API call. */
typedef struct EbucketsType {
/* getter to extract the ExpireMeta from the item */
ExpireMeta* (*getExpireMeta)(const eItem item);
/* Called during ebDestroy(). Set to NULL if not needed. */
void (*onDeleteItem)(eItem item, void *ctx);
/* Is addresses of items are odd in memory. It is taken into consideration
* and used by ebuckets to know how to distinct between ebuckets pointer to
* rax versus a pointer to item which is head of list. */
unsigned int itemsAddrAreOdd;
} EbucketsType;
/* Returned value by `onExpireItem` callback to indicate the action to be taken by
* ebExpire(). */
typedef enum ExpireAction {
ACT_REMOVE_EXP_ITEM=0, /* Remove the item from ebuckets. */
ACT_UPDATE_EXP_ITEM, /* Re-insert the item with updated expiration-time.
Before returning this value, the cb need to
update expiration time of the item by assisting
function ebSetMetaExpTime(). The item will be
kept aside and will be added again to ebuckets
at the end of ebExpire() */
ACT_STOP_ACTIVE_EXP /* Stop active-expiration. It will assume that
provided 'item' wasn't deleted by the callback. */
} ExpireAction;
/* ExpireInfo is used to pass input and output parameters to ebExpire(). */
typedef struct ExpireInfo {
/* onExpireItem - Called during active-expiration by ebExpire() */
ExpireAction (*onExpireItem)(eItem item, void *ctx);
uint64_t maxToExpire; /* [INPUT ] Limit of number expired items to scan */
void *ctx; /* [INPUT ] context to pass to onExpireItem */
uint64_t now; /* [INPUT ] Current time in msec. */
uint64_t nextExpireTime; /* [OUTPUT] Next expiration time. Return 0, if none left. */
/* TODO: Distinct between expired & updated */
uint64_t itemsExpired; /* [OUTPUT] Returns the number of expired or updated items. */
} ExpireInfo;
/* ebuckets API */
static inline ebuckets ebCreate(void) { return NULL; } /* Empty ebuckets */
void ebDestroy(ebuckets *eb, EbucketsType *type, void *deletedItemsCbCtx);
void ebExpire(ebuckets *eb, EbucketsType *type, ExpireInfo *info);
uint64_t ebExpireDryRun(ebuckets eb, EbucketsType *type, uint64_t now);
static inline int ebIsEmpty(ebuckets eb) { return eb == NULL; }
uint64_t ebGetNextTimeToExpire(ebuckets eb, EbucketsType *type);
uint64_t ebGetMaxExpireTime(ebuckets eb, EbucketsType *type, int accurate);
uint64_t ebGetTotalItems(ebuckets eb, EbucketsType *type);
/* Item related API */
int ebRemove(ebuckets *eb, EbucketsType *type, eItem item);
int ebAdd(ebuckets *eb, EbucketsType *type, eItem item, uint64_t expireTime);
uint64_t ebGetExpireTime(EbucketsType *type, eItem item);
typedef eItem (ebDefragFunction)(const eItem item);
eItem ebDefragItem(ebuckets *eb, EbucketsType *type, eItem item, ebDefragFunction *fn);
static inline uint64_t ebGetMetaExpTime(ExpireMeta *expMeta) {
return (((uint64_t)(expMeta)->expireTimeHi << 32) | (expMeta)->expireTimeLo);
}
static inline void ebSetMetaExpTime(ExpireMeta *expMeta, uint64_t t) {
expMeta->expireTimeLo = (uint32_t)(t&0xFFFFFFFF);
expMeta->expireTimeHi = (uint16_t)((t) >> 32);
}
/* Debug API */
void ebValidate(ebuckets eb, EbucketsType *type);
void ebPrint(ebuckets eb, EbucketsType *type);
#ifdef REDIS_TEST
int ebucketsTest(int argc, char *argv[], int flags);
#endif
#endif /* __EBUCKETS_H */
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment