Unverified Commit 3a3cacfe authored by Moti Cohen's avatar Moti Cohen Committed by GitHub
Browse files

Extend modules API to read also expired keys and subkeys (#13526)

The PR extends `RedisModule_OpenKey`'s flags to include
`REDISMODULE_OPEN_KEY_ACCESS_EXPIRED`, which allows to access expired
keys.

It also allows to access expired subkeys. Currently relevant only for
hash fields
and has its impact on `RM_HashGet` and `RM_Scan`.
parent 617909e9
......@@ -24,6 +24,7 @@
/* Flags for expireIfNeeded */
#define EXPIRE_FORCE_DELETE_EXPIRED 1
#define EXPIRE_AVOID_DELETE_EXPIRED 2
#define EXPIRE_ALLOW_ACCESS_EXPIRED 4
/* Return values for expireIfNeeded */
typedef enum {
......@@ -94,6 +95,8 @@ robj *lookupKey(redisDb *db, robj *key, int flags, dictEntry **deref) {
expire_flags |= EXPIRE_FORCE_DELETE_EXPIRED;
if (flags & LOOKUP_NOEXPIRE)
expire_flags |= EXPIRE_AVOID_DELETE_EXPIRED;
if (flags & LOOKUP_ACCESS_EXPIRED)
expire_flags |= EXPIRE_ALLOW_ACCESS_EXPIRED;
if (expireIfNeeded(db, key, expire_flags) != KEY_VALID) {
/* The key is no longer valid. */
val = NULL;
......@@ -2051,14 +2054,17 @@ int keyIsExpired(redisDb *db, robj *key) {
*
* On the other hand, if you just want expiration check, but need to avoid
* the actual key deletion and propagation of the deletion, use the
* EXPIRE_AVOID_DELETE_EXPIRED flag.
* EXPIRE_AVOID_DELETE_EXPIRED flag. If also needed to read expired key (that
* hasn't being deleted yet) then use EXPIRE_ALLOW_ACCESS_EXPIRED.
*
* The return value of the function is KEY_VALID if the key is still valid.
* The function returns KEY_EXPIRED if the key is expired BUT not deleted,
* or returns KEY_DELETED if the key is expired and deleted. */
keyStatus expireIfNeeded(redisDb *db, robj *key, int flags) {
if (server.lazy_expire_disabled) return KEY_VALID;
if (!keyIsExpired(db,key)) return KEY_VALID;
if ((!keyIsExpired(db,key)) ||
(server.lazy_expire_disabled) ||
(flags & EXPIRE_ALLOW_ACCESS_EXPIRED))
return KEY_VALID;
/* If we are running in the context of a replica, instead of
* evicting the expired key from the database, we return ASAP:
......
......@@ -4070,7 +4070,8 @@ static void moduleInitKeyTypeSpecific(RedisModuleKey *key) {
* * REDISMODULE_OPEN_KEY_NONOTIFY - Don't trigger keyspace event on key misses.
* * REDISMODULE_OPEN_KEY_NOSTATS - Don't update keyspace hits/misses counters.
* * REDISMODULE_OPEN_KEY_NOEXPIRE - Avoid deleting lazy expired keys.
* * REDISMODULE_OPEN_KEY_NOEFFECTS - Avoid any effects from fetching the key. */
* * REDISMODULE_OPEN_KEY_NOEFFECTS - Avoid any effects from fetching the key.
* * REDISMODULE_OPEN_KEY_ACCESS_EXPIRED - Access expired keys that have not yet been deleted */
RedisModuleKey *RM_OpenKey(RedisModuleCtx *ctx, robj *keyname, int mode) {
RedisModuleKey *kp;
robj *value;
......@@ -4080,6 +4081,7 @@ RedisModuleKey *RM_OpenKey(RedisModuleCtx *ctx, robj *keyname, int mode) {
flags |= (mode & REDISMODULE_OPEN_KEY_NOSTATS? LOOKUP_NOSTATS: 0);
flags |= (mode & REDISMODULE_OPEN_KEY_NOEXPIRE? LOOKUP_NOEXPIRE: 0);
flags |= (mode & REDISMODULE_OPEN_KEY_NOEFFECTS? LOOKUP_NOEFFECTS: 0);
flags |= (mode & REDISMODULE_OPEN_KEY_ACCESS_EXPIRED ? (LOOKUP_ACCESS_EXPIRED) : 0);
 
if (mode & REDISMODULE_WRITE) {
value = lookupKeyWriteWithFlags(ctx->client->db,keyname, flags);
......@@ -5378,6 +5380,9 @@ int RM_HashGet(RedisModuleKey *key, int flags, ...) {
va_list ap;
if (key->value && key->value->type != OBJ_HASH) return REDISMODULE_ERR;
 
if (key->mode & REDISMODULE_OPEN_KEY_ACCESS_EXPIRED)
hfeFlags = HFE_LAZY_ACCESS_EXPIRED; /* allow read also expired fields */
va_start(ap, flags);
while(1) {
RedisModuleString *field, **valueptr;
......@@ -11087,8 +11092,9 @@ static void moduleScanKeyCallback(void *privdata, const dictEntry *de) {
} else if (o->type == OBJ_HASH) {
sds val = dictGetVal(de);
 
/* If field is expired, then ignore */
if (hfieldIsExpired(key))
/* If field is expired and not indicated to access expired, then ignore */
if ((!(data->key->mode & REDISMODULE_OPEN_KEY_ACCESS_EXPIRED)) &&
(hfieldIsExpired(key)))
return;
 
field = createStringObject(key, hfieldlen(key));
......@@ -11224,7 +11230,8 @@ int RM_ScanKey(RedisModuleKey *key, RedisModuleScanCursor *cursor, RedisModuleSc
p = lpNext(lp, p);
 
/* Skip expired fields */
if (hashTypeIsExpired(o, vllExpire))
if ((!(key->mode & REDISMODULE_OPEN_KEY_ACCESS_EXPIRED)) &&
(hashTypeIsExpired(o, vllExpire)))
continue;
}
 
......
......@@ -60,10 +60,13 @@ typedef long long ustime_t;
#define REDISMODULE_OPEN_KEY_NOEXPIRE (1<<19)
/* Avoid any effects from fetching the key */
#define REDISMODULE_OPEN_KEY_NOEFFECTS (1<<20)
/* Allow access expired key that haven't deleted yet */
#define REDISMODULE_OPEN_KEY_ACCESS_EXPIRED (1<<21)
/* Mask of all REDISMODULE_OPEN_KEY_* values. Any new mode should be added to this list.
* Should not be used directly by the module, use RM_GetOpenKeyModesAll instead.
* Located here so when we will add new modes we will not forget to update it. */
#define _REDISMODULE_OPEN_KEY_ALL REDISMODULE_READ | REDISMODULE_WRITE | REDISMODULE_OPEN_KEY_NOTOUCH | REDISMODULE_OPEN_KEY_NONOTIFY | REDISMODULE_OPEN_KEY_NOSTATS | REDISMODULE_OPEN_KEY_NOEXPIRE | REDISMODULE_OPEN_KEY_NOEFFECTS
#define _REDISMODULE_OPEN_KEY_ALL REDISMODULE_READ | REDISMODULE_WRITE | REDISMODULE_OPEN_KEY_NOTOUCH | REDISMODULE_OPEN_KEY_NONOTIFY | REDISMODULE_OPEN_KEY_NOSTATS | REDISMODULE_OPEN_KEY_NOEXPIRE | REDISMODULE_OPEN_KEY_NOEFFECTS | REDISMODULE_OPEN_KEY_ACCESS_EXPIRED
/* List push and pop */
#define REDISMODULE_LIST_HEAD 0
......
......@@ -3221,6 +3221,7 @@ typedef struct dictExpireMetadata {
#define HFE_LAZY_AVOID_HASH_DEL (1<<1) /* Avoid deleting hash if the field is the last one */
#define HFE_LAZY_NO_NOTIFICATION (1<<2) /* Do not send notification, used when multiple fields
* may expire and only one notification is desired. */
#define HFE_LAZY_ACCESS_EXPIRED (1<<3) /* Avoid lazy expire and allow access to expired fields */
void hashTypeConvert(robj *o, int enc, ebuckets *hexpires);
void hashTypeTryConversion(redisDb *db, robj *subject, robj **argv, int start, int end);
......@@ -3383,11 +3384,12 @@ robj *objectCommandLookupOrReply(client *c, robj *key, robj *reply);
int objectSetLRUOrLFU(robj *val, long long lfu_freq, long long lru_idle,
long long lru_clock, int lru_multiplier);
#define LOOKUP_NONE 0
#define LOOKUP_NOTOUCH (1<<0) /* Don't update LRU. */
#define LOOKUP_NONOTIFY (1<<1) /* Don't trigger keyspace event on key misses. */
#define LOOKUP_NOSTATS (1<<2) /* Don't update keyspace hits/misses counters. */
#define LOOKUP_WRITE (1<<3) /* Delete expired keys even in replicas. */
#define LOOKUP_NOEXPIRE (1<<4) /* Avoid deleting lazy expired keys. */
#define LOOKUP_NOTOUCH (1<<0) /* Don't update LRU. */
#define LOOKUP_NONOTIFY (1<<1) /* Don't trigger keyspace event on key misses. */
#define LOOKUP_NOSTATS (1<<2) /* Don't update keyspace hits/misses counters. */
#define LOOKUP_WRITE (1<<3) /* Delete expired keys even in replicas. */
#define LOOKUP_NOEXPIRE (1<<4) /* Avoid deleting lazy expired keys. */
#define LOOKUP_ACCESS_EXPIRED (1<<5) /* Allow lookup to expired key. */
#define LOOKUP_NOEFFECTS (LOOKUP_NONOTIFY | LOOKUP_NOSTATS | LOOKUP_NOTOUCH | LOOKUP_NOEXPIRE) /* Avoid any effects from fetching the key */
dictEntry *dbAdd(redisDb *db, robj *key, robj *val);
......
......@@ -734,7 +734,7 @@ GetFieldRes hashTypeGetValue(redisDb *db, robj *o, sds field, unsigned char **vs
serverPanic("Unknown hash encoding");
}
if (expiredAt >= (uint64_t) commandTimeSnapshot())
if ((expiredAt >= (uint64_t) commandTimeSnapshot()) || (hfeFlags & HFE_LAZY_ACCESS_EXPIRED))
return GETF_OK;
if (server.masterhost) {
......
......@@ -3,6 +3,8 @@
#include <errno.h>
#include <stdlib.h>
#define UNUSED(x) (void)(x)
/* If a string is ":deleted:", the special value for deleted hash fields is
* returned; otherwise the input string is returned. */
static RedisModuleString *value_or_delete(RedisModuleString *s) {
......@@ -76,15 +78,100 @@ int hash_set(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
return RedisModule_ReplyWithLongLong(ctx, result);
}
RedisModuleKey* openKeyWithMode(RedisModuleCtx *ctx, RedisModuleString *keyName, int mode) {
int supportedMode = RedisModule_GetOpenKeyModesAll();
if (!(supportedMode & REDISMODULE_READ) || ((supportedMode & mode)!=mode)) {
RedisModule_ReplyWithError(ctx, "OpenKey mode is not supported");
return NULL;
}
RedisModuleKey *key = RedisModule_OpenKey(ctx, keyName, REDISMODULE_READ | mode);
if (!key) {
RedisModule_ReplyWithError(ctx, "key not found");
return NULL;
}
return key;
}
int test_open_key_subexpired_hget(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
if (argc<3) {
RedisModule_WrongArity(ctx);
return REDISMODULE_OK;
}
RedisModuleKey *key = openKeyWithMode(ctx, argv[1], REDISMODULE_OPEN_KEY_ACCESS_EXPIRED);
if (!key) return REDISMODULE_OK;
RedisModuleString *value;
RedisModule_HashGet(key,REDISMODULE_HASH_NONE,argv[2],&value,NULL);
/* return the value */
if (value) {
RedisModule_ReplyWithString(ctx, value);
RedisModule_FreeString(ctx, value);
} else {
RedisModule_ReplyWithNull(ctx);
}
RedisModule_CloseKey(key);
return REDISMODULE_OK;
}
int numReplies;
void ScanCallback(RedisModuleKey *key, RedisModuleString *field, RedisModuleString *value, void *privdata) {
UNUSED(key);
RedisModuleCtx *ctx = (RedisModuleCtx *)privdata;
/* Reply with the field and value (or NULL for sets) */
RedisModule_ReplyWithString(ctx, field);
if (value) {
RedisModule_ReplyWithString(ctx, value);
} else {
RedisModule_ReplyWithCString(ctx, "(null)");
}
numReplies+=2;
}
int test_open_key_access_expired_hscan(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
if (argc < 2) {
RedisModule_WrongArity(ctx);
return REDISMODULE_OK;
}
RedisModuleKey *key = openKeyWithMode(ctx, argv[1], REDISMODULE_OPEN_KEY_ACCESS_EXPIRED);
if (!key)
return RedisModule_ReplyWithError(ctx, "ERR key not exists");
/* Verify it is a hash */
if (RedisModule_KeyType(key) != REDISMODULE_KEYTYPE_HASH) {
RedisModule_CloseKey(key);
return RedisModule_ReplyWithError(ctx, "ERR key is not a hash");
}
/* Scan the hash and reply pairs of key-value */
RedisModule_ReplyWithArray(ctx, REDISMODULE_POSTPONED_ARRAY_LEN);
numReplies = 0;
RedisModuleScanCursor *cursor = RedisModule_ScanCursorCreate();
while (RedisModule_ScanKey(key, cursor, ScanCallback, ctx));
RedisModule_ScanCursorDestroy(cursor);
RedisModule_CloseKey(key);
RedisModule_ReplySetArrayLength(ctx, numReplies);
return REDISMODULE_OK;
}
int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
REDISMODULE_NOT_USED(argv);
REDISMODULE_NOT_USED(argc);
if (RedisModule_Init(ctx, "hash", 1, REDISMODULE_APIVER_1) ==
REDISMODULE_OK &&
RedisModule_CreateCommand(ctx, "hash.set", hash_set, "write",
1, 1, 1) == REDISMODULE_OK) {
return REDISMODULE_OK;
} else {
if (RedisModule_Init(ctx, "hash", 1, REDISMODULE_APIVER_1) == REDISMODULE_ERR)
return REDISMODULE_ERR;
}
if (RedisModule_CreateCommand(ctx, "hash.set", hash_set, "write", 1, 1, 1) == REDISMODULE_ERR)
return REDISMODULE_ERR;
if (RedisModule_CreateCommand(ctx, "hash.hget_expired", test_open_key_subexpired_hget,"", 0, 0, 0) == REDISMODULE_ERR)
return REDISMODULE_ERR;
if (RedisModule_CreateCommand(ctx, "hash.hscan_expired", test_open_key_access_expired_hscan,"", 0, 0, 0) == REDISMODULE_ERR)
return REDISMODULE_ERR;
return REDISMODULE_OK;
}
......@@ -60,6 +60,39 @@ start_server {tags {"modules"}} {
r debug set-active-expire 1
} {OK} {needs:debug}
test {test open key with REDISMODULE_OPEN_KEY_ACCESS_EXPIRED to scan expired fields} {
r debug set-active-expire 0
r del H1
r hash.set H1 "n" f1 v1 f2 v2 f3 v3
r hpexpire H1 1 FIELDS 2 f1 f2
after 10
# Scan expired fields with flag REDISMODULE_OPEN_KEY_ACCESS_EXPIRED
assert_equal "f1 f2 f3 v1 v2 v3" [lsort [r hash.hscan_expired H1]]
# Get expired field with flag REDISMODULE_OPEN_KEY_ACCESS_EXPIRED
assert_equal {v1} [r hash.hget_expired H1 f1]
# Verify key doesn't exist on normal access without the flag
assert_equal 0 [r hexists H1 f1]
assert_equal 0 [r hexists H1 f2]
# Scan again expired fields with flag REDISMODULE_OPEN_KEY_ACCESS_EXPIRED
assert_equal "f3 v3" [lsort [r hash.hscan_expired H1]]
r debug set-active-expire 1
}
test {test open key with REDISMODULE_OPEN_KEY_ACCESS_EXPIRED to scan expired key} {
r debug set-active-expire 0
r del H1
r hash.set H1 "n" f1 v1 f2 v2 f3 v3
r pexpire H1 5
after 10
# Scan expired fields with flag REDISMODULE_OPEN_KEY_ACCESS_EXPIRED
assert_equal "f1 f2 f3 v1 v2 v3" [lsort [r hash.hscan_expired H1]]
# Get expired field with flag REDISMODULE_OPEN_KEY_ACCESS_EXPIRED
assert_equal {v1} [r hash.hget_expired H1 f1]
# Verify key doesn't exist on normal access without the flag
assert_equal 0 [r exists H1]
r debug set-active-expire 1
}
test "Unload the module - hash" {
assert_equal {OK} [r module unload hash]
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment