Unverified Commit 3a3cacfe authored by Moti Cohen's avatar Moti Cohen Committed by GitHub
Browse files

Extend modules API to read also expired keys and subkeys (#13526)

The PR extends `RedisModule_OpenKey`'s flags to include
`REDISMODULE_OPEN_KEY_ACCESS_EXPIRED`, which allows to access expired
keys.

It also allows to access expired subkeys. Currently relevant only for
hash fields
and has its impact on `RM_HashGet` and `RM_Scan`.
parent 617909e9
...@@ -24,6 +24,7 @@ ...@@ -24,6 +24,7 @@
/* Flags for expireIfNeeded */ /* Flags for expireIfNeeded */
#define EXPIRE_FORCE_DELETE_EXPIRED 1 #define EXPIRE_FORCE_DELETE_EXPIRED 1
#define EXPIRE_AVOID_DELETE_EXPIRED 2 #define EXPIRE_AVOID_DELETE_EXPIRED 2
#define EXPIRE_ALLOW_ACCESS_EXPIRED 4
/* Return values for expireIfNeeded */ /* Return values for expireIfNeeded */
typedef enum { typedef enum {
...@@ -94,6 +95,8 @@ robj *lookupKey(redisDb *db, robj *key, int flags, dictEntry **deref) { ...@@ -94,6 +95,8 @@ robj *lookupKey(redisDb *db, robj *key, int flags, dictEntry **deref) {
expire_flags |= EXPIRE_FORCE_DELETE_EXPIRED; expire_flags |= EXPIRE_FORCE_DELETE_EXPIRED;
if (flags & LOOKUP_NOEXPIRE) if (flags & LOOKUP_NOEXPIRE)
expire_flags |= EXPIRE_AVOID_DELETE_EXPIRED; expire_flags |= EXPIRE_AVOID_DELETE_EXPIRED;
if (flags & LOOKUP_ACCESS_EXPIRED)
expire_flags |= EXPIRE_ALLOW_ACCESS_EXPIRED;
if (expireIfNeeded(db, key, expire_flags) != KEY_VALID) { if (expireIfNeeded(db, key, expire_flags) != KEY_VALID) {
/* The key is no longer valid. */ /* The key is no longer valid. */
val = NULL; val = NULL;
...@@ -2051,14 +2054,17 @@ int keyIsExpired(redisDb *db, robj *key) { ...@@ -2051,14 +2054,17 @@ int keyIsExpired(redisDb *db, robj *key) {
* *
* On the other hand, if you just want expiration check, but need to avoid * On the other hand, if you just want expiration check, but need to avoid
* the actual key deletion and propagation of the deletion, use the * the actual key deletion and propagation of the deletion, use the
* EXPIRE_AVOID_DELETE_EXPIRED flag. * EXPIRE_AVOID_DELETE_EXPIRED flag. If also needed to read expired key (that
* hasn't being deleted yet) then use EXPIRE_ALLOW_ACCESS_EXPIRED.
* *
* The return value of the function is KEY_VALID if the key is still valid. * The return value of the function is KEY_VALID if the key is still valid.
* The function returns KEY_EXPIRED if the key is expired BUT not deleted, * The function returns KEY_EXPIRED if the key is expired BUT not deleted,
* or returns KEY_DELETED if the key is expired and deleted. */ * or returns KEY_DELETED if the key is expired and deleted. */
keyStatus expireIfNeeded(redisDb *db, robj *key, int flags) { keyStatus expireIfNeeded(redisDb *db, robj *key, int flags) {
if (server.lazy_expire_disabled) return KEY_VALID; if ((!keyIsExpired(db,key)) ||
if (!keyIsExpired(db,key)) return KEY_VALID; (server.lazy_expire_disabled) ||
(flags & EXPIRE_ALLOW_ACCESS_EXPIRED))
return KEY_VALID;
/* If we are running in the context of a replica, instead of /* If we are running in the context of a replica, instead of
* evicting the expired key from the database, we return ASAP: * evicting the expired key from the database, we return ASAP:
......
...@@ -4070,7 +4070,8 @@ static void moduleInitKeyTypeSpecific(RedisModuleKey *key) { ...@@ -4070,7 +4070,8 @@ static void moduleInitKeyTypeSpecific(RedisModuleKey *key) {
* * REDISMODULE_OPEN_KEY_NONOTIFY - Don't trigger keyspace event on key misses. * * REDISMODULE_OPEN_KEY_NONOTIFY - Don't trigger keyspace event on key misses.
* * REDISMODULE_OPEN_KEY_NOSTATS - Don't update keyspace hits/misses counters. * * REDISMODULE_OPEN_KEY_NOSTATS - Don't update keyspace hits/misses counters.
* * REDISMODULE_OPEN_KEY_NOEXPIRE - Avoid deleting lazy expired keys. * * REDISMODULE_OPEN_KEY_NOEXPIRE - Avoid deleting lazy expired keys.
* * REDISMODULE_OPEN_KEY_NOEFFECTS - Avoid any effects from fetching the key. */ * * REDISMODULE_OPEN_KEY_NOEFFECTS - Avoid any effects from fetching the key.
* * REDISMODULE_OPEN_KEY_ACCESS_EXPIRED - Access expired keys that have not yet been deleted */
RedisModuleKey *RM_OpenKey(RedisModuleCtx *ctx, robj *keyname, int mode) { RedisModuleKey *RM_OpenKey(RedisModuleCtx *ctx, robj *keyname, int mode) {
RedisModuleKey *kp; RedisModuleKey *kp;
robj *value; robj *value;
...@@ -4080,6 +4081,7 @@ RedisModuleKey *RM_OpenKey(RedisModuleCtx *ctx, robj *keyname, int mode) { ...@@ -4080,6 +4081,7 @@ RedisModuleKey *RM_OpenKey(RedisModuleCtx *ctx, robj *keyname, int mode) {
flags |= (mode & REDISMODULE_OPEN_KEY_NOSTATS? LOOKUP_NOSTATS: 0); flags |= (mode & REDISMODULE_OPEN_KEY_NOSTATS? LOOKUP_NOSTATS: 0);
flags |= (mode & REDISMODULE_OPEN_KEY_NOEXPIRE? LOOKUP_NOEXPIRE: 0); flags |= (mode & REDISMODULE_OPEN_KEY_NOEXPIRE? LOOKUP_NOEXPIRE: 0);
flags |= (mode & REDISMODULE_OPEN_KEY_NOEFFECTS? LOOKUP_NOEFFECTS: 0); flags |= (mode & REDISMODULE_OPEN_KEY_NOEFFECTS? LOOKUP_NOEFFECTS: 0);
flags |= (mode & REDISMODULE_OPEN_KEY_ACCESS_EXPIRED ? (LOOKUP_ACCESS_EXPIRED) : 0);
   
if (mode & REDISMODULE_WRITE) { if (mode & REDISMODULE_WRITE) {
value = lookupKeyWriteWithFlags(ctx->client->db,keyname, flags); value = lookupKeyWriteWithFlags(ctx->client->db,keyname, flags);
...@@ -5378,6 +5380,9 @@ int RM_HashGet(RedisModuleKey *key, int flags, ...) { ...@@ -5378,6 +5380,9 @@ int RM_HashGet(RedisModuleKey *key, int flags, ...) {
va_list ap; va_list ap;
if (key->value && key->value->type != OBJ_HASH) return REDISMODULE_ERR; if (key->value && key->value->type != OBJ_HASH) return REDISMODULE_ERR;
   
if (key->mode & REDISMODULE_OPEN_KEY_ACCESS_EXPIRED)
hfeFlags = HFE_LAZY_ACCESS_EXPIRED; /* allow read also expired fields */
va_start(ap, flags); va_start(ap, flags);
while(1) { while(1) {
RedisModuleString *field, **valueptr; RedisModuleString *field, **valueptr;
...@@ -11087,8 +11092,9 @@ static void moduleScanKeyCallback(void *privdata, const dictEntry *de) { ...@@ -11087,8 +11092,9 @@ static void moduleScanKeyCallback(void *privdata, const dictEntry *de) {
} else if (o->type == OBJ_HASH) { } else if (o->type == OBJ_HASH) {
sds val = dictGetVal(de); sds val = dictGetVal(de);
   
/* If field is expired, then ignore */ /* If field is expired and not indicated to access expired, then ignore */
if (hfieldIsExpired(key)) if ((!(data->key->mode & REDISMODULE_OPEN_KEY_ACCESS_EXPIRED)) &&
(hfieldIsExpired(key)))
return; return;
   
field = createStringObject(key, hfieldlen(key)); field = createStringObject(key, hfieldlen(key));
...@@ -11224,7 +11230,8 @@ int RM_ScanKey(RedisModuleKey *key, RedisModuleScanCursor *cursor, RedisModuleSc ...@@ -11224,7 +11230,8 @@ int RM_ScanKey(RedisModuleKey *key, RedisModuleScanCursor *cursor, RedisModuleSc
p = lpNext(lp, p); p = lpNext(lp, p);
   
/* Skip expired fields */ /* Skip expired fields */
if (hashTypeIsExpired(o, vllExpire)) if ((!(key->mode & REDISMODULE_OPEN_KEY_ACCESS_EXPIRED)) &&
(hashTypeIsExpired(o, vllExpire)))
continue; continue;
} }
   
......
...@@ -60,10 +60,13 @@ typedef long long ustime_t; ...@@ -60,10 +60,13 @@ typedef long long ustime_t;
#define REDISMODULE_OPEN_KEY_NOEXPIRE (1<<19) #define REDISMODULE_OPEN_KEY_NOEXPIRE (1<<19)
/* Avoid any effects from fetching the key */ /* Avoid any effects from fetching the key */
#define REDISMODULE_OPEN_KEY_NOEFFECTS (1<<20) #define REDISMODULE_OPEN_KEY_NOEFFECTS (1<<20)
/* Allow access expired key that haven't deleted yet */
#define REDISMODULE_OPEN_KEY_ACCESS_EXPIRED (1<<21)
/* Mask of all REDISMODULE_OPEN_KEY_* values. Any new mode should be added to this list. /* Mask of all REDISMODULE_OPEN_KEY_* values. Any new mode should be added to this list.
* Should not be used directly by the module, use RM_GetOpenKeyModesAll instead. * Should not be used directly by the module, use RM_GetOpenKeyModesAll instead.
* Located here so when we will add new modes we will not forget to update it. */ * Located here so when we will add new modes we will not forget to update it. */
#define _REDISMODULE_OPEN_KEY_ALL REDISMODULE_READ | REDISMODULE_WRITE | REDISMODULE_OPEN_KEY_NOTOUCH | REDISMODULE_OPEN_KEY_NONOTIFY | REDISMODULE_OPEN_KEY_NOSTATS | REDISMODULE_OPEN_KEY_NOEXPIRE | REDISMODULE_OPEN_KEY_NOEFFECTS #define _REDISMODULE_OPEN_KEY_ALL REDISMODULE_READ | REDISMODULE_WRITE | REDISMODULE_OPEN_KEY_NOTOUCH | REDISMODULE_OPEN_KEY_NONOTIFY | REDISMODULE_OPEN_KEY_NOSTATS | REDISMODULE_OPEN_KEY_NOEXPIRE | REDISMODULE_OPEN_KEY_NOEFFECTS | REDISMODULE_OPEN_KEY_ACCESS_EXPIRED
/* List push and pop */ /* List push and pop */
#define REDISMODULE_LIST_HEAD 0 #define REDISMODULE_LIST_HEAD 0
......
...@@ -3221,6 +3221,7 @@ typedef struct dictExpireMetadata { ...@@ -3221,6 +3221,7 @@ typedef struct dictExpireMetadata {
#define HFE_LAZY_AVOID_HASH_DEL (1<<1) /* Avoid deleting hash if the field is the last one */ #define HFE_LAZY_AVOID_HASH_DEL (1<<1) /* Avoid deleting hash if the field is the last one */
#define HFE_LAZY_NO_NOTIFICATION (1<<2) /* Do not send notification, used when multiple fields #define HFE_LAZY_NO_NOTIFICATION (1<<2) /* Do not send notification, used when multiple fields
* may expire and only one notification is desired. */ * may expire and only one notification is desired. */
#define HFE_LAZY_ACCESS_EXPIRED (1<<3) /* Avoid lazy expire and allow access to expired fields */
void hashTypeConvert(robj *o, int enc, ebuckets *hexpires); void hashTypeConvert(robj *o, int enc, ebuckets *hexpires);
void hashTypeTryConversion(redisDb *db, robj *subject, robj **argv, int start, int end); void hashTypeTryConversion(redisDb *db, robj *subject, robj **argv, int start, int end);
...@@ -3383,11 +3384,12 @@ robj *objectCommandLookupOrReply(client *c, robj *key, robj *reply); ...@@ -3383,11 +3384,12 @@ robj *objectCommandLookupOrReply(client *c, robj *key, robj *reply);
int objectSetLRUOrLFU(robj *val, long long lfu_freq, long long lru_idle, int objectSetLRUOrLFU(robj *val, long long lfu_freq, long long lru_idle,
long long lru_clock, int lru_multiplier); long long lru_clock, int lru_multiplier);
#define LOOKUP_NONE 0 #define LOOKUP_NONE 0
#define LOOKUP_NOTOUCH (1<<0) /* Don't update LRU. */ #define LOOKUP_NOTOUCH (1<<0) /* Don't update LRU. */
#define LOOKUP_NONOTIFY (1<<1) /* Don't trigger keyspace event on key misses. */ #define LOOKUP_NONOTIFY (1<<1) /* Don't trigger keyspace event on key misses. */
#define LOOKUP_NOSTATS (1<<2) /* Don't update keyspace hits/misses counters. */ #define LOOKUP_NOSTATS (1<<2) /* Don't update keyspace hits/misses counters. */
#define LOOKUP_WRITE (1<<3) /* Delete expired keys even in replicas. */ #define LOOKUP_WRITE (1<<3) /* Delete expired keys even in replicas. */
#define LOOKUP_NOEXPIRE (1<<4) /* Avoid deleting lazy expired keys. */ #define LOOKUP_NOEXPIRE (1<<4) /* Avoid deleting lazy expired keys. */
#define LOOKUP_ACCESS_EXPIRED (1<<5) /* Allow lookup to expired key. */
#define LOOKUP_NOEFFECTS (LOOKUP_NONOTIFY | LOOKUP_NOSTATS | LOOKUP_NOTOUCH | LOOKUP_NOEXPIRE) /* Avoid any effects from fetching the key */ #define LOOKUP_NOEFFECTS (LOOKUP_NONOTIFY | LOOKUP_NOSTATS | LOOKUP_NOTOUCH | LOOKUP_NOEXPIRE) /* Avoid any effects from fetching the key */
dictEntry *dbAdd(redisDb *db, robj *key, robj *val); dictEntry *dbAdd(redisDb *db, robj *key, robj *val);
......
...@@ -734,7 +734,7 @@ GetFieldRes hashTypeGetValue(redisDb *db, robj *o, sds field, unsigned char **vs ...@@ -734,7 +734,7 @@ GetFieldRes hashTypeGetValue(redisDb *db, robj *o, sds field, unsigned char **vs
serverPanic("Unknown hash encoding"); serverPanic("Unknown hash encoding");
} }
if (expiredAt >= (uint64_t) commandTimeSnapshot()) if ((expiredAt >= (uint64_t) commandTimeSnapshot()) || (hfeFlags & HFE_LAZY_ACCESS_EXPIRED))
return GETF_OK; return GETF_OK;
if (server.masterhost) { if (server.masterhost) {
......
...@@ -3,6 +3,8 @@ ...@@ -3,6 +3,8 @@
#include <errno.h> #include <errno.h>
#include <stdlib.h> #include <stdlib.h>
#define UNUSED(x) (void)(x)
/* If a string is ":deleted:", the special value for deleted hash fields is /* If a string is ":deleted:", the special value for deleted hash fields is
* returned; otherwise the input string is returned. */ * returned; otherwise the input string is returned. */
static RedisModuleString *value_or_delete(RedisModuleString *s) { static RedisModuleString *value_or_delete(RedisModuleString *s) {
...@@ -76,15 +78,100 @@ int hash_set(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { ...@@ -76,15 +78,100 @@ int hash_set(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
return RedisModule_ReplyWithLongLong(ctx, result); return RedisModule_ReplyWithLongLong(ctx, result);
} }
RedisModuleKey* openKeyWithMode(RedisModuleCtx *ctx, RedisModuleString *keyName, int mode) {
int supportedMode = RedisModule_GetOpenKeyModesAll();
if (!(supportedMode & REDISMODULE_READ) || ((supportedMode & mode)!=mode)) {
RedisModule_ReplyWithError(ctx, "OpenKey mode is not supported");
return NULL;
}
RedisModuleKey *key = RedisModule_OpenKey(ctx, keyName, REDISMODULE_READ | mode);
if (!key) {
RedisModule_ReplyWithError(ctx, "key not found");
return NULL;
}
return key;
}
int test_open_key_subexpired_hget(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
if (argc<3) {
RedisModule_WrongArity(ctx);
return REDISMODULE_OK;
}
RedisModuleKey *key = openKeyWithMode(ctx, argv[1], REDISMODULE_OPEN_KEY_ACCESS_EXPIRED);
if (!key) return REDISMODULE_OK;
RedisModuleString *value;
RedisModule_HashGet(key,REDISMODULE_HASH_NONE,argv[2],&value,NULL);
/* return the value */
if (value) {
RedisModule_ReplyWithString(ctx, value);
RedisModule_FreeString(ctx, value);
} else {
RedisModule_ReplyWithNull(ctx);
}
RedisModule_CloseKey(key);
return REDISMODULE_OK;
}
int numReplies;
void ScanCallback(RedisModuleKey *key, RedisModuleString *field, RedisModuleString *value, void *privdata) {
UNUSED(key);
RedisModuleCtx *ctx = (RedisModuleCtx *)privdata;
/* Reply with the field and value (or NULL for sets) */
RedisModule_ReplyWithString(ctx, field);
if (value) {
RedisModule_ReplyWithString(ctx, value);
} else {
RedisModule_ReplyWithCString(ctx, "(null)");
}
numReplies+=2;
}
int test_open_key_access_expired_hscan(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
if (argc < 2) {
RedisModule_WrongArity(ctx);
return REDISMODULE_OK;
}
RedisModuleKey *key = openKeyWithMode(ctx, argv[1], REDISMODULE_OPEN_KEY_ACCESS_EXPIRED);
if (!key)
return RedisModule_ReplyWithError(ctx, "ERR key not exists");
/* Verify it is a hash */
if (RedisModule_KeyType(key) != REDISMODULE_KEYTYPE_HASH) {
RedisModule_CloseKey(key);
return RedisModule_ReplyWithError(ctx, "ERR key is not a hash");
}
/* Scan the hash and reply pairs of key-value */
RedisModule_ReplyWithArray(ctx, REDISMODULE_POSTPONED_ARRAY_LEN);
numReplies = 0;
RedisModuleScanCursor *cursor = RedisModule_ScanCursorCreate();
while (RedisModule_ScanKey(key, cursor, ScanCallback, ctx));
RedisModule_ScanCursorDestroy(cursor);
RedisModule_CloseKey(key);
RedisModule_ReplySetArrayLength(ctx, numReplies);
return REDISMODULE_OK;
}
int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
REDISMODULE_NOT_USED(argv); REDISMODULE_NOT_USED(argv);
REDISMODULE_NOT_USED(argc); REDISMODULE_NOT_USED(argc);
if (RedisModule_Init(ctx, "hash", 1, REDISMODULE_APIVER_1) == if (RedisModule_Init(ctx, "hash", 1, REDISMODULE_APIVER_1) == REDISMODULE_ERR)
REDISMODULE_OK &&
RedisModule_CreateCommand(ctx, "hash.set", hash_set, "write",
1, 1, 1) == REDISMODULE_OK) {
return REDISMODULE_OK;
} else {
return REDISMODULE_ERR; return REDISMODULE_ERR;
}
if (RedisModule_CreateCommand(ctx, "hash.set", hash_set, "write", 1, 1, 1) == REDISMODULE_ERR)
return REDISMODULE_ERR;
if (RedisModule_CreateCommand(ctx, "hash.hget_expired", test_open_key_subexpired_hget,"", 0, 0, 0) == REDISMODULE_ERR)
return REDISMODULE_ERR;
if (RedisModule_CreateCommand(ctx, "hash.hscan_expired", test_open_key_access_expired_hscan,"", 0, 0, 0) == REDISMODULE_ERR)
return REDISMODULE_ERR;
return REDISMODULE_OK;
} }
...@@ -60,6 +60,39 @@ start_server {tags {"modules"}} { ...@@ -60,6 +60,39 @@ start_server {tags {"modules"}} {
r debug set-active-expire 1 r debug set-active-expire 1
} {OK} {needs:debug} } {OK} {needs:debug}
test {test open key with REDISMODULE_OPEN_KEY_ACCESS_EXPIRED to scan expired fields} {
r debug set-active-expire 0
r del H1
r hash.set H1 "n" f1 v1 f2 v2 f3 v3
r hpexpire H1 1 FIELDS 2 f1 f2
after 10
# Scan expired fields with flag REDISMODULE_OPEN_KEY_ACCESS_EXPIRED
assert_equal "f1 f2 f3 v1 v2 v3" [lsort [r hash.hscan_expired H1]]
# Get expired field with flag REDISMODULE_OPEN_KEY_ACCESS_EXPIRED
assert_equal {v1} [r hash.hget_expired H1 f1]
# Verify key doesn't exist on normal access without the flag
assert_equal 0 [r hexists H1 f1]
assert_equal 0 [r hexists H1 f2]
# Scan again expired fields with flag REDISMODULE_OPEN_KEY_ACCESS_EXPIRED
assert_equal "f3 v3" [lsort [r hash.hscan_expired H1]]
r debug set-active-expire 1
}
test {test open key with REDISMODULE_OPEN_KEY_ACCESS_EXPIRED to scan expired key} {
r debug set-active-expire 0
r del H1
r hash.set H1 "n" f1 v1 f2 v2 f3 v3
r pexpire H1 5
after 10
# Scan expired fields with flag REDISMODULE_OPEN_KEY_ACCESS_EXPIRED
assert_equal "f1 f2 f3 v1 v2 v3" [lsort [r hash.hscan_expired H1]]
# Get expired field with flag REDISMODULE_OPEN_KEY_ACCESS_EXPIRED
assert_equal {v1} [r hash.hget_expired H1 f1]
# Verify key doesn't exist on normal access without the flag
assert_equal 0 [r exists H1]
r debug set-active-expire 1
}
test "Unload the module - hash" { test "Unload the module - hash" {
assert_equal {OK} [r module unload hash] assert_equal {OK} [r module unload hash]
} }
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment