Unverified Commit 03fcc211 authored by guybe7's avatar guybe7 Committed by GitHub
Browse files

A better approach for COMMAND INFO for movablekeys commands (#8324)

Fix #7297

The problem:

Today, there is no way for a client library or app to know the key name indexes for commands such as
ZUNIONSTORE/EVAL and others with "numkeys", since COMMAND INFO returns no useful info for them.

For cluster-aware redis clients, this requires to 'patch' the client library code specifically for each of these commands or to
resolve each execution of these commands with COMMAND GETKEYS.

The solution:

Introducing key specs other than the legacy "range" (first,last,step)

The 8th element of the command info array, if exists, holds an array of key specs. The array may be empty, which indicates
the command doesn't take any key arguments or may contain one or more key-specs, each one may leads to the discovery
of 0 or more key arguments.

A client library that doesn't support this key-spec feature will keep using the first,last,step and movablekeys flag which will
obviously remain unchanged.

A cl...
parent b5a879e1
...@@ -34,6 +34,7 @@ $TCLSH tests/test_helper.tcl \ ...@@ -34,6 +34,7 @@ $TCLSH tests/test_helper.tcl \
--single unit/moduleapi/getkeys \ --single unit/moduleapi/getkeys \
--single unit/moduleapi/test_lazyfree \ --single unit/moduleapi/test_lazyfree \
--single unit/moduleapi/defrag \ --single unit/moduleapi/defrag \
--single unit/moduleapi/keyspecs \
--single unit/moduleapi/hash \ --single unit/moduleapi/hash \
--single unit/moduleapi/zset \ --single unit/moduleapi/zset \
--single unit/moduleapi/list \ --single unit/moduleapi/list \
......
...@@ -1225,7 +1225,7 @@ int ACLCheckCommandPerm(client *c, int *keyidxptr) { ...@@ -1225,7 +1225,7 @@ int ACLCheckCommandPerm(client *c, int *keyidxptr) {
/* Check if the user can execute commands explicitly touching the keys /* Check if the user can execute commands explicitly touching the keys
* mentioned in the command arguments. */ * mentioned in the command arguments. */
if (!(c->user->flags & USER_FLAG_ALLKEYS) && if (!(c->user->flags & USER_FLAG_ALLKEYS) &&
(c->cmd->getkeys_proc || c->cmd->firstkey)) (c->cmd->getkeys_proc || c->cmd->key_specs_num))
{ {
getKeysResult result = GETKEYS_RESULT_INIT; getKeysResult result = GETKEYS_RESULT_INIT;
int numkeys = getKeysFromCommand(c->cmd,c->argv,c->argc,&result); int numkeys = getKeysFromCommand(c->cmd,c->argv,c->argc,&result);
......
...@@ -1575,23 +1575,30 @@ int *getKeysPrepareResult(getKeysResult *result, int numkeys) { ...@@ -1575,23 +1575,30 @@ int *getKeysPrepareResult(getKeysResult *result, int numkeys) {
} }
/* The base case is to use the keys position as given in the command table /* The base case is to use the keys position as given in the command table
* (firstkey, lastkey, step). */ * (firstkey, lastkey, step).
int getKeysUsingCommandTable(struct redisCommand *cmd,robj **argv, int argc, getKeysResult *result) { * This function works only on command with the legacy_range_key_spec,
int j, i = 0, last, *keys; * all other commands should be handled by getkeys_proc. */
int getKeysUsingLegacyRangeSpec(struct redisCommand *cmd, robj **argv, int argc, getKeysResult *result) {
int j, i = 0, last, first, step, *keys;
UNUSED(argv); UNUSED(argv);
if (cmd->firstkey == 0) { if (cmd->legacy_range_key_spec.begin_search_type == KSPEC_BS_INVALID) {
result->numkeys = 0; result->numkeys = 0;
return 0; return 0;
} }
last = cmd->lastkey; first = cmd->legacy_range_key_spec.bs.index.pos;
last = cmd->legacy_range_key_spec.fk.range.lastkey;
if (last >= 0)
last += first;
step = cmd->legacy_range_key_spec.fk.range.keystep;
if (last < 0) last = argc+last; if (last < 0) last = argc+last;
int count = ((last - cmd->firstkey)+1); int count = ((last - first)+1);
keys = getKeysPrepareResult(result, count); keys = getKeysPrepareResult(result, count);
for (j = cmd->firstkey; j <= last; j += cmd->keystep) { for (j = first; j <= last; j += step) {
if (j >= argc) { if (j >= argc) {
/* Modules commands, and standard commands with a not fixed number /* Modules commands, and standard commands with a not fixed number
* of arguments (negative arity parameter) do not have dispatch * of arguments (negative arity parameter) do not have dispatch
...@@ -1629,7 +1636,7 @@ int getKeysFromCommand(struct redisCommand *cmd, robj **argv, int argc, getKeysR ...@@ -1629,7 +1636,7 @@ int getKeysFromCommand(struct redisCommand *cmd, robj **argv, int argc, getKeysR
} else if (!(cmd->flags & CMD_MODULE) && cmd->getkeys_proc) { } else if (!(cmd->flags & CMD_MODULE) && cmd->getkeys_proc) {
return cmd->getkeys_proc(cmd,argv,argc,result); return cmd->getkeys_proc(cmd,argv,argc,result);
} else { } else {
return getKeysUsingCommandTable(cmd,argv,argc,result); return getKeysUsingLegacyRangeSpec(cmd,argv,argc,result);
} }
} }
......
...@@ -808,6 +808,26 @@ int64_t commandFlagsFromString(char *s) { ...@@ -808,6 +808,26 @@ int64_t commandFlagsFromString(char *s) {
return flags; return flags;
} }
/* Helper for RM_CreateCommand(). Turns a string representing keys spec
* flags into the keys spec flags used by the Redis core.
*
* It returns the set of flags, or -1 if unknown flags are found. */
int64_t commandKeySpecsFlagsFromString(const char *s) {
int count, j;
int64_t flags = 0;
sds *tokens = sdssplitlen(s,strlen(s)," ",1,&count);
for (j = 0; j < count; j++) {
char *t = tokens[j];
if (!strcasecmp(t,"write")) flags |= CMD_KEY_WRITE;
else if (!strcasecmp(t,"read")) flags |= CMD_KEY_READ;
else if (!strcasecmp(t,"incomplete")) flags |= CMD_KEY_INCOMPLETE;
else break;
}
sdsfreesplitres(tokens,count);
if (j != count) return -1; /* Some token not processed correctly. */
return flags;
}
/* Register a new command in the Redis server, that will be handled by /* Register a new command in the Redis server, that will be handled by
* calling the function pointer 'cmdfunc' using the RedisModule calling * calling the function pointer 'cmdfunc' using the RedisModule calling
* convention. The function returns REDISMODULE_ERR if the specified command * convention. The function returns REDISMODULE_ERR if the specified command
...@@ -882,6 +902,12 @@ int64_t commandFlagsFromString(char *s) { ...@@ -882,6 +902,12 @@ int64_t commandFlagsFromString(char *s) {
* 0 for commands with no keys. * 0 for commands with no keys.
* *
* This information is used by ACL, Cluster and the 'COMMAND' command. * This information is used by ACL, Cluster and the 'COMMAND' command.
*
* NOTE: The scheme described above serves a limited purpose and can
* only be used to find keys that exist at constant indices.
* For non-trivial key arguments, you may pass 0,0,0 and use
* RedisModule_AddCommandKeySpec (see documentation).
*
*/ */
int RM_CreateCommand(RedisModuleCtx *ctx, const char *name, RedisModuleCmdFunc cmdfunc, const char *strflags, int firstkey, int lastkey, int keystep) { int RM_CreateCommand(RedisModuleCtx *ctx, const char *name, RedisModuleCmdFunc cmdfunc, const char *strflags, int firstkey, int lastkey, int keystep) {
int64_t flags = strflags ? commandFlagsFromString((char*)strflags) : 0; int64_t flags = strflags ? commandFlagsFromString((char*)strflags) : 0;
...@@ -915,9 +941,26 @@ int RM_CreateCommand(RedisModuleCtx *ctx, const char *name, RedisModuleCmdFunc c ...@@ -915,9 +941,26 @@ int RM_CreateCommand(RedisModuleCtx *ctx, const char *name, RedisModuleCmdFunc c
cp->rediscmd->arity = -1; cp->rediscmd->arity = -1;
cp->rediscmd->flags = flags | CMD_MODULE; cp->rediscmd->flags = flags | CMD_MODULE;
cp->rediscmd->getkeys_proc = (redisGetKeysProc*)(unsigned long)cp; cp->rediscmd->getkeys_proc = (redisGetKeysProc*)(unsigned long)cp;
cp->rediscmd->firstkey = firstkey; cp->rediscmd->key_specs_max = STATIC_KEY_SPECS_NUM;
cp->rediscmd->lastkey = lastkey; cp->rediscmd->key_specs = cp->rediscmd->key_specs_static;
cp->rediscmd->keystep = keystep; if (firstkey != 0) {
cp->rediscmd->key_specs_num = 1;
cp->rediscmd->key_specs[0].flags = 0;
cp->rediscmd->key_specs[0].begin_search_type = KSPEC_BS_INDEX;
cp->rediscmd->key_specs[0].bs.index.pos = firstkey;
cp->rediscmd->key_specs[0].find_keys_type = KSPEC_FK_RANGE;
cp->rediscmd->key_specs[0].fk.range.lastkey = lastkey < 0 ? lastkey : (lastkey-firstkey);
cp->rediscmd->key_specs[0].fk.range.keystep = keystep;
cp->rediscmd->key_specs[0].fk.range.limit = 0;
/* Copy the default range to legacy_range_key_spec */
cp->rediscmd->legacy_range_key_spec = cp->rediscmd->key_specs[0];
} else {
cp->rediscmd->key_specs_num = 0;
cp->rediscmd->legacy_range_key_spec.begin_search_type = KSPEC_BS_INVALID;
cp->rediscmd->legacy_range_key_spec.find_keys_type = KSPEC_FK_INVALID;
}
populateCommandMovableKeys(cp->rediscmd);
cp->rediscmd->microseconds = 0; cp->rediscmd->microseconds = 0;
cp->rediscmd->calls = 0; cp->rediscmd->calls = 0;
cp->rediscmd->rejected_calls = 0; cp->rediscmd->rejected_calls = 0;
...@@ -928,6 +971,224 @@ int RM_CreateCommand(RedisModuleCtx *ctx, const char *name, RedisModuleCmdFunc c ...@@ -928,6 +971,224 @@ int RM_CreateCommand(RedisModuleCtx *ctx, const char *name, RedisModuleCmdFunc c
return REDISMODULE_OK; return REDISMODULE_OK;
} }
void extendKeySpecsIfNeeded(struct redisCommand *cmd) {
/* We extend even if key_specs_num == key_specs_max because
* this function is called prior to adding a new spec */
if (cmd->key_specs_num < cmd->key_specs_max)
return;
cmd->key_specs_max++;
if (cmd->key_specs == cmd->key_specs_static) {
cmd->key_specs = zmalloc(sizeof(keySpec) * cmd->key_specs_max);
memcpy(cmd->key_specs, cmd->key_specs_static, sizeof(keySpec) * cmd->key_specs_num);
} else {
cmd->key_specs = zrealloc(cmd->key_specs, sizeof(keySpec) * cmd->key_specs_max);
}
}
int moduleAddCommandKeySpec(RedisModuleCtx *ctx, const char *name, const char *specflags, int *index) {
int64_t flags = specflags ? commandKeySpecsFlagsFromString(specflags) : 0;
if (flags == -1)
return REDISMODULE_ERR;
struct redisCommand *cmd = lookupCommandByCString(name);
if (cmd == NULL)
return REDISMODULE_ERR;
if (!(cmd->flags & CMD_MODULE))
return REDISMODULE_ERR;
RedisModuleCommandProxy *cp = (RedisModuleCommandProxy*)(unsigned long)cmd->getkeys_proc;
if (cp->module != ctx->module)
return REDISMODULE_ERR;
extendKeySpecsIfNeeded(cmd);
*index = cmd->key_specs_num;
cmd->key_specs[cmd->key_specs_num].begin_search_type = KSPEC_BS_INVALID;
cmd->key_specs[cmd->key_specs_num].find_keys_type = KSPEC_FK_INVALID;
cmd->key_specs[cmd->key_specs_num].flags = flags;
cmd->key_specs_num++;
return REDISMODULE_OK;
}
int moduleSetCommandKeySpecBeginSearch(RedisModuleCtx *ctx, const char *name, int index, keySpec *spec) {
struct redisCommand *cmd = lookupCommandByCString(name);
if (cmd == NULL)
return REDISMODULE_ERR;
if (!(cmd->flags & CMD_MODULE))
return REDISMODULE_ERR;
RedisModuleCommandProxy *cp = (RedisModuleCommandProxy*)(unsigned long)cmd->getkeys_proc;
if (cp->module != ctx->module)
return REDISMODULE_ERR;
if (index >= cmd->key_specs_num)
return REDISMODULE_ERR;
cmd->key_specs[index].begin_search_type = spec->begin_search_type;
cmd->key_specs[index].bs = spec->bs;
return REDISMODULE_OK;
}
int moduleSetCommandKeySpecFindKeys(RedisModuleCtx *ctx, const char *name, int index, keySpec *spec) {
struct redisCommand *cmd = lookupCommandByCString(name);
if (cmd == NULL)
return REDISMODULE_ERR;
if (!(cmd->flags & CMD_MODULE))
return REDISMODULE_ERR;
RedisModuleCommandProxy *cp = (RedisModuleCommandProxy*)(unsigned long)cmd->getkeys_proc;
if (cp->module != ctx->module)
return REDISMODULE_ERR;
if (index >= cmd->key_specs_num)
return REDISMODULE_ERR;
cmd->key_specs[index].find_keys_type = spec->find_keys_type;
cmd->key_specs[index].fk = spec->fk;
/* Refresh legacy range */
populateCommandLegacyRangeSpec(cmd);
/* Refresh movablekeys flag */
populateCommandMovableKeys(cmd);
return REDISMODULE_OK;
}
/* Key specs is a scheme that tries to describe the location
* of key arguments better than the old [first,last,step] scheme
* which is limited and doesn't fit many commands.
*
* This information is used by ACL, Cluster and the 'COMMAND' command.
*
* There are two steps to retrieve the key arguments:
*
* - begin_search (BS): in which index should we start seacrhing for keys?
* - find_keys (FK): relative to the output of BS, how can we will which args are keys?
*
* There are two types of BS:
*
* - index: key args start at a constant index
* - keyword: key args start just after a specific keyword
*
* There are two kinds of FK:
*
* - range: keys end at a specific index (or relative to the last argument)
* - keynum: there's an arg that contains the number of key args somewhere before the keys themselves
*
* This function adds a new key spec to a command, returning a unique id in `spec_id`.
* The caller must then call one of the RedisModule_SetCommandKeySpecBeginSearch* APIs
* followed by one of the RedisModule_SetCommandKeySpecFindKeys* APIs.
*
* It should be called just after RedisModule_CreateCommand.
*
* Example:
*
* if (RedisModule_CreateCommand(ctx,"kspec.smove",kspec_legacy,"",0,0,0) == REDISMODULE_ERR)
* return REDISMODULE_ERR;
*
* if (RedisModule_AddCommandKeySpec(ctx,"kspec.smove","read write",&spec_id) == REDISMODULE_ERR)
* return REDISMODULE_ERR;
* if (RedisModule_SetCommandKeySpecBeginSearchIndex(ctx,"kspec.smove",spec_id,1) == REDISMODULE_ERR)
* return REDISMODULE_ERR;
* if (RedisModule_SetCommandKeySpecFindKeysRange(ctx,"kspec.smove",spec_id,0,1,0) == REDISMODULE_ERR)
* return REDISMODULE_ERR;
*
* if (RedisModule_AddCommandKeySpec(ctx,"kspec.smove","write",&spec_id) == REDISMODULE_ERR)
* return REDISMODULE_ERR;
* if (RedisModule_SetCommandKeySpecBeginSearchIndex(ctx,"kspec.smove",spec_id,2) == REDISMODULE_ERR)
* return REDISMODULE_ERR;
* if (RedisModule_SetCommandKeySpecFindKeysRange(ctx,"kspec.smove",spec_id,0,1,0) == REDISMODULE_ERR)
* return REDISMODULE_ERR;
*
* Returns REDISMODULE_OK on success
*/
int RM_AddCommandKeySpec(RedisModuleCtx *ctx, const char *name, const char *specflags, int *spec_id) {
return moduleAddCommandKeySpec(ctx, name, specflags, spec_id);
}
/* Set a "index" key arguments spec to a command (begin_search step).
* See RedisModule_AddCommandKeySpec's doc.
*
* - `index`: The index from which we start the search for keys
*
* Returns REDISMODULE_OK */
int RM_SetCommandKeySpecBeginSearchIndex(RedisModuleCtx *ctx, const char *name, int spec_id, int index) {
keySpec spec;
spec.begin_search_type = KSPEC_BS_INDEX;
spec.bs.index.pos = index;
return moduleSetCommandKeySpecBeginSearch(ctx, name, spec_id, &spec);
}
/* Set a "keyword" key arguments spec to a command (begin_search step).
* See RedisModule_AddCommandKeySpec's doc.
*
* - `keyword`: The keyword that indicates the beginning of key args
* - `startfrom`: An index in argv from which to start searching.
* Can be negative, which means start search from the end, in reverse
* (Example: -2 means to start in reverse from the panultimate arg)
*
* Returns REDISMODULE_OK */
int RM_SetCommandKeySpecBeginSearchKeyword(RedisModuleCtx *ctx, const char *name, int spec_id, const char *keyword, int startfrom) {
keySpec spec;
spec.begin_search_type = KSPEC_BS_KEYWORD;
spec.bs.keyword.keyword = keyword;
spec.bs.keyword.startfrom = startfrom;
return moduleSetCommandKeySpecBeginSearch(ctx, name, spec_id, &spec);
}
/* Set a "range" key arguments spec to a command (find_keys step).
* See RedisModule_AddCommandKeySpec's doc.
*
* - `lastkey`: Relative index (to the result of the begin_search step) where the last key is.
* Can be negative, in which case it's not relative. -1 indicating till the last argument,
* -2 one before the last and so on.
* - `keystep`: How many args should we skip after finding a key, in order to find the next one.
* - `limit`: If lastkey is -1, we use limit to stop the search by a factor. 0 and 1 mean no limit.
* 2 means 1/2 of the remaining args, 3 means 1/3, and so on.
*
* Returns REDISMODULE_OK */
int RM_SetCommandKeySpecFindKeysRange(RedisModuleCtx *ctx, const char *name, int spec_id, int lastkey, int keystep, int limit) {
keySpec spec;
spec.find_keys_type = KSPEC_FK_RANGE;
spec.fk.range.lastkey = lastkey;
spec.fk.range.keystep = keystep;
spec.fk.range.limit = limit;
return moduleSetCommandKeySpecFindKeys(ctx, name, spec_id, &spec);
}
/* Set a "keynum" key arguments spec to a command (find_keys step).
* See RedisModule_AddCommandKeySpec's doc.
*
* - `keynumidx`: Relative index (to the result of the begin_search step) where the arguments that
* contains the number of keys is.
* - `firstkey`: Relative index (to the result of the begin_search step) where the first key is
* found (Usually it's just after keynumidx, so it should be keynumidx+1)
* - `keystep`: How many args should we skip after finding a key, in order to find the next one.
*
* Returns REDISMODULE_OK */
int RM_SetCommandKeySpecFindKeysKeynum(RedisModuleCtx *ctx, const char *name, int spec_id, int keynumidx, int firstkey, int keystep) {
keySpec spec;
spec.find_keys_type = KSPEC_FK_KEYNUM;
spec.fk.keynum.keynumidx = keynumidx;
spec.fk.keynum.firstkey = firstkey;
spec.fk.keynum.keystep = keystep;
return moduleSetCommandKeySpecFindKeys(ctx, name, spec_id, &spec);
}
/* -------------------------------------------------------------------------- /* --------------------------------------------------------------------------
* ## Module information and time measurement * ## Module information and time measurement
* -------------------------------------------------------------------------- */ * -------------------------------------------------------------------------- */
...@@ -9033,6 +9294,8 @@ void moduleUnregisterCommands(struct RedisModule *module) { ...@@ -9033,6 +9294,8 @@ void moduleUnregisterCommands(struct RedisModule *module) {
(void*)(unsigned long)cmd->getkeys_proc; (void*)(unsigned long)cmd->getkeys_proc;
sds cmdname = cp->rediscmd->name; sds cmdname = cp->rediscmd->name;
if (cp->module == module) { if (cp->module == module) {
if (cp->rediscmd->key_specs != cp->rediscmd->key_specs_static)
zfree(cp->rediscmd->key_specs);
dictDelete(server.commands,cmdname); dictDelete(server.commands,cmdname);
dictDelete(server.orig_commands,cmdname); dictDelete(server.orig_commands,cmdname);
sdsfree(cmdname); sdsfree(cmdname);
...@@ -9516,7 +9779,7 @@ int *RM_GetCommandKeys(RedisModuleCtx *ctx, RedisModuleString **argv, int argc, ...@@ -9516,7 +9779,7 @@ int *RM_GetCommandKeys(RedisModuleCtx *ctx, RedisModuleString **argv, int argc,
} }
/* Bail out if command has no keys */ /* Bail out if command has no keys */
if (cmd->getkeys_proc == NULL && cmd->firstkey == 0) { if (cmd->getkeys_proc == NULL && cmd->key_specs_num == 0) {
errno = 0; errno = 0;
return NULL; return NULL;
} }
...@@ -10072,4 +10335,9 @@ void moduleRegisterCoreAPI(void) { ...@@ -10072,4 +10335,9 @@ void moduleRegisterCoreAPI(void) {
REGISTER_API(DefragShouldStop); REGISTER_API(DefragShouldStop);
REGISTER_API(DefragCursorSet); REGISTER_API(DefragCursorSet);
REGISTER_API(DefragCursorGet); REGISTER_API(DefragCursorGet);
REGISTER_API(AddCommandKeySpec);
REGISTER_API(SetCommandKeySpecBeginSearchIndex);
REGISTER_API(SetCommandKeySpecBeginSearchKeyword);
REGISTER_API(SetCommandKeySpecFindKeysRange);
REGISTER_API(SetCommandKeySpecFindKeysKeynum);
} }
...@@ -826,6 +826,11 @@ REDISMODULE_API int (*RedisModule_GetKeyspaceNotificationFlagsAll)() REDISMODULE ...@@ -826,6 +826,11 @@ REDISMODULE_API int (*RedisModule_GetKeyspaceNotificationFlagsAll)() REDISMODULE
REDISMODULE_API int (*RedisModule_IsSubEventSupported)(RedisModuleEvent event, uint64_t subevent) REDISMODULE_ATTR; REDISMODULE_API int (*RedisModule_IsSubEventSupported)(RedisModuleEvent event, uint64_t subevent) REDISMODULE_ATTR;
REDISMODULE_API int (*RedisModule_GetServerVersion)() REDISMODULE_ATTR; REDISMODULE_API int (*RedisModule_GetServerVersion)() REDISMODULE_ATTR;
REDISMODULE_API int (*RedisModule_GetTypeMethodVersion)() REDISMODULE_ATTR; REDISMODULE_API int (*RedisModule_GetTypeMethodVersion)() REDISMODULE_ATTR;
REDISMODULE_API int (*RedisModule_AddCommandKeySpec)(RedisModuleCtx *ctx, const char *name, const char *specflags, int *spec_id) REDISMODULE_ATTR;
REDISMODULE_API int (*RedisModule_SetCommandKeySpecBeginSearchIndex)(RedisModuleCtx *ctx, const char *name, int spec_id, int index) REDISMODULE_ATTR;
REDISMODULE_API int (*RedisModule_SetCommandKeySpecBeginSearchKeyword)(RedisModuleCtx *ctx, const char *name, int spec_id, const char *keyword, int startfrom) REDISMODULE_ATTR;
REDISMODULE_API int (*RedisModule_SetCommandKeySpecFindKeysRange)(RedisModuleCtx *ctx, const char *name, int spec_id, int lastkey, int keystep, int limit) REDISMODULE_ATTR;
REDISMODULE_API int (*RedisModule_SetCommandKeySpecFindKeysKeynum)(RedisModuleCtx *ctx, const char *name, int spec_id, int keynumidx, int firstkey, int keystep) REDISMODULE_ATTR;
/* Experimental APIs */ /* Experimental APIs */
#ifdef REDISMODULE_EXPERIMENTAL_API #ifdef REDISMODULE_EXPERIMENTAL_API
...@@ -1132,6 +1137,11 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int ...@@ -1132,6 +1137,11 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int
REDISMODULE_GET_API(IsSubEventSupported); REDISMODULE_GET_API(IsSubEventSupported);
REDISMODULE_GET_API(GetServerVersion); REDISMODULE_GET_API(GetServerVersion);
REDISMODULE_GET_API(GetTypeMethodVersion); REDISMODULE_GET_API(GetTypeMethodVersion);
REDISMODULE_GET_API(AddCommandKeySpec);
REDISMODULE_GET_API(SetCommandKeySpecBeginSearchIndex);
REDISMODULE_GET_API(SetCommandKeySpecBeginSearchKeyword);
REDISMODULE_GET_API(SetCommandKeySpecFindKeysRange);
REDISMODULE_GET_API(SetCommandKeySpecFindKeysKeynum);
#ifdef REDISMODULE_EXPERIMENTAL_API #ifdef REDISMODULE_EXPERIMENTAL_API
REDISMODULE_GET_API(GetThreadSafeContext); REDISMODULE_GET_API(GetThreadSafeContext);
......
...@@ -465,21 +465,21 @@ void sentinelConfigGetCommand(client *c); ...@@ -465,21 +465,21 @@ void sentinelConfigGetCommand(client *c);
void sentinelConfigSetCommand(client *c); void sentinelConfigSetCommand(client *c);
struct redisCommand sentinelcmds[] = { struct redisCommand sentinelcmds[] = {
{"ping",pingCommand,1,"fast @connection",0,NULL,0,0,0,0,0}, {"ping",pingCommand,1,"fast @connection"},
{"sentinel",sentinelCommand,-2,"admin",0,NULL,0,0,0,0,0}, {"sentinel",sentinelCommand,-2,"admin"},
{"subscribe",subscribeCommand,-2,"pub-sub",0,NULL,0,0,0,0,0}, {"subscribe",subscribeCommand,-2,"pub-sub"},
{"unsubscribe",unsubscribeCommand,-1,"pub-sub",0,NULL,0,0,0,0,0}, {"unsubscribe",unsubscribeCommand,-1,"pub-sub"},
{"psubscribe",psubscribeCommand,-2,"pub-sub",0,NULL,0,0,0,0,0}, {"psubscribe",psubscribeCommand,-2,"pub-sub"},
{"punsubscribe",punsubscribeCommand,-1,"pub-sub",0,NULL,0,0,0,0,0}, {"punsubscribe",punsubscribeCommand,-1,"pub-sub"},
{"publish",sentinelPublishCommand,3,"pub-sub fast",0,NULL,0,0,0,0,0}, {"publish",sentinelPublishCommand,3,"pub-sub fast"},
{"info",sentinelInfoCommand,-1,"random @dangerous",0,NULL,0,0,0,0,0}, {"info",sentinelInfoCommand,-1,"random @dangerous"},
{"role",sentinelRoleCommand,1,"fast read-only @dangerous",0,NULL,0,0,0,0,0}, {"role",sentinelRoleCommand,1,"fast read-only @dangerous"},
{"client",clientCommand,-2,"admin random @connection",0,NULL,0,0,0,0,0}, {"client",clientCommand,-2,"admin random @connection"},
{"shutdown",shutdownCommand,-1,"admin",0,NULL,0,0,0,0,0}, {"shutdown",shutdownCommand,-1,"admin"},
{"auth",authCommand,-2,"no-auth fast @connection",0,NULL,0,0,0,0,0}, {"auth",authCommand,-2,"no-auth fast @connection"},
{"hello",helloCommand,-1,"no-auth fast @connection",0,NULL,0,0,0,0,0}, {"hello",helloCommand,-1,"no-auth fast @connection"},
{"acl",aclCommand,-2,"admin",0,NULL,0,0,0,0,0,0}, {"acl",aclCommand,-2,"admin"},
{"command",commandCommand,-1, "random @connection", 0,NULL,0,0,0,0,0,0} {"command",commandCommand,-1, "random @connection"}
}; };
/* this array is used for sentinel config lookup, which need to be loaded /* this array is used for sentinel config lookup, which need to be loaded
...@@ -525,7 +525,7 @@ void initSentinel(void) { ...@@ -525,7 +525,7 @@ void initSentinel(void) {
/* Translate the command string flags description into an actual /* Translate the command string flags description into an actual
* set of flags. */ * set of flags. */
if (populateCommandTableParseFlags(cmd,cmd->sflags) == C_ERR) if (populateSingleCommand(cmd,cmd->sflags) == C_ERR)
serverPanic("Unsupported command flag"); serverPanic("Unsupported command flag");
} }
......
This diff is collapsed.
...@@ -84,6 +84,10 @@ typedef long long ustime_t; /* microsecond time type. */ ...@@ -84,6 +84,10 @@ typedef long long ustime_t; /* microsecond time type. */
#include "endianconv.h" #include "endianconv.h"
#include "crc64.h" #include "crc64.h"
/* min/max */
#define min(a, b) ((a) < (b) ? (a) : (b))
#define max(a, b) ((a) > (b) ? (a) : (b))
/* Error codes */ /* Error codes */
#define C_OK 0 #define C_OK 0
#define C_ERR -1 #define C_ERR -1
...@@ -193,6 +197,12 @@ extern int configOOMScoreAdjValuesDefaults[CONFIG_OOM_COUNT]; ...@@ -193,6 +197,12 @@ extern int configOOMScoreAdjValuesDefaults[CONFIG_OOM_COUNT];
#define CMD_NO_AUTH (1ULL<<15) /* "no-auth" flag */ #define CMD_NO_AUTH (1ULL<<15) /* "no-auth" flag */
#define CMD_MAY_REPLICATE (1ULL<<16) /* "may-replicate" flag */ #define CMD_MAY_REPLICATE (1ULL<<16) /* "may-replicate" flag */
/* Key argument flags. Please check the command table defined in the server.c file
* for more information about the meaning of every flag. */
#define CMD_KEY_WRITE (1ULL<<0) /* "write" flag */
#define CMD_KEY_READ (1ULL<<1) /* "read" flag */
#define CMD_KEY_INCOMPLETE (1ULL<<2) /* "incomplete" flag (meaning that the keyspec might not point out to all keys it should cover) */
/* Command flags used by the module system. */ /* Command flags used by the module system. */
#define CMD_MODULE_GETKEYS (1ULL<<17) /* Use the modules getkeys interface. */ #define CMD_MODULE_GETKEYS (1ULL<<17) /* Use the modules getkeys interface. */
#define CMD_MODULE_NO_CLUSTER (1ULL<<18) /* Deny on Redis Cluster. */ #define CMD_MODULE_NO_CLUSTER (1ULL<<18) /* Deny on Redis Cluster. */
...@@ -1700,27 +1710,120 @@ typedef struct { ...@@ -1700,27 +1710,120 @@ typedef struct {
} getKeysResult; } getKeysResult;
#define GETKEYS_RESULT_INIT { {0}, NULL, 0, MAX_KEYS_BUFFER } #define GETKEYS_RESULT_INIT { {0}, NULL, 0, MAX_KEYS_BUFFER }
/* Key specs definitions.
*
* Brief: This is a scheme that tries to describe the location
* of key arguments better than the old [first,last,step] scheme
* which is limited and doesn't fit many commands.
*
* There are two steps:
* 1. begin_search (BS): in which index should we start seacrhing for keys?
* 2. find_keys (FK): relative to the output of BS, how can we will which args are keys?
*
* There are two types of BS:
* 1. index: key args start at a constant index
* 2. keyword: key args start just after a specific keyword
*
* There are two kinds of FK:
* 1. range: keys end at a specific index (or relative to the last argument)
* 2. keynum: there's an arg that contains the number of key args somewhere before the keys themselves
*/
typedef enum {
KSPEC_BS_INVALID = 0, /* Must be 0 */
KSPEC_BS_UNKNOWN,
KSPEC_BS_INDEX,
KSPEC_BS_KEYWORD
} kspec_bs_type;
typedef enum {
KSPEC_FK_INVALID = 0, /* Must be 0 */
KSPEC_FK_UNKNOWN,
KSPEC_FK_RANGE,
KSPEC_FK_KEYNUM
} kspec_fk_type;
typedef struct {
/* Declarative data */
const char *sflags;
kspec_bs_type begin_search_type;
union {
struct {
/* The index from which we start the search for keys */
int pos;
} index;
struct {
/* The keyword that indicates the beginning of key args */
const char *keyword;
/* An index in argv from which to start searching.
* Can be negative, which means start search from the end, in reverse
* (Example: -2 means to start in reverse from the panultimate arg) */
int startfrom;
} keyword;
} bs;
kspec_fk_type find_keys_type;
union {
/* NOTE: Indices in this struct are relative to the result of the begin_search step!
* These are: range.lastkey, keynum.keynumidx, keynum.firstkey */
struct {
/* Index of the last key.
* Can be negative, in which case it's not relative. -1 indicating till the last argument,
* -2 one before the last and so on. */
int lastkey;
/* How many args should we skip after finding a key, in order to find the next one. */
int keystep;
/* If lastkey is -1, we use limit to stop the search by a factor. 0 and 1 mean no limit.
* 2 means 1/2 of the remaining args, 3 means 1/3, and so on. */
int limit;
} range;
struct {
/* Index of the argument containing the number of keys to come */
int keynumidx;
/* Index of the fist key (Usually it's just after keynumidx, in
* which case it should be set to keynumidx+1). */
int firstkey;
/* How many args should we skip after finding a key, in order to find the next one. */
int keystep;
} keynum;
} fk;
/* Runtime data */
uint64_t flags;
} keySpec;
/* Number of static key specs */
#define STATIC_KEY_SPECS_NUM 4
typedef void redisCommandProc(client *c); typedef void redisCommandProc(client *c);
typedef int redisGetKeysProc(struct redisCommand *cmd, robj **argv, int argc, getKeysResult *result); typedef int redisGetKeysProc(struct redisCommand *cmd, robj **argv, int argc, getKeysResult *result);
struct redisCommand { struct redisCommand {
/* Declarative data */
char *name; char *name;
redisCommandProc *proc; redisCommandProc *proc;
int arity; int arity;
char *sflags; /* Flags as string representation, one char per flag. */ char *sflags; /* Flags as string representation, one char per flag. */
uint64_t flags; /* The actual flags, obtained from the 'sflags' field. */ keySpec key_specs_static[STATIC_KEY_SPECS_NUM];
/* Use a function to determine keys arguments in a command line. /* Use a function to determine keys arguments in a command line.
* Used for Redis Cluster redirect. */ * Used for Redis Cluster redirect. */
redisGetKeysProc *getkeys_proc; redisGetKeysProc *getkeys_proc;
/* Runtime data */
uint64_t flags; /* The actual flags, obtained from the 'sflags' field. */
/* What keys should be loaded in background when calling this command? */ /* What keys should be loaded in background when calling this command? */
int firstkey; /* The first argument that's a key (0 = no keys) */
int lastkey; /* The last argument that's a key */
int keystep; /* The step between first and last key */
long long microseconds, calls, rejected_calls, failed_calls; long long microseconds, calls, rejected_calls, failed_calls;
int id; /* Command ID. This is a progressive ID starting from 0 that int id; /* Command ID. This is a progressive ID starting from 0 that
is assigned at runtime, and is used in order to check is assigned at runtime, and is used in order to check
ACLs. A connection is able to execute a given command if ACLs. A connection is able to execute a given command if
the user associated to the connection has this command the user associated to the connection has this command
bit set in the bitmap of allowed commands. */ bit set in the bitmap of allowed commands. */
keySpec *key_specs;
keySpec legacy_range_key_spec; /* The legacy (first,last,step) key spec is
* still maintained (if applicable) so that
* we can still support the reply format of
* COMMAND INFO and COMMAND GETKEYS */
int key_specs_num;
int key_specs_max;
int movablekeys; /* See populateCommandMovableKeys */
}; };
struct redisError { struct redisError {
...@@ -1810,6 +1913,9 @@ extern dict *modules; ...@@ -1810,6 +1913,9 @@ extern dict *modules;
* Functions prototypes * Functions prototypes
*----------------------------------------------------------------------------*/ *----------------------------------------------------------------------------*/
/* Key arguments specs */
void populateCommandLegacyRangeSpec(struct redisCommand *c);
/* Modules */ /* Modules */
void moduleInitModulesSystem(void); void moduleInitModulesSystem(void);
void moduleInitModulesSystemLast(void); void moduleInitModulesSystemLast(void);
...@@ -2791,7 +2897,8 @@ void serverLogHexDump(int level, char *descr, void *value, size_t len); ...@@ -2791,7 +2897,8 @@ void serverLogHexDump(int level, char *descr, void *value, size_t len);
int memtest_preserving_test(unsigned long *m, size_t bytes, int passes); int memtest_preserving_test(unsigned long *m, size_t bytes, int passes);
void mixDigest(unsigned char *digest, void *ptr, size_t len); void mixDigest(unsigned char *digest, void *ptr, size_t len);
void xorDigest(unsigned char *digest, void *ptr, size_t len); void xorDigest(unsigned char *digest, void *ptr, size_t len);
int populateCommandTableParseFlags(struct redisCommand *c, char *strflags); int populateSingleCommand(struct redisCommand *c, char *strflags);
void populateCommandMovableKeys(struct redisCommand *cmd);
void debugDelay(int usec); void debugDelay(int usec);
void killIOThreads(void); void killIOThreads(void);
void killThreads(void); void killThreads(void);
......
...@@ -37,6 +37,7 @@ TEST_MODULES = \ ...@@ -37,6 +37,7 @@ TEST_MODULES = \
test_lazyfree.so \ test_lazyfree.so \
timer.so \ timer.so \
defragtest.so \ defragtest.so \
keyspecs.so \
hash.so \ hash.so \
zset.so \ zset.so \
stream.so \ stream.so \
......
#include "redismodule.h"
#define UNUSED(V) ((void) V)
int kspec_legacy(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
UNUSED(argv);
UNUSED(argc);
RedisModule_ReplyWithSimpleString(ctx, "OK");
return REDISMODULE_OK;
}
int kspec_complex1(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
UNUSED(argv);
UNUSED(argc);
RedisModule_ReplyWithSimpleString(ctx, "OK");
return REDISMODULE_OK;
}
int kspec_complex2(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
UNUSED(argv);
UNUSED(argc);
RedisModule_ReplyWithSimpleString(ctx, "OK");
return REDISMODULE_OK;
}
int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
REDISMODULE_NOT_USED(argv);
REDISMODULE_NOT_USED(argc);
int spec_id;
if (RedisModule_Init(ctx, "keyspecs", 1, REDISMODULE_APIVER_1)== REDISMODULE_ERR)
return REDISMODULE_ERR;
/* Test legacy range "gluing" */
if (RedisModule_CreateCommand(ctx,"kspec.legacy",kspec_legacy,"",0,0,0) == REDISMODULE_ERR)
return REDISMODULE_ERR;
if (RedisModule_AddCommandKeySpec(ctx,"kspec.legacy","read",&spec_id) == REDISMODULE_ERR)
return REDISMODULE_ERR;
if (RedisModule_SetCommandKeySpecBeginSearchIndex(ctx,"kspec.legacy",spec_id,1) == REDISMODULE_ERR)
return REDISMODULE_ERR;
if (RedisModule_SetCommandKeySpecFindKeysRange(ctx,"kspec.legacy",spec_id,0,1,0) == REDISMODULE_ERR)
return REDISMODULE_ERR;
if (RedisModule_AddCommandKeySpec(ctx,"kspec.legacy","write",&spec_id) == REDISMODULE_ERR)
return REDISMODULE_ERR;
if (RedisModule_SetCommandKeySpecBeginSearchIndex(ctx,"kspec.legacy",spec_id,2) == REDISMODULE_ERR)
return REDISMODULE_ERR;
if (RedisModule_SetCommandKeySpecFindKeysRange(ctx,"kspec.legacy",spec_id,0,1,0) == REDISMODULE_ERR)
return REDISMODULE_ERR;
/* First is legacy, rest are new specs */
if (RedisModule_CreateCommand(ctx,"kspec.complex1",kspec_complex1,"",1,1,1) == REDISMODULE_ERR)
return REDISMODULE_ERR;
if (RedisModule_AddCommandKeySpec(ctx,"kspec.complex1","write",&spec_id) == REDISMODULE_ERR)
return REDISMODULE_ERR;
if (RedisModule_SetCommandKeySpecBeginSearchKeyword(ctx,"kspec.complex1",spec_id,"STORE",2) == REDISMODULE_ERR)
return REDISMODULE_ERR;
if (RedisModule_SetCommandKeySpecFindKeysRange(ctx,"kspec.complex1",spec_id,0,1,0) == REDISMODULE_ERR)
return REDISMODULE_ERR;
if (RedisModule_AddCommandKeySpec(ctx,"kspec.complex1","read",&spec_id) == REDISMODULE_ERR)
return REDISMODULE_ERR;
if (RedisModule_SetCommandKeySpecBeginSearchKeyword(ctx,"kspec.complex1",spec_id,"KEYS",2) == REDISMODULE_ERR)
return REDISMODULE_ERR;
if (RedisModule_SetCommandKeySpecFindKeysKeynum(ctx,"kspec.complex1",spec_id,0,1,1) == REDISMODULE_ERR)
return REDISMODULE_ERR;
/* First is not legacy, more than STATIC_KEYS_SPECS_NUM specs */
if (RedisModule_CreateCommand(ctx,"kspec.complex2",kspec_complex2,"",0,0,0) == REDISMODULE_ERR)
return REDISMODULE_ERR;
if (RedisModule_AddCommandKeySpec(ctx,"kspec.complex2","write",&spec_id) == REDISMODULE_ERR)
return REDISMODULE_ERR;
if (RedisModule_SetCommandKeySpecBeginSearchKeyword(ctx,"kspec.complex2",spec_id,"STORE",5) == REDISMODULE_ERR)
return REDISMODULE_ERR;
if (RedisModule_SetCommandKeySpecFindKeysRange(ctx,"kspec.complex2",spec_id,0,1,0) == REDISMODULE_ERR)
return REDISMODULE_ERR;
if (RedisModule_AddCommandKeySpec(ctx,"kspec.complex2","read",&spec_id) == REDISMODULE_ERR)
return REDISMODULE_ERR;
if (RedisModule_SetCommandKeySpecBeginSearchIndex(ctx,"kspec.complex2",spec_id,1) == REDISMODULE_ERR)
return REDISMODULE_ERR;
if (RedisModule_SetCommandKeySpecFindKeysRange(ctx,"kspec.complex2",spec_id,0,1,0) == REDISMODULE_ERR)
return REDISMODULE_ERR;
if (RedisModule_AddCommandKeySpec(ctx,"kspec.complex2","read",&spec_id) == REDISMODULE_ERR)
return REDISMODULE_ERR;
if (RedisModule_SetCommandKeySpecBeginSearchIndex(ctx,"kspec.complex2",spec_id,2) == REDISMODULE_ERR)
return REDISMODULE_ERR;
if (RedisModule_SetCommandKeySpecFindKeysRange(ctx,"kspec.complex2",spec_id,0,1,0) == REDISMODULE_ERR)
return REDISMODULE_ERR;
if (RedisModule_AddCommandKeySpec(ctx,"kspec.complex2","write",&spec_id) == REDISMODULE_ERR)
return REDISMODULE_ERR;
if (RedisModule_SetCommandKeySpecBeginSearchIndex(ctx,"kspec.complex2",spec_id,3) == REDISMODULE_ERR)
return REDISMODULE_ERR;
if (RedisModule_SetCommandKeySpecFindKeysKeynum(ctx,"kspec.complex2",spec_id,0,1,1) == REDISMODULE_ERR)
return REDISMODULE_ERR;
if (RedisModule_AddCommandKeySpec(ctx,"kspec.complex2","write",&spec_id) == REDISMODULE_ERR)
return REDISMODULE_ERR;
if (RedisModule_SetCommandKeySpecBeginSearchKeyword(ctx,"kspec.complex2",spec_id,"MOREKEYS",5) == REDISMODULE_ERR)
return REDISMODULE_ERR;
if (RedisModule_SetCommandKeySpecFindKeysRange(ctx,"kspec.complex2",spec_id,-1,1,0) == REDISMODULE_ERR)
return REDISMODULE_ERR;
return REDISMODULE_OK;
}
set testmodule [file normalize tests/modules/keyspecs.so]
start_server {tags {"modules"}} {
r module load $testmodule
test "Module key specs: Legacy" {
set reply [r command info kspec.legacy]
assert_equal $reply {{kspec.legacy -1 {} 1 2 1 {} {{flags read begin_search {type index spec {index 1}} find_keys {type range spec {lastkey 0 keystep 1 limit 0}}} {flags write begin_search {type index spec {index 2}} find_keys {type range spec {lastkey 0 keystep 1 limit 0}}}}}}
}
test "Module key specs: Complex specs, case 1" {
set reply [r command info kspec.complex1]
assert_equal $reply {{kspec.complex1 -1 movablekeys 1 1 1 {} {{flags {} begin_search {type index spec {index 1}} find_keys {type range spec {lastkey 0 keystep 1 limit 0}}} {flags write begin_search {type keyword spec {keyword STORE startfrom 2}} find_keys {type range spec {lastkey 0 keystep 1 limit 0}}} {flags read begin_search {type keyword spec {keyword KEYS startfrom 2}} find_keys {type keynum spec {keynumidx 0 firstkey 1 keystep 1}}}}}}
}
test "Module key specs: Complex specs, case 2" {
set reply [r command info kspec.complex2]
assert_equal $reply {{kspec.complex2 -1 movablekeys 1 2 1 {} {{flags write begin_search {type keyword spec {keyword STORE startfrom 5}} find_keys {type range spec {lastkey 0 keystep 1 limit 0}}} {flags read begin_search {type index spec {index 1}} find_keys {type range spec {lastkey 0 keystep 1 limit 0}}} {flags read begin_search {type index spec {index 2}} find_keys {type range spec {lastkey 0 keystep 1 limit 0}}} {flags write begin_search {type index spec {index 3}} find_keys {type keynum spec {keynumidx 0 firstkey 1 keystep 1}}} {flags write begin_search {type keyword spec {keyword MOREKEYS startfrom 5}} find_keys {type range spec {lastkey -1 keystep 1 limit 0}}}}}}
}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment