Commit a83e3663 authored by Oran Agra's avatar Oran Agra
Browse files

Merge remote-tracking branch 'origin/unstable' into 7.0

parents d5915a16 5860fa3d
......@@ -4,8 +4,14 @@
"complexity": "O(1)",
"group": "stream",
"since": "5.0.0",
"arity": 3,
"arity": -3,
"function": "xsetidCommand",
"history": [
[
"7.0.0",
"Added the `entries_added` and `max_deleted_entry_id` arguments."
]
],
"command_flags": [
"WRITE",
"DENYOOM",
......@@ -43,6 +49,18 @@
{
"name": "last-id",
"type": "string"
},
{
"name": "entries_added",
"token": "ENTRIESADDED",
"type": "integer",
"optional": true
},
{
"name": "max_deleted_entry_id",
"token": "MAXDELETEDID",
"type": "string",
"optional": true
}
]
}
......
......@@ -178,6 +178,21 @@ static int connSocketWrite(connection *conn, const void *data, size_t data_len)
return ret;
}
static int connSocketWritev(connection *conn, const struct iovec *iov, int iovcnt) {
int ret = writev(conn->fd, iov, iovcnt);
if (ret < 0 && errno != EAGAIN) {
conn->last_errno = errno;
/* Don't overwrite the state of a connection that is not already
* connected, not to mess with handler callbacks.
*/
if (errno != EINTR && conn->state == CONN_STATE_CONNECTED)
conn->state = CONN_STATE_ERROR;
}
return ret;
}
static int connSocketRead(connection *conn, void *buf, size_t buf_len) {
int ret = read(conn->fd, buf, buf_len);
if (!ret) {
......@@ -349,6 +364,7 @@ ConnectionType CT_Socket = {
.ae_handler = connSocketEventHandler,
.close = connSocketClose,
.write = connSocketWrite,
.writev = connSocketWritev,
.read = connSocketRead,
.accept = connSocketAccept,
.connect = connSocketConnect,
......
......@@ -32,6 +32,7 @@
#define __REDIS_CONNECTION_H
#include <errno.h>
#include <sys/uio.h>
#define CONN_INFO_LEN 32
......@@ -59,6 +60,7 @@ typedef struct ConnectionType {
void (*ae_handler)(struct aeEventLoop *el, int fd, void *clientData, int mask);
int (*connect)(struct connection *conn, const char *addr, int port, const char *source_addr, ConnectionCallbackFunc connect_handler);
int (*write)(struct connection *conn, const void *data, size_t data_len);
int (*writev)(struct connection *conn, const struct iovec *iov, int iovcnt);
int (*read)(struct connection *conn, void *buf, size_t buf_len);
void (*close)(struct connection *conn);
int (*accept)(struct connection *conn, ConnectionCallbackFunc accept_handler);
......@@ -142,6 +144,18 @@ static inline int connWrite(connection *conn, const void *data, size_t data_len)
return conn->type->write(conn, data, data_len);
}
/* Gather output data from the iovcnt buffers specified by the members of the iov
* array: iov[0], iov[1], ..., iov[iovcnt-1] and write to connection, behaves the same as writev(3).
*
* Like writev(3), a short write is possible. A -1 return indicates an error.
*
* The caller should NOT rely on errno. Testing for an EAGAIN-like condition, use
* connGetState() to see if the connection state is still CONN_STATE_CONNECTED.
*/
static inline int connWritev(connection *conn, const struct iovec *iov, int iovcnt) {
return conn->type->writev(conn, iov, iovcnt);
}
/* Read from the connection, behaves the same as read(2).
*
* Like read(2), a short read is possible. A return value of 0 will indicate the
......
......@@ -83,16 +83,16 @@ robj *lookupKey(redisDb *db, robj *key, int flags) {
robj *val = NULL;
if (de) {
val = dictGetVal(de);
int force_delete_expired = flags & LOOKUP_WRITE;
if (force_delete_expired) {
/* Forcing deletion of expired keys on a replica makes the replica
* inconsistent with the master. The reason it's allowed for write
* commands is to make writable replicas behave consistently. It
* shall not be used in readonly commands. Modules are accepted so
* that we don't break old modules. */
client *c = server.in_script ? scriptGetClient() : server.current_client;
serverAssert(!c || !c->cmd || (c->cmd->flags & (CMD_WRITE|CMD_MODULE)));
}
/* Forcing deletion of expired keys on a replica makes the replica
* inconsistent with the master. We forbid it on readonly replicas, but
* we have to allow it on writable replicas to make write commands
* behave consistently.
*
* It's possible that the WRITE flag is set even during a readonly
* command, since the command may trigger events that cause modules to
* perform additional writes. */
int is_ro_replica = server.masterhost && server.repl_slave_ro;
int force_delete_expired = flags & LOOKUP_WRITE && !is_ro_replica;
if (expireIfNeeded(db, key, force_delete_expired)) {
/* The key is no longer valid. */
val = NULL;
......@@ -1340,6 +1340,11 @@ int dbSwapDatabases(int id1, int id2) {
redisDb aux = server.db[id1];
redisDb *db1 = &server.db[id1], *db2 = &server.db[id2];
/* Swapdb should make transaction fail if there is any
* client watching keys */
touchAllWatchedKeysInDb(db1, db2);
touchAllWatchedKeysInDb(db2, db1);
/* Swap hash tables. Note that we don't swap blocking_keys,
* ready_keys and watched_keys, since we want clients to
* remain in the same DB they were. */
......@@ -1361,14 +1366,9 @@ int dbSwapDatabases(int id1, int id2) {
* However normally we only do this check for efficiency reasons
* in dbAdd() when a list is created. So here we need to rescan
* the list of clients blocked on lists and signal lists as ready
* if needed.
*
* Also the swapdb should make transaction fail if there is any
* client watching keys */
* if needed. */
scanDatabaseForReadyLists(db1);
touchAllWatchedKeysInDb(db1, db2);
scanDatabaseForReadyLists(db2);
touchAllWatchedKeysInDb(db2, db1);
return C_OK;
}
......@@ -1387,6 +1387,10 @@ void swapMainDbWithTempDb(redisDb *tempDb) {
redisDb aux = server.db[i];
redisDb *activedb = &server.db[i], *newdb = &tempDb[i];
/* Swapping databases should make transaction fail if there is any
* client watching keys. */
touchAllWatchedKeysInDb(activedb, newdb);
/* Swap hash tables. Note that we don't swap blocking_keys,
* ready_keys and watched_keys, since clients
* remain in the same DB they were. */
......@@ -1408,12 +1412,8 @@ void swapMainDbWithTempDb(redisDb *tempDb) {
* However normally we only do this check for efficiency reasons
* in dbAdd() when a list is created. So here we need to rescan
* the list of clients blocked on lists and signal lists as ready
* if needed.
*
* Also the swapdb should make transaction fail if there is any
* client watching keys. */
* if needed. */
scanDatabaseForReadyLists(activedb);
touchAllWatchedKeysInDb(activedb, newdb);
}
trackingInvalidateKeysOnFlush(1);
......@@ -1692,7 +1692,7 @@ int64_t getAllKeySpecsFlags(struct redisCommand *cmd, int inv) {
/* Fetch the keys based of the provided key specs. Returns the number of keys found, or -1 on error.
* There are several flags that can be used to modify how this function finds keys in a command.
*
* GET_KEYSPEC_INCLUDE_CHANNELS: Return channels as if they were keys.
* GET_KEYSPEC_INCLUDE_NOT_KEYS: Return 'fake' keys as if they were keys.
* GET_KEYSPEC_RETURN_PARTIAL: Skips invalid and incomplete keyspecs but returns the keys
* found in other valid keyspecs.
*/
......@@ -1703,8 +1703,8 @@ int getKeysUsingKeySpecs(struct redisCommand *cmd, robj **argv, int argc, int se
for (j = 0; j < cmd->key_specs_num; j++) {
keySpec *spec = cmd->key_specs + j;
serverAssert(spec->begin_search_type != KSPEC_BS_INVALID);
/* Skip specs that represent channels instead of keys */
if ((spec->flags & CMD_KEY_CHANNEL) && !(search_flags & GET_KEYSPEC_INCLUDE_CHANNELS)) {
/* Skip specs that represent 'fake' keys */
if ((spec->flags & CMD_KEY_NOT_KEY) && !(search_flags & GET_KEYSPEC_INCLUDE_NOT_KEYS)) {
continue;
}
......@@ -1821,31 +1821,123 @@ invalid_spec:
* associated with how Redis will access the key.
*
* 'cmd' must be point to the corresponding entry into the redisCommand
* table, according to the command name in argv[0].
*
* This function uses the command's key specs, which contain the key-spec flags,
* (e.g. RO / RW) and only resorts to the command-specific helper function if
* any of the keys-specs are marked as INCOMPLETE. */
* table, according to the command name in argv[0]. */
int getKeysFromCommandWithSpecs(struct redisCommand *cmd, robj **argv, int argc, int search_flags, getKeysResult *result) {
if (cmd->flags & CMD_MODULE_GETKEYS) {
/* The command has at least one key-spec not marked as NOT_KEY */
int has_keyspec = (getAllKeySpecsFlags(cmd, 1) & CMD_KEY_NOT_KEY);
/* The command has at least one key-spec marked as VARIABLE_FLAGS */
int has_varflags = (getAllKeySpecsFlags(cmd, 0) & CMD_KEY_VARIABLE_FLAGS);
/* Flags indicating that we have a getkeys callback */
int has_module_getkeys = cmd->flags & CMD_MODULE_GETKEYS;
int has_native_getkeys = !(cmd->flags & CMD_MODULE) && cmd->getkeys_proc;
/* The key-spec that's auto generated by RM_CreateCommand sets VARIABLE_FLAGS since no flags are given.
* If the module provides getkeys callback, we'll prefer it, but if it didn't, we'll use key-spec anyway. */
if ((cmd->flags & CMD_MODULE) && has_varflags && !has_module_getkeys)
has_varflags = 0;
/* We prefer key-specs if there are any, and their flags are reliable. */
if (has_keyspec && !has_varflags) {
int ret = getKeysUsingKeySpecs(cmd,argv,argc,search_flags,result);
if (ret >= 0)
return ret;
/* If the specs returned with an error (probably an INVALID or INCOMPLETE spec),
* fallback to the callback method. */
}
/* Resort to getkeys callback methods. */
if (has_module_getkeys)
return moduleGetCommandKeysViaAPI(cmd,argv,argc,result);
} else {
if (!(getAllKeySpecsFlags(cmd, 0) & CMD_KEY_VARIABLE_FLAGS)) {
int ret = getKeysUsingKeySpecs(cmd,argv,argc,search_flags,result);
if (ret >= 0)
return ret;
}
if (!(cmd->flags & CMD_MODULE) && cmd->getkeys_proc)
return cmd->getkeys_proc(cmd,argv,argc,result);
return 0;
}
/* We use native getkeys as a last resort, since not all these native getkeys provide
* flags properly (only the ones that correspond to INVALID, INCOMPLETE or VARIABLE_FLAGS do.*/
if (has_native_getkeys)
return cmd->getkeys_proc(cmd,argv,argc,result);
return 0;
}
/* This function returns a sanity check if the command may have keys. */
int doesCommandHaveKeys(struct redisCommand *cmd) {
return (!(cmd->flags & CMD_MODULE) && cmd->getkeys_proc) || /* has getkeys_proc (non modules) */
(cmd->flags & CMD_MODULE_GETKEYS) || /* module with GETKEYS */
(getAllKeySpecsFlags(cmd, 1) & CMD_KEY_CHANNEL); /* has at least one key-spec not marked as CHANNEL */
(getAllKeySpecsFlags(cmd, 1) & CMD_KEY_NOT_KEY); /* has at least one key-spec not marked as NOT_KEY */
}
/* A simplified channel spec table that contains all of the redis commands
* and which channels they have and how they are accessed. */
typedef struct ChannelSpecs {
redisCommandProc *proc; /* Command procedure to match against */
uint64_t flags; /* CMD_CHANNEL_* flags for this command */
int start; /* The initial position of the first channel */
int count; /* The number of channels, or -1 if all remaining
* arguments are channels. */
} ChannelSpecs;
ChannelSpecs commands_with_channels[] = {
{subscribeCommand, CMD_CHANNEL_SUBSCRIBE, 1, -1},
{ssubscribeCommand, CMD_CHANNEL_SUBSCRIBE, 1, -1},
{unsubscribeCommand, CMD_CHANNEL_UNSUBSCRIBE, 1, -1},
{sunsubscribeCommand, CMD_CHANNEL_UNSUBSCRIBE, 1, -1},
{psubscribeCommand, CMD_CHANNEL_PATTERN | CMD_CHANNEL_SUBSCRIBE, 1, -1},
{punsubscribeCommand, CMD_CHANNEL_PATTERN | CMD_CHANNEL_UNSUBSCRIBE, 1, -1},
{publishCommand, CMD_CHANNEL_PUBLISH, 1, 1},
{spublishCommand, CMD_CHANNEL_PUBLISH, 1, 1},
{NULL,0} /* Terminator. */
};
/* Returns 1 if the command may access any channels matched by the flags
* argument. */
int doesCommandHaveChannelsWithFlags(struct redisCommand *cmd, int flags) {
/* If a module declares get channels, we are just going to assume
* has channels. This API is allowed to return false positives. */
if (cmd->flags & CMD_MODULE_GETCHANNELS) {
return 1;
}
for (ChannelSpecs *spec = commands_with_channels; spec->proc != NULL; spec += 1) {
if (cmd->proc == spec->proc) {
return !!(spec->flags & flags);
}
}
return 0;
}
/* Return all the arguments that are channels in the command passed via argc / argv.
* This function behaves similar to getKeysFromCommandWithSpecs, but with channels
* instead of keys.
*
* The command returns the positions of all the channel arguments inside the array,
* so the actual return value is a heap allocated array of integers. The
* length of the array is returned by reference into *numkeys.
*
* Along with the position, this command also returns the flags that are
* associated with how Redis will access the channel.
*
* 'cmd' must be point to the corresponding entry into the redisCommand
* table, according to the command name in argv[0]. */
int getChannelsFromCommand(struct redisCommand *cmd, robj **argv, int argc, getKeysResult *result) {
keyReference *keys;
/* If a module declares get channels, use that. */
if (cmd->flags & CMD_MODULE_GETCHANNELS) {
return moduleGetCommandChannelsViaAPI(cmd, argv, argc, result);
}
/* Otherwise check the channel spec table */
for (ChannelSpecs *spec = commands_with_channels; spec != NULL; spec += 1) {
if (cmd->proc == spec->proc) {
int start = spec->start;
int stop = (spec->count == -1) ? argc : start + spec->count;
if (stop > argc) stop = argc;
int count = 0;
keys = getKeysPrepareResult(result, stop - start);
for (int i = start; i < stop; i++ ) {
keys[count].pos = i;
keys[count++].flags = spec->flags;
}
result->numkeys = count;
return count;
}
}
return 0;
}
/* The base case is to use the keys position as given in the command table
......
......@@ -482,6 +482,10 @@ void debugCommand(client *c) {
" Show low level client eviction pools info (maxmemory-clients).",
"PAUSE-CRON <0|1>",
" Stop periodic cron job processing.",
"REPLYBUFFER-PEAK-RESET-TIME <NEVER||RESET|time>",
" Sets the time (in milliseconds) to wait between client reply buffer peak resets.",
" In case NEVER is provided the last observed peak will never be reset",
" In case RESET is provided the peak reset time will be restored to the default value",
NULL
};
addReplyHelp(c, help);
......@@ -825,7 +829,7 @@ NULL
int memerr;
unsigned long long sz = memtoull((const char *)c->argv[2]->ptr, &memerr);
if (memerr || !quicklistisSetPackedThreshold(sz)) {
addReplyError(c, "argument must be a memory value bigger then 1 and smaller than 4gb");
addReplyError(c, "argument must be a memory value bigger than 1 and smaller than 4gb");
} else {
addReply(c,shared.ok);
}
......@@ -921,12 +925,12 @@ NULL
addReplyStatus(c,"Apparently Redis did not crash: test passed");
} else if (!strcasecmp(c->argv[1]->ptr,"set-disable-deny-scripts") && c->argc == 3)
{
server.script_disable_deny_script = atoi(c->argv[2]->ptr);;
server.script_disable_deny_script = atoi(c->argv[2]->ptr);
addReply(c,shared.ok);
} else if (!strcasecmp(c->argv[1]->ptr,"config-rewrite-force-all") && c->argc == 2)
{
if (rewriteConfig(server.configfile, 1) == -1)
addReplyError(c, "CONFIG-REWRITE-FORCE-ALL failed");
addReplyErrorFormat(c, "CONFIG-REWRITE-FORCE-ALL failed: %s", strerror(errno));
else
addReply(c, shared.ok);
} else if(!strcasecmp(c->argv[1]->ptr,"client-eviction") && c->argc == 2) {
......@@ -958,6 +962,16 @@ NULL
{
server.pause_cron = atoi(c->argv[2]->ptr);
addReply(c,shared.ok);
} else if (!strcasecmp(c->argv[1]->ptr,"replybuffer-peak-reset-time") && c->argc == 3 ) {
if (!strcasecmp(c->argv[2]->ptr, "never")) {
server.reply_buffer_peak_reset_time = -1;
} else if(!strcasecmp(c->argv[2]->ptr, "reset")) {
server.reply_buffer_peak_reset_time = REPLY_BUFFER_DEFAULT_PEAK_RESET_TIME;
} else {
if (getLongFromObjectOrReply(c, c->argv[2], &server.reply_buffer_peak_reset_time, NULL) != C_OK)
return;
}
addReply(c, shared.ok);
} else {
addReplySubcommandSyntaxError(c);
return;
......@@ -1681,13 +1695,19 @@ void logStackTrace(void *eip, int uplevel) {
void logServerInfo(void) {
sds infostring, clients;
serverLogRaw(LL_WARNING|LL_RAW, "\n------ INFO OUTPUT ------\n");
infostring = genRedisInfoString("all");
int all = 0, everything = 0;
robj *argv[1];
argv[0] = createStringObject("all", strlen("all"));
dict *section_dict = genInfoSectionDict(argv, 1, NULL, &all, &everything);
infostring = genRedisInfoString(section_dict, all, everything);
serverLogRaw(LL_WARNING|LL_RAW, infostring);
serverLogRaw(LL_WARNING|LL_RAW, "\n------ CLIENT LIST OUTPUT ------\n");
clients = getAllClientsInfoString(-1);
serverLogRaw(LL_WARNING|LL_RAW, clients);
sdsfree(infostring);
sdsfree(clients);
releaseInfoSectionDict(section_dict);
decrRefCount(argv[0]);
}
/* Log certain config values, which can be used for debuggin */
......@@ -1723,10 +1743,10 @@ void logCurrentClient(void) {
sdsfree(client);
for (j = 0; j < cc->argc; j++) {
robj *decoded;
decoded = getDecodedObject(cc->argv[j]);
serverLog(LL_WARNING|LL_RAW,"argv[%d]: '%s'\n", j,
(char*)decoded->ptr);
sds repr = sdscatrepr(sdsempty(),decoded->ptr, min(sdslen(decoded->ptr), 128));
serverLog(LL_WARNING|LL_RAW,"argv[%d]: '%s'\n", j, (char*)repr);
sdsfree(repr);
decrRefCount(decoded);
}
/* Check if the first argument, usually a key, is found inside the
......@@ -1764,7 +1784,10 @@ int memtest_test_linux_anonymous_maps(void) {
if (!fd) return 0;
fp = fopen("/proc/self/maps","r");
if (!fp) return 0;
if (!fp) {
closeDirectLogFiledes(fd);
return 0;
}
while(fgets(line,sizeof(line),fp) != NULL) {
char *start, *end, *p = line;
......
......@@ -30,6 +30,9 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef _REDIS_DEBUGMACRO_H_
#define _REDIS_DEBUGMACRO_H_
#include <stdio.h>
#define D(...) \
do { \
......@@ -39,3 +42,5 @@
fprintf(fp,"\n"); \
fclose(fp); \
} while (0)
#endif /* _REDIS_DEBUGMACRO_H_ */
......@@ -128,6 +128,27 @@ robj *activeDefragStringOb(robj* ob, long *defragged) {
return ret;
}
/* Defrag helper for lua scripts
*
* returns NULL in case the allocation wasn't moved.
* when it returns a non-null value, the old pointer was already released
* and should NOT be accessed. */
luaScript *activeDefragLuaScript(luaScript *script, long *defragged) {
luaScript *ret = NULL;
/* try to defrag script struct */
if ((ret = activeDefragAlloc(script))) {
script = ret;
(*defragged)++;
}
/* try to defrag actual script object */
robj *ob = activeDefragStringOb(script->body, defragged);
if (ob) script->body = ob;
return ret;
}
/* Defrag helper for dictEntries to be used during dict iteration (called on
* each step). Returns a stat of how many pointers were moved. */
long dictIterDefragEntry(dictIterator *iter) {
......@@ -256,6 +277,7 @@ long activeDefragZsetEntry(zset *zs, dictEntry *de) {
#define DEFRAG_SDS_DICT_VAL_IS_SDS 1
#define DEFRAG_SDS_DICT_VAL_IS_STROB 2
#define DEFRAG_SDS_DICT_VAL_VOID_PTR 3
#define DEFRAG_SDS_DICT_VAL_LUA_SCRIPT 4
/* Defrag a dict with sds key and optional value (either ptr, sds or robj string) */
long activeDefragSdsDict(dict* d, int val_type) {
......@@ -280,6 +302,10 @@ long activeDefragSdsDict(dict* d, int val_type) {
void *newptr, *ptr = dictGetVal(de);
if ((newptr = activeDefragAlloc(ptr)))
de->v.val = newptr, defragged++;
} else if (val_type == DEFRAG_SDS_DICT_VAL_LUA_SCRIPT) {
void *newptr, *ptr = dictGetVal(de);
if ((newptr = activeDefragLuaScript(ptr, &defragged)))
de->v.val = newptr;
}
defragged += dictIterDefragEntry(di);
}
......@@ -939,7 +965,7 @@ long defragOtherGlobals() {
/* there are many more pointers to defrag (e.g. client argv, output / aof buffers, etc.
* but we assume most of these are short lived, we only need to defrag allocations
* that remain static for a long time */
defragged += activeDefragSdsDict(evalScriptsDict(), DEFRAG_SDS_DICT_VAL_IS_STROB);
defragged += activeDefragSdsDict(evalScriptsDict(), DEFRAG_SDS_DICT_VAL_LUA_SCRIPT);
defragged += moduleDefragGlobals();
return defragged;
}
......@@ -1130,7 +1156,7 @@ void activeDefragCycle(void) {
/* Move on to next database, and stop if we reached the last one. */
if (++current_db >= server.dbnum) {
/* defrag other items not part of the db / keys */
defragOtherGlobals();
server.stat_active_defrag_hits += defragOtherGlobals();
long long now = ustime();
size_t frag_bytes;
......
......@@ -47,11 +47,6 @@ void ldbEnable(client *c);
void evalGenericCommandWithDebugging(client *c, int evalsha);
sds ldbCatStackValue(sds s, lua_State *lua, int idx);
typedef struct luaScript {
uint64_t flags;
robj *body;
} luaScript;
static void dictLuaScriptDestructor(dict *d, void *val) {
UNUSED(d);
if (val == NULL) return; /* Lazy freeing will set value to NULL. */
......@@ -63,7 +58,7 @@ static uint64_t dictStrCaseHash(const void *key) {
return dictGenCaseHashFunction((unsigned char*)key, strlen((char*)key));
}
/* server.lua_scripts sha (as sds string) -> scripts (as robj) cache. */
/* server.lua_scripts sha (as sds string) -> scripts (as luaScript) cache. */
dictType shaScriptObjectDictType = {
dictStrCaseHash, /* hash function */
NULL, /* key dup */
......@@ -246,11 +241,14 @@ void scriptingInit(int setup) {
" if i and i.what == 'C' then\n"
" i = dbg.getinfo(3,'nSl')\n"
" end\n"
" if type(err) ~= 'table' then\n"
" err = {err='ERR' .. tostring(err)}"
" end"
" if i then\n"
" return i.source .. ':' .. i.currentline .. ': ' .. err\n"
" else\n"
" return err\n"
" end\n"
" err['source'] = i.source\n"
" err['line'] = i.currentline\n"
" end"
" return err\n"
"end\n";
luaL_loadbuffer(lua,errh_func,strlen(errh_func),"@err_handler_def");
lua_pcall(lua,0,0,0);
......@@ -392,7 +390,7 @@ sds luaCreateFunction(client *c, robj *body) {
if (luaL_loadbuffer(lctx.lua,funcdef,sdslen(funcdef),"@user_script")) {
if (c != NULL) {
addReplyErrorFormat(c,
"Error compiling script (new function): %s\n",
"Error compiling script (new function): %s",
lua_tostring(lctx.lua,-1));
}
lua_pop(lctx.lua,1);
......@@ -403,7 +401,7 @@ sds luaCreateFunction(client *c, robj *body) {
if (lua_pcall(lctx.lua,0,0,0)) {
if (c != NULL) {
addReplyErrorFormat(c,"Error running script (new function): %s\n",
addReplyErrorFormat(c,"Error running script (new function): %s",
lua_tostring(lctx.lua,-1));
}
lua_pop(lctx.lua,1);
......@@ -1479,8 +1477,8 @@ int ldbRepl(lua_State *lua) {
while((argv = ldbReplParseCommand(&argc, &err)) == NULL) {
char buf[1024];
if (err) {
lua_pushstring(lua, err);
lua_error(lua);
luaPushError(lua, err);
luaError(lua);
}
int nread = connRead(ldb.conn,buf,sizeof(buf));
if (nread <= 0) {
......@@ -1497,8 +1495,8 @@ int ldbRepl(lua_State *lua) {
if (sdslen(ldb.cbuf) > 1<<20) {
sdsfree(ldb.cbuf);
ldb.cbuf = sdsempty();
lua_pushstring(lua, "max client buffer reached");
lua_error(lua);
luaPushError(lua, "max client buffer reached");
luaError(lua);
}
}
......@@ -1558,8 +1556,8 @@ ldbLog(sdsnew(" next line of code."));
ldbEval(lua,argv,argc);
ldbSendLogs();
} else if (!strcasecmp(argv[0],"a") || !strcasecmp(argv[0],"abort")) {
lua_pushstring(lua, "script aborted for user request");
lua_error(lua);
luaPushError(lua, "script aborted for user request");
luaError(lua);
} else if (argc > 1 &&
(!strcasecmp(argv[0],"r") || !strcasecmp(argv[0],"redis"))) {
ldbRedis(lua,argv,argc);
......@@ -1640,8 +1638,8 @@ void luaLdbLineHook(lua_State *lua, lua_Debug *ar) {
/* If the client closed the connection and we have a timeout
* connection, let's kill the script otherwise the process
* will remain blocked indefinitely. */
lua_pushstring(lua, "timeout during Lua debugging with client closing connection");
lua_error(lua);
luaPushError(lua, "timeout during Lua debugging with client closing connection");
luaError(lua);
}
rctx->start_time = getMonotonicUs();
rctx->snapshot_time = mstime();
......
......@@ -86,8 +86,8 @@ static void luaEngineLoadHook(lua_State *lua, lua_Debug *ar) {
if (duration > LOAD_TIMEOUT_MS) {
lua_sethook(lua, luaEngineLoadHook, LUA_MASKLINE, 0);
lua_pushstring(lua,"FUNCTION LOAD timeout");
lua_error(lua);
luaPushError(lua,"FUNCTION LOAD timeout");
luaError(lua);
}
}
......@@ -151,10 +151,13 @@ static int luaEngineCreate(void *engine_ctx, functionLibInfo *li, sds blob, sds
lua_sethook(lua,luaEngineLoadHook,LUA_MASKCOUNT,100000);
/* Run the compiled code to allow it to register functions */
if (lua_pcall(lua,0,0,0)) {
*err = sdscatprintf(sdsempty(), "Error registering functions: %s", lua_tostring(lua, -1));
errorInfo err_info = {0};
luaExtractErrorInformation(lua, &err_info);
*err = sdscatprintf(sdsempty(), "Error registering functions: %s", err_info.msg);
lua_pop(lua, 2); /* pops the error and globals table */
lua_sethook(lua,NULL,0,0); /* Disable hook */
luaSaveOnRegistry(lua, REGISTRY_LOAD_CTX_NAME, NULL);
luaErrorInformationDiscard(&err_info);
return C_ERR;
}
lua_sethook(lua,NULL,0,0); /* Disable hook */
......@@ -429,11 +432,11 @@ static int luaRegisterFunction(lua_State *lua) {
loadCtx *load_ctx = luaGetFromRegistry(lua, REGISTRY_LOAD_CTX_NAME);
if (!load_ctx) {
luaPushError(lua, "redis.register_function can only be called on FUNCTION LOAD command");
return luaRaiseError(lua);
return luaError(lua);
}
if (luaRegisterFunctionReadArgs(lua, &register_f_args) != C_OK) {
return luaRaiseError(lua);
return luaError(lua);
}
sds err = NULL;
......@@ -441,7 +444,7 @@ static int luaRegisterFunction(lua_State *lua) {
luaRegisterFunctionArgsDispose(lua, &register_f_args);
luaPushError(lua, err);
sdsfree(err);
return luaRaiseError(lua);
return luaError(lua);
}
return 0;
......@@ -475,11 +478,14 @@ int luaEngineInitEngine() {
" if i and i.what == 'C' then\n"
" i = dbg.getinfo(3,'nSl')\n"
" end\n"
" if type(err) ~= 'table' then\n"
" err = {err='ERR' .. tostring(err)}"
" end"
" if i then\n"
" return i.source .. ':' .. i.currentline .. ': ' .. err\n"
" else\n"
" return err\n"
" end\n"
" err['source'] = i.source\n"
" err['line'] = i.currentline\n"
" end"
" return err\n"
"end\n"
"return error_handler";
luaL_loadbuffer(lua_engine_ctx->lua, errh_func, strlen(errh_func), "@err_handler_def");
......
......@@ -808,7 +808,7 @@ void functionFlushCommand(client *c) {
void functionHelpCommand(client *c) {
const char *help[] = {
"LOAD <ENGINE NAME> <LIBRARY NAME> [REPLACE] [DESC <LIBRARY DESCRIPTION>] <LIBRARY CODE>",
"LOAD <ENGINE NAME> <LIBRARY NAME> [REPLACE] [DESCRIPTION <LIBRARY DESCRIPTION>] <LIBRARY CODE>",
" Create a new library with the given library name and code.",
"DELETE <LIBRARY NAME>",
" Delete the given library.",
......
......@@ -34,7 +34,6 @@
#include <stddef.h>
#include <stdint.h>
#include <stdint.h>
#if defined(__cplusplus)
extern "C" {
......
......@@ -91,8 +91,8 @@ uint8_t geohashEstimateStepsByRadius(double range_meters, double lat) {
* \-----------------/ -------- \-----------------/
* \ / / \ \ /
* \ (long,lat) / / (long,lat) \ \ (long,lat) /
* \ / / \ / \
* --------- /----------------\ /--------------\
* \ / / \ / \
* --------- /----------------\ /---------------\
* Northern Hemisphere Southern Hemisphere Around the equator
*/
int geohashBoundingBox(GeoShape *shape, double *bounds) {
......@@ -164,14 +164,14 @@ GeoHashRadius geohashCalculateAreasByShapeWGS84(GeoShape *shape) {
geohashDecode(long_range, lat_range, neighbors.east, &east);
geohashDecode(long_range, lat_range, neighbors.west, &west);
if (geohashGetDistance(longitude,latitude,longitude,north.latitude.max)
< radius_meters) decrease_step = 1;
if (geohashGetDistance(longitude,latitude,longitude,south.latitude.min)
< radius_meters) decrease_step = 1;
if (geohashGetDistance(longitude,latitude,east.longitude.max,latitude)
< radius_meters) decrease_step = 1;
if (geohashGetDistance(longitude,latitude,west.longitude.min,latitude)
< radius_meters) decrease_step = 1;
if (north.latitude.max < max_lat)
decrease_step = 1;
if (south.latitude.min > min_lat)
decrease_step = 1;
if (east.longitude.max < max_lon)
decrease_step = 1;
if (west.longitude.min > min_lon)
decrease_step = 1;
}
if (steps > 1 && decrease_step) {
......
/* Automatically generated by ./utils/generate-command-help.rb, do not edit. */
/* Automatically generated by utils/generate-command-help.rb, do not edit. */
#ifndef __REDIS_HELP_H
#define __REDIS_HELP_H
......@@ -429,6 +429,11 @@ struct commandHelp {
"Extract keys given a full Redis command",
9,
"2.8.13" },
{ "COMMAND GETKEYSANDFLAGS",
"",
"Extract keys given a full Redis command",
9,
"7.0.0" },
{ "COMMAND HELP",
"",
"Show helpful text about the different subcommands",
......@@ -571,12 +576,12 @@ struct commandHelp {
"6.2.0" },
{ "FCALL",
"function numkeys key [key ...] arg [arg ...]",
"PATCH__TBD__38__",
"Invoke a function",
10,
"7.0.0" },
{ "FCALL_RO",
"function numkeys key [key ...] arg [arg ...]",
"PATCH__TBD__7__",
"Invoke a read-only function",
10,
"7.0.0" },
{ "FLUSHALL",
......@@ -595,7 +600,7 @@ struct commandHelp {
10,
"7.0.0" },
{ "FUNCTION DELETE",
"function-name",
"library-name",
"Delete a function by name",
10,
"7.0.0" },
......@@ -625,7 +630,7 @@ struct commandHelp {
10,
"7.0.0" },
{ "FUNCTION LOAD",
"engine-name library-name [REPLACE] [DESC library-description] function-code",
"engine-name library-name [REPLACE] [DESCRIPTION library-description] function-code",
"Create a function with the given arguments (name, code, description)",
10,
"7.0.0" },
......@@ -820,7 +825,7 @@ struct commandHelp {
1,
"2.6.0" },
{ "INFO",
"[section]",
"[section [section ...]]",
"Get information and statistics about the server",
9,
"1.0.0" },
......@@ -1180,7 +1185,7 @@ struct commandHelp {
6,
"7.0.0" },
{ "PUBSUB SHARDNUMSUB",
"",
"[channel [channel ...]]",
"Get the count of subscribers for shard channels",
6,
"7.0.0" },
......@@ -1391,7 +1396,7 @@ struct commandHelp {
"1.0.0" },
{ "SLAVEOF",
"host port",
"Make the server a replica of another instance, or promote it as master. Deprecated starting with Redis 5. Use REPLICAOF instead.",
"Make the server a replica of another instance, or promote it as master.",
9,
"1.0.0" },
{ "SLOWLOG",
......@@ -1590,7 +1595,7 @@ struct commandHelp {
14,
"5.0.0" },
{ "XGROUP CREATE",
"key groupname id|$ [MKSTREAM]",
"key groupname id|$ [MKSTREAM] [ENTRIESREAD entries_read]",
"Create a consumer group.",
14,
"5.0.0" },
......@@ -1615,7 +1620,7 @@ struct commandHelp {
14,
"5.0.0" },
{ "XGROUP SETID",
"key groupname id|$",
"key groupname id|$ [ENTRIESREAD entries_read]",
"Set a consumer group to an arbitrary last delivered ID value.",
14,
"5.0.0" },
......@@ -1675,7 +1680,7 @@ struct commandHelp {
14,
"5.0.0" },
{ "XSETID",
"key last-id",
"key last-id [ENTRIESADDED entries_added] [MAXDELETEDID max_deleted_entry_id]",
"An internal command for replicating stream values",
14,
"5.0.0" },
......
......@@ -70,7 +70,7 @@
typedef struct RedisModuleInfoCtx {
struct RedisModule *module;
const char *requested_section;
dict *requested_sections;
sds info; /* info string we collected so far */
int sections; /* number of sections we collected so far */
int in_section; /* indication if we're in an active section or not */
......@@ -154,7 +154,8 @@ struct RedisModuleCtx {
gets called for clients blocked
on keys. */
/* Used if there is the REDISMODULE_CTX_KEYS_POS_REQUEST flag set. */
/* Used if there is the REDISMODULE_CTX_KEYS_POS_REQUEST or
* REDISMODULE_CTX_CHANNEL_POS_REQUEST flag set. */
getKeysResult *keys_result;
struct RedisModulePoolAllocBlock *pa_head;
......@@ -173,6 +174,7 @@ typedef struct RedisModuleCtx RedisModuleCtx;
when the context is destroyed */
#define REDISMODULE_CTX_NEW_CLIENT (1<<7) /* Free client object when the
context is destroyed */
#define REDISMODULE_CTX_CHANNELS_POS_REQUEST (1<<8)
/* This represents a Redis key opened with RM_OpenKey(). */
struct RedisModuleKey {
......@@ -390,6 +392,7 @@ typedef struct RedisModuleKeyOptCtx {
In most cases, only 'from_dbid' is valid, but in callbacks such
as `copy2`, 'from_dbid' and 'to_dbid' are both valid. */
} RedisModuleKeyOptCtx;
/* --------------------------------------------------------------------------
* Prototypes
* -------------------------------------------------------------------------- */
......@@ -404,6 +407,16 @@ static void moduleInitKeyTypeSpecific(RedisModuleKey *key);
void RM_FreeDict(RedisModuleCtx *ctx, RedisModuleDict *d);
void RM_FreeServerInfo(RedisModuleCtx *ctx, RedisModuleServerInfoData *data);
/* Helpers for RM_SetCommandInfo. */
static int moduleValidateCommandInfo(const RedisModuleCommandInfo *info);
static int64_t moduleConvertKeySpecsFlags(int64_t flags, int from_api);
static int moduleValidateCommandArgs(RedisModuleCommandArg *args,
const RedisModuleCommandInfoVersion *version);
static struct redisCommandArg *moduleCopyCommandArgs(RedisModuleCommandArg *args,
const RedisModuleCommandInfoVersion *version);
static redisCommandArgType moduleConvertArgType(RedisModuleCommandArgType type, int *error);
static int moduleConvertArgFlags(int flags);
/* --------------------------------------------------------------------------
* ## Heap allocation raw functions
*
......@@ -770,6 +783,25 @@ int moduleGetCommandKeysViaAPI(struct redisCommand *cmd, robj **argv, int argc,
return result->numkeys;
}
/* This function returns the list of channels, with the same interface as
* moduleGetCommandKeysViaAPI, for modules that declare "getchannels-api"
* during registration. Unlike keys, this is the only way to declare channels. */
int moduleGetCommandChannelsViaAPI(struct redisCommand *cmd, robj **argv, int argc, getKeysResult *result) {
RedisModuleCommand *cp = (void*)(unsigned long)cmd->getkeys_proc;
RedisModuleCtx ctx;
moduleCreateContext(&ctx, cp->module, REDISMODULE_CTX_CHANNELS_POS_REQUEST);
/* Initialize getKeysResult */
getKeysPrepareResult(result, MAX_KEYS_BUFFER);
ctx.keys_result = result;
cp->func(&ctx,(void**)argv,argc);
/* We currently always use the array allocated by RM_RM_ChannelAtPosWithFlags() and don't try
* to optimize for the pre-allocated buffer. */
moduleFreeContext(&ctx);
return result->numkeys;
}
/* --------------------------------------------------------------------------
* ## Commands API
*
......@@ -789,17 +821,23 @@ int RM_IsKeysPositionRequest(RedisModuleCtx *ctx) {
* keys, since it was flagged as "getkeys-api" during the registration,
* the command implementation checks for this special call using the
* RedisModule_IsKeysPositionRequest() API and uses this function in
* order to report keys, like in the following example:
* order to report keys.
*
* The supported flags are the ones used by RM_SetCommandInfo, see REDISMODULE_CMD_KEY_*.
*
*
* The following is an example of how it could be used:
*
* if (RedisModule_IsKeysPositionRequest(ctx)) {
* RedisModule_KeyAtPos(ctx,1);
* RedisModule_KeyAtPos(ctx,2);
* RedisModule_KeyAtPosWithFlags(ctx, 2, REDISMODULE_CMD_KEY_RO | REDISMODULE_CMD_KEY_ACCESS);
* RedisModule_KeyAtPosWithFlags(ctx, 1, REDISMODULE_CMD_KEY_RW | REDISMODULE_CMD_KEY_UPDATE | REDISMODULE_CMD_KEY_ACCESS);
* }
*
* Note: in the example below the get keys API would not be needed since
* keys are at fixed positions. This interface is only used for commands
* with a more complex structure. */
void RM_KeyAtPos(RedisModuleCtx *ctx, int pos) {
* Note: in the example above the get keys API could have been handled by key-specs (preferred).
* Implementing the getkeys-api is required only when is it not possible to declare key-specs that cover all keys.
*
*/
void RM_KeyAtPosWithFlags(RedisModuleCtx *ctx, int pos, int flags) {
if (!(ctx->flags & REDISMODULE_CTX_KEYS_POS_REQUEST) || !ctx->keys_result) return;
if (pos <= 0) return;
......@@ -811,7 +849,74 @@ void RM_KeyAtPos(RedisModuleCtx *ctx, int pos) {
getKeysPrepareResult(res, newsize);
}
res->keys[res->numkeys++].pos = pos;
res->keys[res->numkeys].pos = pos;
res->keys[res->numkeys].flags = moduleConvertKeySpecsFlags(flags, 1);
res->numkeys++;
}
/* This API existed before RM_KeyAtPosWithFlags was added, now deprecated and
* can be used for compatibility with older versions, before key-specs and flags
* were introduced. */
void RM_KeyAtPos(RedisModuleCtx *ctx, int pos) {
/* Default flags require full access */
int flags = moduleConvertKeySpecsFlags(CMD_KEY_FULL_ACCESS, 0);
RM_KeyAtPosWithFlags(ctx, pos, flags);
}
/* Return non-zero if a module command, that was declared with the
* flag "getchannels-api", is called in a special way to get the channel positions
* and not to get executed. Otherwise zero is returned. */
int RM_IsChannelsPositionRequest(RedisModuleCtx *ctx) {
return (ctx->flags & REDISMODULE_CTX_CHANNELS_POS_REQUEST) != 0;
}
/* When a module command is called in order to obtain the position of
* channels, since it was flagged as "getchannels-api" during the
* registration, the command implementation checks for this special call
* using the RedisModule_IsChannelsPositionRequest() API and uses this
* function in order to report the channels.
*
* The supported flags are:
* * REDISMODULE_CMD_CHANNEL_SUBSCRIBE: This command will subscribe to the channel.
* * REDISMODULE_CMD_CHANNEL_UNSUBSCRIBE: This command will unsubscribe from this channel.
* * REDISMODULE_CMD_CHANNEL_PUBLISH: This command will publish to this channel.
* * REDISMODULE_CMD_CHANNEL_PATTERN: Instead of acting on a specific channel, will act on any
* channel specified by the pattern. This is the same access
* used by the PSUBSCRIBE and PUNSUBSCRIBE commands available
* in Redis. Not intended to be used with PUBLISH permissions.
*
* The following is an example of how it could be used:
*
* if (RedisModule_IsChannelsPositionRequest(ctx)) {
* RedisModule_ChannelAtPosWithFlags(ctx, 1, REDISMODULE_CMD_CHANNEL_SUBSCRIBE | REDISMODULE_CMD_CHANNEL_PATTERN);
* RedisModule_ChannelAtPosWithFlags(ctx, 1, REDISMODULE_CMD_CHANNEL_PUBLISH);
* }
*
* Note: One usage of declaring channels is for evaluating ACL permissions. In this context,
* unsubscribing is always allowed, so commands will only be checked against subscribe and
* publish permissions. This is preferred over using RM_ACLCheckChannelPermissions, since
* it allows the ACLs to be checked before the command is executed. */
void RM_ChannelAtPosWithFlags(RedisModuleCtx *ctx, int pos, int flags) {
if (!(ctx->flags & REDISMODULE_CTX_CHANNELS_POS_REQUEST) || !ctx->keys_result) return;
if (pos <= 0) return;
getKeysResult *res = ctx->keys_result;
/* Check overflow */
if (res->numkeys == res->size) {
int newsize = res->size + (res->size > 8192 ? 8192 : res->size);
getKeysPrepareResult(res, newsize);
}
int new_flags = 0;
if (flags & REDISMODULE_CMD_CHANNEL_SUBSCRIBE) new_flags |= CMD_CHANNEL_SUBSCRIBE;
if (flags & REDISMODULE_CMD_CHANNEL_UNSUBSCRIBE) new_flags |= CMD_CHANNEL_UNSUBSCRIBE;
if (flags & REDISMODULE_CMD_CHANNEL_PUBLISH) new_flags |= CMD_CHANNEL_PUBLISH;
if (flags & REDISMODULE_CMD_CHANNEL_PATTERN) new_flags |= CMD_CHANNEL_PATTERN;
res->keys[res->numkeys].pos = pos;
res->keys[res->numkeys].flags = new_flags;
res->numkeys++;
}
/* Helper for RM_CreateCommand(). Turns a string representing command
......@@ -840,6 +945,7 @@ int64_t commandFlagsFromString(char *s) {
else if (!strcasecmp(t,"no-auth")) flags |= CMD_NO_AUTH;
else if (!strcasecmp(t,"may-replicate")) flags |= CMD_MAY_REPLICATE;
else if (!strcasecmp(t,"getkeys-api")) flags |= CMD_MODULE_GETKEYS;
else if (!strcasecmp(t,"getchannels-api")) flags |= CMD_MODULE_GETCHANNELS;
else if (!strcasecmp(t,"no-cluster")) flags |= CMD_MODULE_NO_CLUSTER;
else if (!strcasecmp(t,"no-mandatory-keys")) flags |= CMD_NO_MANDATORY_KEYS;
else if (!strcasecmp(t,"allow-busy")) flags |= CMD_ALLOW_BUSY;
......@@ -850,33 +956,6 @@ int64_t commandFlagsFromString(char *s) {
return flags;
}
/* Helper for RM_CreateCommand(). Turns a string representing keys spec
* flags into the keys spec flags used by the Redis core.
*
* It returns the set of flags, or -1 if unknown flags are found. */
int64_t commandKeySpecsFlagsFromString(const char *s) {
int count, j;
int64_t flags = 0;
sds *tokens = sdssplitlen(s,strlen(s)," ",1,&count);
for (j = 0; j < count; j++) {
char *t = tokens[j];
if (!strcasecmp(t,"RO")) flags |= CMD_KEY_RO;
else if (!strcasecmp(t,"RW")) flags |= CMD_KEY_RW;
else if (!strcasecmp(t,"OW")) flags |= CMD_KEY_OW;
else if (!strcasecmp(t,"RM")) flags |= CMD_KEY_RM;
else if (!strcasecmp(t,"access")) flags |= CMD_KEY_ACCESS;
else if (!strcasecmp(t,"insert")) flags |= CMD_KEY_INSERT;
else if (!strcasecmp(t,"update")) flags |= CMD_KEY_UPDATE;
else if (!strcasecmp(t,"delete")) flags |= CMD_KEY_DELETE;
else if (!strcasecmp(t,"channel")) flags |= CMD_KEY_CHANNEL;
else if (!strcasecmp(t,"incomplete")) flags |= CMD_KEY_INCOMPLETE;
else break;
}
sdsfreesplitres(tokens,count);
if (j != count) return -1; /* Some token not processed correctly. */
return flags;
}
RedisModuleCommand *moduleCreateCommandProxy(struct RedisModule *module, sds declared_name, sds fullname, RedisModuleCmdFunc cmdfunc, int64_t flags, int firstkey, int lastkey, int keystep);
/* Register a new command in the Redis server, that will be handled by
......@@ -946,6 +1025,8 @@ RedisModuleCommand *moduleCreateCommandProxy(struct RedisModule *module, sds dec
* * **"allow-busy"**: Permit the command while the server is blocked either by
* a script or by a slow module command, see
* RM_Yield.
* * **"getchannels-api"**: The command implements the interface to return
* the arguments that are channels.
*
* The last three parameters specify which arguments of the new command are
* Redis keys. See https://redis.io/commands/command for more information.
......@@ -965,9 +1046,7 @@ RedisModuleCommand *moduleCreateCommandProxy(struct RedisModule *module, sds dec
* NOTE: The scheme described above serves a limited purpose and can
* only be used to find keys that exist at constant indices.
* For non-trivial key arguments, you may pass 0,0,0 and use
* RedisModule_AddCommandKeySpec (see documentation).
*
*/
* RedisModule_SetCommandInfo to set key specs using a more advanced scheme. */
int RM_CreateCommand(RedisModuleCtx *ctx, const char *name, RedisModuleCmdFunc cmdfunc, const char *strflags, int firstkey, int lastkey, int keystep) {
int64_t flags = strflags ? commandFlagsFromString((char*)strflags) : 0;
if (flags == -1) return REDISMODULE_ERR;
......@@ -1020,7 +1099,7 @@ RedisModuleCommand *moduleCreateCommandProxy(struct RedisModule *module, sds dec
cp->rediscmd->key_specs = cp->rediscmd->key_specs_static;
if (firstkey != 0) {
cp->rediscmd->key_specs_num = 1;
cp->rediscmd->key_specs[0].flags = 0;
cp->rediscmd->key_specs[0].flags = CMD_KEY_FULL_ACCESS | CMD_KEY_VARIABLE_FLAGS;
cp->rediscmd->key_specs[0].begin_search_type = KSPEC_BS_INDEX;
cp->rediscmd->key_specs[0].bs.index.pos = firstkey;
cp->rediscmd->key_specs[0].find_keys_type = KSPEC_FK_RANGE;
......@@ -1121,216 +1200,735 @@ int RM_CreateSubcommand(RedisModuleCommand *parent, const char *name, RedisModul
return REDISMODULE_OK;
}
/* Return `struct RedisModule *` as `void *` to avoid exposing it outside of module.c. */
void *moduleGetHandleByName(char *modulename) {
return dictFetchValue(modules,modulename);
}
/* Accessors of array elements of structs where the element size is stored
* separately in the version struct. */
static RedisModuleCommandHistoryEntry *
moduleCmdHistoryEntryAt(const RedisModuleCommandInfoVersion *version,
RedisModuleCommandHistoryEntry *entries, int index) {
off_t offset = index * version->sizeof_historyentry;
return (RedisModuleCommandHistoryEntry *)((char *)(entries) + offset);
}
static RedisModuleCommandKeySpec *
moduleCmdKeySpecAt(const RedisModuleCommandInfoVersion *version,
RedisModuleCommandKeySpec *keyspecs, int index) {
off_t offset = index * version->sizeof_keyspec;
return (RedisModuleCommandKeySpec *)((char *)(keyspecs) + offset);
}
static RedisModuleCommandArg *
moduleCmdArgAt(const RedisModuleCommandInfoVersion *version,
const RedisModuleCommandArg *args, int index) {
off_t offset = index * version->sizeof_arg;
return (RedisModuleCommandArg *)((char *)(args) + offset);
}
/* Set additional command information.
*
* Affects the output of `COMMAND`, `COMMAND INFO` and `COMMAND DOCS`, Cluster,
* ACL and is used to filter commands with the wrong number of arguments before
* the call reaches the module code.
*
* This function can be called after creating a command using RM_CreateCommand
* and fetching the command pointer using RM_GetCommand. The information can
* only be set once for each command and has the following structure:
*
* typedef struct RedisModuleCommandInfo {
* const RedisModuleCommandInfoVersion *version;
* const char *summary;
* const char *complexity;
* const char *since;
* RedisModuleCommandHistoryEntry *history;
* const char *tips;
* int arity;
* RedisModuleCommandKeySpec *key_specs;
* RedisModuleCommandArg *args;
* } RedisModuleCommandInfo;
*
* All fields except `version` are optional. Explanation of the fields:
*
* - `version`: This field enables compatibility with different Redis versions.
* Always set this field to REDISMODULE_COMMAND_INFO_VERSION.
*
* - `summary`: A short description of the command (optional).
*
* - `complexity`: Complexity description (optional).
*
* - `since`: The version where the command was introduced (optional).
* Note: The version specified should be the module's, not Redis version.
*
* - `history`: An array of RedisModuleCommandHistoryEntry (optional), which is
* a struct with the following fields:
*
* const char *since;
* const char *changes;
*
* `since` is a version string and `changes` is a string describing the
* changes. The array is terminated by a zeroed entry, i.e. an entry with
* both strings set to NULL.
*
* - `tips`: A string of space-separated tips regarding this command, meant for
* clients and proxies. See https://redis.io/topics/command-tips.
*
* - `arity`: Number of arguments, including the command name itself. A positive
* number specifies an exact number of arguments and a negative number
* specifies a minimum number of arguments, so use -N to say >= N. Redis
* validates a call before passing it to a module, so this can replace an
* arity check inside the module command implementation. A value of 0 (or an
* omitted arity field) is equivalent to -2 if the command has sub commands
* and -1 otherwise.
*
* - `key_specs`: An array of RedisModuleCommandKeySpec, terminated by an
* element memset to zero. This is a scheme that tries to describe the
* positions of key arguments better than the old RM_CreateCommand arguments
* `firstkey`, `lastkey`, `keystep` and is needed if those three are not
* enough to describe the key positions. There are two steps to retrieve key
* positions: *begin search* (BS) in which index should find the first key and
* *find keys* (FK) which, relative to the output of BS, describes how can we
* will which arguments are keys. Additionally, there are key specific flags.
*
* Key-specs cause the triplet (firstkey, lastkey, keystep) given in
* RM_CreateCommand to be recomputed, but it is still useful to provide
* these three parameters in RM_CreateCommand, to better support old Redis
* versions where RM_SetCommandInfo is not available.
*
* Note that key-specs don't fully replace the "getkeys-api" (see
* RM_CreateCommand, RM_IsKeysPositionRequest and RM_KeyAtPosWithFlags) so
* it may be a good idea to supply both key-specs and implement the
* getkeys-api.
*
* A key-spec has the following structure:
*
* typedef struct RedisModuleCommandKeySpec {
* const char *notes;
* uint64_t flags;
* RedisModuleKeySpecBeginSearchType begin_search_type;
* union {
* struct {
* int pos;
* } index;
* struct {
* const char *keyword;
* int startfrom;
* } keyword;
* } bs;
* RedisModuleKeySpecFindKeysType find_keys_type;
* union {
* struct {
* int lastkey;
* int keystep;
* int limit;
* } range;
* struct {
* int keynumidx;
* int firstkey;
* int keystep;
* } keynum;
* } fk;
* } RedisModuleCommandKeySpec;
*
* Explanation of the fields of RedisModuleCommandKeySpec:
*
* * `notes`: Optional notes or clarifications about this key spec.
*
* * `flags`: A bitwise or of key-spec flags described below.
*
* * `begin_search_type`: This describes how the first key is discovered.
* There are two ways to determine the first key:
*
* * `REDISMODULE_KSPEC_BS_UNKNOWN`: There is no way to tell where the
* key args start.
* * `REDISMODULE_KSPEC_BS_INDEX`: Key args start at a constant index.
* * `REDISMODULE_KSPEC_BS_KEYWORD`: Key args start just after a
* specific keyword.
*
* * `bs`: This is a union in which the `index` or `keyword` branch is used
* depending on the value of the `begin_search_type` field.
*
* * `bs.index.pos`: The index from which we start the search for keys.
* (`REDISMODULE_KSPEC_BS_INDEX` only.)
*
* * `bs.keyword.keyword`: The keyword (string) that indicates the
* beginning of key arguments. (`REDISMODULE_KSPEC_BS_KEYWORD` only.)
*
* * `bs.keyword.startfrom`: An index in argv from which to start
* searching. Can be negative, which means start search from the end,
* in reverse. Example: -2 means to start in reverse from the
* penultimate argument. (`REDISMODULE_KSPEC_BS_KEYWORD` only.)
*
* * `find_keys_type`: After the "begin search", this describes which
* arguments are keys. The strategies are:
*
* * `REDISMODULE_KSPEC_BS_UNKNOWN`: There is no way to tell where the
* key args are located.
* * `REDISMODULE_KSPEC_FK_RANGE`: Keys end at a specific index (or
* relative to the last argument).
* * `REDISMODULE_KSPEC_FK_KEYNUM`: There's an argument that contains
* the number of key args somewhere before the keys themselves.
*
* `find_keys_type` and `fk` can be omitted if this keyspec describes
* exactly one key.
*
* * `fk`: This is a union in which the `range` or `keynum` branch is used
* depending on the value of the `find_keys_type` field.
*
* * `fk.range` (for `REDISMODULE_KSPEC_FK_RANGE`): A struct with the
* following fields:
*
* * `lastkey`: Index of the last key relative to the result of the
* begin search step. Can be negative, in which case it's not
* relative. -1 indicates the last argument, -2 one before the
* last and so on.
*
* * `keystep`: How many arguments should we skip after finding a
* key, in order to find the next one?
*
* * `limit`: If `lastkey` is -1, we use `limit` to stop the search
* by a factor. 0 and 1 mean no limit. 2 means 1/2 of the
* remaining args, 3 means 1/3, and so on.
*
* * `fk.keynum` (for `REDISMODULE_KSPEC_FK_KEYNUM`): A struct with the
* following fields:
*
* * `keynumidx`: Index of the argument containing the number of
* keys to come, relative to the result of the begin search step.
*
* * `firstkey`: Index of the fist key relative to the result of the
* begin search step. (Usually it's just after `keynumidx`, in
* which case it should be set to `keynumidx + 1`.)
*
* * `keystep`: How many argumentss should we skip after finding a
* key, in order to find the next one?
*
* Key-spec flags:
*
* The first four refer to what the command actually does with the *value or
* metadata of the key*, and not necessarily the user data or how it affects
* it. Each key-spec may must have exactly one of these. Any operation
* that's not distinctly deletion, overwrite or read-only would be marked as
* RW.
*
* * `REDISMODULE_CMD_KEY_RO`: Read-Only. Reads the value of the key, but
* doesn't necessarily return it.
*
* * `REDISMODULE_CMD_KEY_RW`: Read-Write. Modifies the data stored in the
* value of the key or its metadata.
*
* * `REDISMODULE_CMD_KEY_OW`: Overwrite. Overwrites the data stored in the
* value of the key.
*
* * `REDISMODULE_CMD_KEY_RM`: Deletes the key.
*
* The next four refer to *user data inside the value of the key*, not the
* metadata like LRU, type, cardinality. It refers to the logical operation
* on the user's data (actual input strings or TTL), being
* used/returned/copied/changed. It doesn't refer to modification or
* returning of metadata (like type, count, presence of data). ACCESS can be
* combined with one of the write operations INSERT, DELETE or UPDATE. Any
* write that's not an INSERT or a DELETE would be UPDATE.
*
* * `REDISMODULE_CMD_KEY_ACCESS`: Returns, copies or uses the user data
* from the value of the key.
*
* * `REDISMODULE_CMD_KEY_UPDATE`: Updates data to the value, new value may
* depend on the old value.
*
* * `REDISMODULE_CMD_KEY_INSERT`: Adds data to the value with no chance of
* modification or deletion of existing data.
*
* * `REDISMODULE_CMD_KEY_DELETE`: Explicitly deletes some content from the
* value of the key.
*
* Other flags:
*
* * `REDISMODULE_CMD_KEY_NOT_KEY`: The key is not actually a key, but
* should be routed in cluster mode as if it was a key.
*
* * `REDISMODULE_CMD_KEY_INCOMPLETE`: The keyspec might not point out all
* the keys it should cover.
*
* * `REDISMODULE_CMD_KEY_VARIABLE_FLAGS`: Some keys might have different
* flags depending on arguments.
*
* - `args`: An array of RedisModuleCommandArg, terminated by an element memset
* to zero. RedisModuleCommandArg is a structure with at the fields described
* below.
*
* typedef struct RedisModuleCommandArg {
* const char *name;
* RedisModuleCommandArgType type;
* int key_spec_index;
* const char *token;
* const char *summary;
* const char *since;
* int flags;
* struct RedisModuleCommandArg *subargs;
* } RedisModuleCommandArg;
*
* Explanation of the fields:
*
* * `name`: Name of the argument.
*
* * `type`: The type of the argument. See below for details. The types
* `REDISMODULE_ARG_TYPE_ONEOF` and `REDISMODULE_ARG_TYPE_BLOCK` require
* an argument to have sub-arguments, i.e. `subargs`.
*
* * `key_spec_index`: If the `type` is `REDISMODULE_ARG_TYPE_KEY` you must
* provide the index of the key-spec associated with this argument. See
* `key_specs` above. If the argument is not a key, you may specify -1.
*
* * `token`: The token preceding the argument (optional). Example: the
* argument `seconds` in `SET` has a token `EX`. If the argument consists
* of only a token (for example `NX` in `SET`) the type should be
* `REDISMODULE_ARG_TYPE_PURE_TOKEN` and `value` should be NULL.
*
* * `summary`: A short description of the argument (optional).
*
* * `since`: The first version which included this argument (optional).
*
* * `flags`: A bitwise or of the macros `REDISMODULE_CMD_ARG_*`. See below.
*
* * `value`: The display-value of the argument. This string is what should
* be displayed when creating the command syntax from the output of
* `COMMAND`. If `token` is not NULL, it should also be displayed.
*
* Explanation of `RedisModuleCommandArgType`:
*
* * `REDISMODULE_ARG_TYPE_STRING`: String argument.
* * `REDISMODULE_ARG_TYPE_INTEGER`: Integer argument.
* * `REDISMODULE_ARG_TYPE_DOUBLE`: Double-precision float argument.
* * `REDISMODULE_ARG_TYPE_KEY`: String argument representing a keyname.
* * `REDISMODULE_ARG_TYPE_PATTERN`: String, but regex pattern.
* * `REDISMODULE_ARG_TYPE_UNIX_TIME`: Integer, but Unix timestamp.
* * `REDISMODULE_ARG_TYPE_PURE_TOKEN`: Argument doesn't have a placeholder.
* It's just a token without a value. Example: the `KEEPTTL` option of the
* `SET` command.
* * `REDISMODULE_ARG_TYPE_ONEOF`: Used when the user can choose only one of
* a few sub-arguments. Requires `subargs`. Example: the `NX` and `XX`
* options of `SET`.
* * `REDISMODULE_ARG_TYPE_BLOCK`: Used when one wants to group together
* several sub-arguments, usually to apply something on all of them, like
* making the entire group "optional". Requires `subargs`. Example: the
* `LIMIT offset count` parameters in `ZRANGE`.
*
* Explanation of the command argument flags:
*
* * `REDISMODULE_CMD_ARG_OPTIONAL`: The argument is optional (like GET in
* the SET command).
* * `REDISMODULE_CMD_ARG_MULTIPLE`: The argument may repeat itself (like
* key in DEL).
* * `REDISMODULE_CMD_ARG_MULTIPLE_TOKEN`: The argument may repeat itself,
* and so does its token (like `GET pattern` in SORT).
*
* On success REDISMODULE_OK is returned. On error REDISMODULE_ERR is returned
* and `errno` is set to EINVAL if invalid info was provided or EEXIST if info
* has already been set. If the info is invalid, a warning is logged explaining
* which part of the info is invalid and why. */
int RM_SetCommandInfo(RedisModuleCommand *command, const RedisModuleCommandInfo *info) {
if (!moduleValidateCommandInfo(info)) {
errno = EINVAL;
return REDISMODULE_ERR;
}
/* Returns 1 if `cmd` is a command of the module `modulename`. 0 otherwise. */
int moduleIsModuleCommand(void *module_handle, struct redisCommand *cmd) {
if (cmd->proc != RedisModuleCommandDispatcher)
return 0;
if (module_handle == NULL)
return 0;
RedisModuleCommand *cp = (void*)(unsigned long)cmd->getkeys_proc;
return (cp->module == module_handle);
}
struct redisCommand *cmd = command->rediscmd;
void extendKeySpecsIfNeeded(struct redisCommand *cmd) {
/* We extend even if key_specs_num == key_specs_max because
* this function is called prior to adding a new spec */
if (cmd->key_specs_num < cmd->key_specs_max)
return;
/* Check if any info has already been set. Overwriting info involves freeing
* the old info, which is not implemented. */
if (cmd->summary || cmd->complexity || cmd->since || cmd->history ||
cmd->tips || cmd->args ||
!(cmd->key_specs_num == 0 ||
/* Allow key spec populated from legacy (first,last,step) to exist. */
(cmd->key_specs_num == 1 && cmd->key_specs == cmd->key_specs_static &&
cmd->key_specs[0].begin_search_type == KSPEC_BS_INDEX &&
cmd->key_specs[0].find_keys_type == KSPEC_FK_RANGE))) {
errno = EEXIST;
return REDISMODULE_ERR;
}
cmd->key_specs_max++;
if (info->summary) cmd->summary = zstrdup(info->summary);
if (info->complexity) cmd->complexity = zstrdup(info->complexity);
if (info->since) cmd->since = zstrdup(info->since);
if (cmd->key_specs == cmd->key_specs_static) {
cmd->key_specs = zmalloc(sizeof(keySpec) * cmd->key_specs_max);
memcpy(cmd->key_specs, cmd->key_specs_static, sizeof(keySpec) * cmd->key_specs_num);
} else {
cmd->key_specs = zrealloc(cmd->key_specs, sizeof(keySpec) * cmd->key_specs_max);
const RedisModuleCommandInfoVersion *version = info->version;
if (info->history) {
size_t count = 0;
while (moduleCmdHistoryEntryAt(version, info->history, count)->since)
count++;
serverAssert(count < SIZE_MAX / sizeof(commandHistory));
cmd->history = zmalloc(sizeof(commandHistory) * (count + 1));
for (size_t j = 0; j < count; j++) {
RedisModuleCommandHistoryEntry *entry =
moduleCmdHistoryEntryAt(version, info->history, j);
cmd->history[j].since = zstrdup(entry->since);
cmd->history[j].changes = zstrdup(entry->changes);
}
cmd->history[count].since = NULL;
cmd->history[count].changes = NULL;
cmd->num_history = count;
}
if (info->tips) {
int count;
sds *tokens = sdssplitlen(info->tips, strlen(info->tips), " ", 1, &count);
if (tokens) {
cmd->tips = zmalloc(sizeof(char *) * (count + 1));
for (int j = 0; j < count; j++) {
cmd->tips[j] = zstrdup(tokens[j]);
}
cmd->tips[count] = NULL;
cmd->num_tips = count;
sdsfreesplitres(tokens, count);
}
}
}
int moduleAddCommandKeySpec(RedisModuleCommand *command, const char *specflags, int *index) {
int64_t flags = specflags ? commandKeySpecsFlagsFromString(specflags) : 0;
if (flags == -1)
return REDISMODULE_ERR;
if (info->arity) cmd->arity = info->arity;
struct redisCommand *cmd = command->rediscmd;
if (info->key_specs) {
/* Count and allocate the key specs. */
size_t count = 0;
while (moduleCmdKeySpecAt(version, info->key_specs, count)->begin_search_type)
count++;
serverAssert(count < INT_MAX);
if (count <= STATIC_KEY_SPECS_NUM) {
cmd->key_specs_max = STATIC_KEY_SPECS_NUM;
cmd->key_specs = cmd->key_specs_static;
} else {
cmd->key_specs_max = count;
cmd->key_specs = zmalloc(sizeof(keySpec) * count);
}
extendKeySpecsIfNeeded(cmd);
/* Copy the contents of the RedisModuleCommandKeySpec array. */
cmd->key_specs_num = count;
for (size_t j = 0; j < count; j++) {
RedisModuleCommandKeySpec *spec =
moduleCmdKeySpecAt(version, info->key_specs, j);
cmd->key_specs[j].notes = spec->notes ? zstrdup(spec->notes) : NULL;
cmd->key_specs[j].flags = moduleConvertKeySpecsFlags(spec->flags, 1);
switch (spec->begin_search_type) {
case REDISMODULE_KSPEC_BS_UNKNOWN:
cmd->key_specs[j].begin_search_type = KSPEC_BS_UNKNOWN;
break;
case REDISMODULE_KSPEC_BS_INDEX:
cmd->key_specs[j].begin_search_type = KSPEC_BS_INDEX;
cmd->key_specs[j].bs.index.pos = spec->bs.index.pos;
break;
case REDISMODULE_KSPEC_BS_KEYWORD:
cmd->key_specs[j].begin_search_type = KSPEC_BS_KEYWORD;
cmd->key_specs[j].bs.keyword.keyword = zstrdup(spec->bs.keyword.keyword);
cmd->key_specs[j].bs.keyword.startfrom = spec->bs.keyword.startfrom;
break;
default:
/* Can't happen; stopped in moduleValidateCommandInfo(). */
serverPanic("Unknown begin_search_type");
}
*index = cmd->key_specs_num;
cmd->key_specs[cmd->key_specs_num].begin_search_type = KSPEC_BS_INVALID;
cmd->key_specs[cmd->key_specs_num].find_keys_type = KSPEC_FK_INVALID;
cmd->key_specs[cmd->key_specs_num].flags = flags;
cmd->key_specs_num++;
return REDISMODULE_OK;
}
switch (spec->find_keys_type) {
case REDISMODULE_KSPEC_FK_OMITTED:
/* Omitted field is shorthand to say that it's a single key. */
cmd->key_specs[j].find_keys_type = KSPEC_FK_RANGE;
cmd->key_specs[j].fk.range.lastkey = 0;
cmd->key_specs[j].fk.range.keystep = 1;
cmd->key_specs[j].fk.range.limit = 0;
break;
case REDISMODULE_KSPEC_FK_UNKNOWN:
cmd->key_specs[j].find_keys_type = KSPEC_FK_UNKNOWN;
break;
case REDISMODULE_KSPEC_FK_RANGE:
cmd->key_specs[j].find_keys_type = KSPEC_FK_RANGE;
cmd->key_specs[j].fk.range.lastkey = spec->fk.range.lastkey;
cmd->key_specs[j].fk.range.keystep = spec->fk.range.keystep;
cmd->key_specs[j].fk.range.limit = spec->fk.range.limit;
break;
case REDISMODULE_KSPEC_FK_KEYNUM:
cmd->key_specs[j].find_keys_type = KSPEC_FK_KEYNUM;
cmd->key_specs[j].fk.keynum.keynumidx = spec->fk.keynum.keynumidx;
cmd->key_specs[j].fk.keynum.firstkey = spec->fk.keynum.firstkey;
cmd->key_specs[j].fk.keynum.keystep = spec->fk.keynum.keystep;
break;
default:
/* Can't happen; stopped in moduleValidateCommandInfo(). */
serverPanic("Unknown find_keys_type");
}
}
int moduleSetCommandKeySpecBeginSearch(RedisModuleCommand *command, int index, keySpec *spec) {
struct redisCommand *cmd = command->rediscmd;
/* Update the legacy (first,last,step) spec used by the COMMAND command,
* by trying to "glue" consecutive range key specs. */
populateCommandLegacyRangeSpec(cmd);
populateCommandMovableKeys(cmd);
}
if (index >= cmd->key_specs_num)
return REDISMODULE_ERR;
if (info->args) {
cmd->args = moduleCopyCommandArgs(info->args, version);
/* Populate arg.num_args with the number of subargs, recursively */
cmd->num_args = populateArgsStructure(cmd->args);
}
cmd->key_specs[index].begin_search_type = spec->begin_search_type;
cmd->key_specs[index].bs = spec->bs;
/* Fields added in future versions to be added here, under conditions like
* `if (info->version >= 2) { access version 2 fields here }` */
return REDISMODULE_OK;
}
int moduleSetCommandKeySpecFindKeys(RedisModuleCommand *command, int index, keySpec *spec) {
struct redisCommand *cmd = command->rediscmd;
/* Returns 1 if v is a power of two, 0 otherwise. */
static inline int isPowerOfTwo(uint64_t v) {
return v && !(v & (v - 1));
}
if (index >= cmd->key_specs_num)
return REDISMODULE_ERR;
/* Returns 1 if the command info is valid and 0 otherwise. */
static int moduleValidateCommandInfo(const RedisModuleCommandInfo *info) {
const RedisModuleCommandInfoVersion *version = info->version;
if (!version) {
serverLog(LL_WARNING, "Invalid command info: version missing");
return 0;
}
/* No validation for the fields summary, complexity, since, tips (strings or
* NULL) and arity (any integer). */
/* History: If since is set, changes must also be set. */
if (info->history) {
for (size_t j = 0;
moduleCmdHistoryEntryAt(version, info->history, j)->since;
j++)
{
if (!moduleCmdHistoryEntryAt(version, info->history, j)->changes) {
serverLog(LL_WARNING, "Invalid command info: history[%zd].changes missing", j);
return 0;
}
}
}
cmd->key_specs[index].find_keys_type = spec->find_keys_type;
cmd->key_specs[index].fk = spec->fk;
/* Key specs. */
if (info->key_specs) {
for (size_t j = 0;
moduleCmdKeySpecAt(version, info->key_specs, j)->begin_search_type;
j++)
{
RedisModuleCommandKeySpec *spec =
moduleCmdKeySpecAt(version, info->key_specs, j);
if (j >= INT_MAX) {
serverLog(LL_WARNING, "Invalid command info: Too many key specs");
return 0; /* redisCommand.key_specs_num is an int. */
}
/* Refresh legacy range */
populateCommandLegacyRangeSpec(cmd);
/* Refresh movablekeys flag */
populateCommandMovableKeys(cmd);
/* Flags. Exactly one flag in a group is set if and only if the
* masked bits is a power of two. */
uint64_t key_flags =
REDISMODULE_CMD_KEY_RO | REDISMODULE_CMD_KEY_RW |
REDISMODULE_CMD_KEY_OW | REDISMODULE_CMD_KEY_RM;
uint64_t write_flags =
REDISMODULE_CMD_KEY_INSERT | REDISMODULE_CMD_KEY_DELETE |
REDISMODULE_CMD_KEY_UPDATE;
if (!isPowerOfTwo(spec->flags & key_flags)) {
serverLog(LL_WARNING,
"Invalid command info: key_specs[%zd].flags: "
"Exactly one of the flags RO, RW, OW, RM reqired", j);
return 0;
}
if ((spec->flags & write_flags) != 0 &&
!isPowerOfTwo(spec->flags & write_flags))
{
serverLog(LL_WARNING,
"Invalid command info: key_specs[%zd].flags: "
"INSERT, DELETE and UPDATE are mutually exclusive", j);
return 0;
}
return REDISMODULE_OK;
}
switch (spec->begin_search_type) {
case REDISMODULE_KSPEC_BS_UNKNOWN: break;
case REDISMODULE_KSPEC_BS_INDEX: break;
case REDISMODULE_KSPEC_BS_KEYWORD:
if (spec->bs.keyword.keyword == NULL) {
serverLog(LL_WARNING,
"Invalid command info: key_specs[%zd].bs.keyword.keyword "
"required when begin_search_type is KEYWORD", j);
return 0;
}
break;
default:
serverLog(LL_WARNING,
"Invalid command info: key_specs[%zd].begin_search_type: "
"Invalid value %d", j, spec->begin_search_type);
return 0;
}
/* **The key spec API is not officially released and it is going to be changed
* in Redis 7.0. It has been disabled temporarily.**
*
* Key specs is a scheme that tries to describe the location
* of key arguments better than the old [first,last,step] scheme
* which is limited and doesn't fit many commands.
*
* This information is used by ACL, Cluster and the `COMMAND` command.
*
* There are two steps to retrieve the key arguments:
*
* - `begin_search` (BS): in which index should we start seacrhing for keys?
* - `find_keys` (FK): relative to the output of BS, how can we will which args are keys?
*
* There are two types of BS:
*
* - `index`: key args start at a constant index
* - `keyword`: key args start just after a specific keyword
*
* There are two kinds of FK:
*
* - `range`: keys end at a specific index (or relative to the last argument)
* - `keynum`: there's an arg that contains the number of key args somewhere before the keys themselves
*
* This function adds a new key spec to a command, returning a unique id in `spec_id`.
* The caller must then call one of the RedisModule_SetCommandKeySpecBeginSearch* APIs
* followed by one of the RedisModule_SetCommandKeySpecFindKeys* APIs.
*
* It should be called just after RedisModule_CreateCommand.
*
* Example:
*
* if (RedisModule_CreateCommand(ctx,"kspec.smove",kspec_legacy,"",0,0,0) == REDISMODULE_ERR)
* return REDISMODULE_ERR;
*
* if (RedisModule_AddCommandKeySpec(ctx,"kspec.smove","RW access delete",&spec_id) == REDISMODULE_ERR)
* return REDISMODULE_ERR;
* if (RedisModule_SetCommandKeySpecBeginSearchIndex(ctx,"kspec.smove",spec_id,1) == REDISMODULE_ERR)
* return REDISMODULE_ERR;
* if (RedisModule_SetCommandKeySpecFindKeysRange(ctx,"kspec.smove",spec_id,0,1,0) == REDISMODULE_ERR)
* return REDISMODULE_ERR;
*
* if (RedisModule_AddCommandKeySpec(ctx,"kspec.smove","RW insert",&spec_id) == REDISMODULE_ERR)
* return REDISMODULE_ERR;
* if (RedisModule_SetCommandKeySpecBeginSearchIndex(ctx,"kspec.smove",spec_id,2) == REDISMODULE_ERR)
* return REDISMODULE_ERR;
* if (RedisModule_SetCommandKeySpecFindKeysRange(ctx,"kspec.smove",spec_id,0,1,0) == REDISMODULE_ERR)
* return REDISMODULE_ERR;
*
* It is also possible to use this API on subcommands (See RedisModule_CreateSubcommand).
* The name of the subcommand should be the name of the parent command + "|" + name of subcommand.
*
* Example:
*
* RedisModule_AddCommandKeySpec(ctx,"module.object|encoding","RO",&spec_id)
*
* Returns REDISMODULE_OK on success
*/
int RM_AddCommandKeySpec(RedisModuleCommand *command, const char *specflags, int *spec_id) {
return moduleAddCommandKeySpec(command, specflags, spec_id);
}
/* Validate find_keys_type. */
switch (spec->find_keys_type) {
case REDISMODULE_KSPEC_FK_OMITTED: break; /* short for RANGE {0,1,0} */
case REDISMODULE_KSPEC_FK_UNKNOWN: break;
case REDISMODULE_KSPEC_FK_RANGE: break;
case REDISMODULE_KSPEC_FK_KEYNUM: break;
default:
serverLog(LL_WARNING,
"Invalid command info: key_specs[%zd].find_keys_type: "
"Invalid value %d", j, spec->find_keys_type);
return 0;
}
}
}
/* Set a "index" key arguments spec to a command (begin_search step).
* See RedisModule_AddCommandKeySpec's doc.
*
* - `index`: The index from which we start the search for keys
*
* Returns REDISMODULE_OK */
int RM_SetCommandKeySpecBeginSearchIndex(RedisModuleCommand *command, int spec_id, int index) {
keySpec spec;
spec.begin_search_type = KSPEC_BS_INDEX;
spec.bs.index.pos = index;
/* Args, subargs (recursive) */
return moduleValidateCommandArgs(info->args, version);
}
/* When from_api is true, converts from REDISMODULE_CMD_KEY_* flags to CMD_KEY_* flags.
* When from_api is false, converts from CMD_KEY_* flags to REDISMODULE_CMD_KEY_* flags. */
static int64_t moduleConvertKeySpecsFlags(int64_t flags, int from_api) {
int64_t out = 0;
int64_t map[][2] = {
{REDISMODULE_CMD_KEY_RO, CMD_KEY_RO},
{REDISMODULE_CMD_KEY_RW, CMD_KEY_RW},
{REDISMODULE_CMD_KEY_OW, CMD_KEY_OW},
{REDISMODULE_CMD_KEY_RM, CMD_KEY_RM},
{REDISMODULE_CMD_KEY_ACCESS, CMD_KEY_ACCESS},
{REDISMODULE_CMD_KEY_INSERT, CMD_KEY_INSERT},
{REDISMODULE_CMD_KEY_UPDATE, CMD_KEY_UPDATE},
{REDISMODULE_CMD_KEY_DELETE, CMD_KEY_DELETE},
{REDISMODULE_CMD_KEY_NOT_KEY, CMD_KEY_NOT_KEY},
{REDISMODULE_CMD_KEY_INCOMPLETE, CMD_KEY_INCOMPLETE},
{REDISMODULE_CMD_KEY_VARIABLE_FLAGS, CMD_KEY_VARIABLE_FLAGS},
{0,0}};
int from_idx = from_api ? 0 : 1, to_idx = !from_idx;
for (int i=0; map[i][0]; i++)
if (flags & map[i][from_idx]) out |= map[i][to_idx];
return out;
}
/* Validates an array of RedisModuleCommandArg. Returns 1 if it's valid and 0 if
* it's invalid. */
static int moduleValidateCommandArgs(RedisModuleCommandArg *args,
const RedisModuleCommandInfoVersion *version) {
if (args == NULL) return 1; /* Missing args is OK. */
for (size_t j = 0; moduleCmdArgAt(version, args, j)->name != NULL; j++) {
RedisModuleCommandArg *arg = moduleCmdArgAt(version, args, j);
int arg_type_error = 0;
moduleConvertArgType(arg->type, &arg_type_error);
if (arg_type_error) {
serverLog(LL_WARNING,
"Invalid command info: Argument \"%s\": Undefined type %d",
arg->name, arg->type);
return 0;
}
if (arg->type == REDISMODULE_ARG_TYPE_PURE_TOKEN && !arg->token) {
serverLog(LL_WARNING,
"Invalid command info: Argument \"%s\": "
"token required when type is PURE_TOKEN", args[j].name);
return 0;
}
return moduleSetCommandKeySpecBeginSearch(command, spec_id, &spec);
}
if (arg->type == REDISMODULE_ARG_TYPE_KEY) {
if (arg->key_spec_index < 0) {
serverLog(LL_WARNING,
"Invalid command info: Argument \"%s\": "
"key_spec_index required when type is KEY",
arg->name);
return 0;
}
} else if (arg->key_spec_index != -1 && arg->key_spec_index != 0) {
/* 0 is allowed for convenience, to allow it to be omitted in
* compound struct literals on the form `.field = value`. */
serverLog(LL_WARNING,
"Invalid command info: Argument \"%s\": "
"key_spec_index specified but type isn't KEY",
arg->name);
return 0;
}
/* Set a "keyword" key arguments spec to a command (begin_search step).
* See RedisModule_AddCommandKeySpec's doc.
*
* - `keyword`: The keyword that indicates the beginning of key args
* - `startfrom`: An index in argv from which to start searching.
* Can be negative, which means start search from the end, in reverse
* (Example: -2 means to start in reverse from the panultimate arg)
*
* Returns REDISMODULE_OK */
int RM_SetCommandKeySpecBeginSearchKeyword(RedisModuleCommand *command, int spec_id, const char *keyword, int startfrom) {
keySpec spec;
spec.begin_search_type = KSPEC_BS_KEYWORD;
spec.bs.keyword.keyword = keyword;
spec.bs.keyword.startfrom = startfrom;
if (arg->flags & ~(_REDISMODULE_CMD_ARG_NEXT - 1)) {
serverLog(LL_WARNING,
"Invalid command info: Argument \"%s\": Invalid flags",
arg->name);
return 0;
}
return moduleSetCommandKeySpecBeginSearch(command, spec_id, &spec);
if (arg->type == REDISMODULE_ARG_TYPE_ONEOF ||
arg->type == REDISMODULE_ARG_TYPE_BLOCK)
{
if (arg->subargs == NULL) {
serverLog(LL_WARNING,
"Invalid command info: Argument \"%s\": "
"subargs required when type is ONEOF or BLOCK",
arg->name);
return 0;
}
if (!moduleValidateCommandArgs(arg->subargs, version)) return 0;
} else {
if (arg->subargs != NULL) {
serverLog(LL_WARNING,
"Invalid command info: Argument \"%s\": "
"subargs specified but type isn't ONEOF nor BLOCK",
arg->name);
return 0;
}
}
}
return 1;
}
/* Set a "range" key arguments spec to a command (find_keys step).
* See RedisModule_AddCommandKeySpec's doc.
*
* - `lastkey`: Relative index (to the result of the begin_search step) where the last key is.
* Can be negative, in which case it's not relative. -1 indicating till the last argument,
* -2 one before the last and so on.
* - `keystep`: How many args should we skip after finding a key, in order to find the next one.
* - `limit`: If lastkey is -1, we use limit to stop the search by a factor. 0 and 1 mean no limit.
* 2 means 1/2 of the remaining args, 3 means 1/3, and so on.
*
* Returns REDISMODULE_OK */
int RM_SetCommandKeySpecFindKeysRange(RedisModuleCommand *command, int spec_id, int lastkey, int keystep, int limit) {
keySpec spec;
spec.find_keys_type = KSPEC_FK_RANGE;
spec.fk.range.lastkey = lastkey;
spec.fk.range.keystep = keystep;
spec.fk.range.limit = limit;
/* Converts an array of RedisModuleCommandArg into a freshly allocated array of
* struct redisCommandArg. */
static struct redisCommandArg *moduleCopyCommandArgs(RedisModuleCommandArg *args,
const RedisModuleCommandInfoVersion *version) {
size_t count = 0;
while (moduleCmdArgAt(version, args, count)->name) count++;
serverAssert(count < SIZE_MAX / sizeof(struct redisCommandArg));
struct redisCommandArg *realargs = zcalloc((count+1) * sizeof(redisCommandArg));
for (size_t j = 0; j < count; j++) {
RedisModuleCommandArg *arg = moduleCmdArgAt(version, args, j);
realargs[j].name = zstrdup(arg->name);
realargs[j].type = moduleConvertArgType(arg->type, NULL);
if (arg->type == REDISMODULE_ARG_TYPE_KEY)
realargs[j].key_spec_index = arg->key_spec_index;
else
realargs[j].key_spec_index = -1;
if (arg->token) realargs[j].token = zstrdup(arg->token);
if (arg->summary) realargs[j].summary = zstrdup(arg->summary);
if (arg->since) realargs[j].since = zstrdup(arg->since);
realargs[j].flags = moduleConvertArgFlags(arg->flags);
if (arg->subargs) realargs[j].subargs = moduleCopyCommandArgs(arg->subargs, version);
}
return realargs;
}
static redisCommandArgType moduleConvertArgType(RedisModuleCommandArgType type, int *error) {
if (error) *error = 0;
switch (type) {
case REDISMODULE_ARG_TYPE_STRING: return ARG_TYPE_STRING;
case REDISMODULE_ARG_TYPE_INTEGER: return ARG_TYPE_INTEGER;
case REDISMODULE_ARG_TYPE_DOUBLE: return ARG_TYPE_DOUBLE;
case REDISMODULE_ARG_TYPE_KEY: return ARG_TYPE_KEY;
case REDISMODULE_ARG_TYPE_PATTERN: return ARG_TYPE_PATTERN;
case REDISMODULE_ARG_TYPE_UNIX_TIME: return ARG_TYPE_UNIX_TIME;
case REDISMODULE_ARG_TYPE_PURE_TOKEN: return ARG_TYPE_PURE_TOKEN;
case REDISMODULE_ARG_TYPE_ONEOF: return ARG_TYPE_ONEOF;
case REDISMODULE_ARG_TYPE_BLOCK: return ARG_TYPE_BLOCK;
default:
if (error) *error = 1;
return -1;
}
}
return moduleSetCommandKeySpecFindKeys(command, spec_id, &spec);
static int moduleConvertArgFlags(int flags) {
int realflags = 0;
if (flags & REDISMODULE_CMD_ARG_OPTIONAL) realflags |= CMD_ARG_OPTIONAL;
if (flags & REDISMODULE_CMD_ARG_MULTIPLE) realflags |= CMD_ARG_MULTIPLE;
if (flags & REDISMODULE_CMD_ARG_MULTIPLE_TOKEN) realflags |= CMD_ARG_MULTIPLE_TOKEN;
return realflags;
}
/* Set a "keynum" key arguments spec to a command (find_keys step).
* See RedisModule_AddCommandKeySpec's doc.
*
* - `keynumidx`: Relative index (to the result of the begin_search step) where the arguments that
* contains the number of keys is.
* - `firstkey`: Relative index (to the result of the begin_search step) where the first key is
* found (Usually it's just after keynumidx, so it should be keynumidx+1)
* - `keystep`: How many args should we skip after finding a key, in order to find the next one.
*
* Returns REDISMODULE_OK */
int RM_SetCommandKeySpecFindKeysKeynum(RedisModuleCommand *command, int spec_id, int keynumidx, int firstkey, int keystep) {
keySpec spec;
spec.find_keys_type = KSPEC_FK_KEYNUM;
spec.fk.keynum.keynumidx = keynumidx;
spec.fk.keynum.firstkey = firstkey;
spec.fk.keynum.keystep = keystep;
/* Return `struct RedisModule *` as `void *` to avoid exposing it outside of module.c. */
void *moduleGetHandleByName(char *modulename) {
return dictFetchValue(modules,modulename);
}
return moduleSetCommandKeySpecFindKeys(command, spec_id, &spec);
/* Returns 1 if `cmd` is a command of the module `modulename`. 0 otherwise. */
int moduleIsModuleCommand(void *module_handle, struct redisCommand *cmd) {
if (cmd->proc != RedisModuleCommandDispatcher)
return 0;
if (module_handle == NULL)
return 0;
RedisModuleCommand *cp = (void*)(unsigned long)cmd->getkeys_proc;
return (cp->module == module_handle);
}
/* --------------------------------------------------------------------------
......@@ -2399,6 +2997,15 @@ int RM_ReplyWithCallReply(RedisModuleCtx *ctx, RedisModuleCallReply *reply) {
size_t proto_len;
const char *proto = callReplyGetProto(reply, &proto_len);
addReplyProto(c, proto, proto_len);
/* Propagate the error list from that reply to the other client, to do some
* post error reply handling, like statistics.
* Note that if the original reply had an array with errors, and the module
* replied with just a portion of the original reply, and not the entire
* reply, the errors are currently not propagated and the errors stats
* will not get propagated. */
list *errors = callReplyDeferredErrorList(reply);
if (errors)
deferredAfterErrorReply(c, errors);
return REDISMODULE_OK;
}
......@@ -5051,7 +5658,7 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch
errno = ENOENT;
goto cleanup;
}
c->cmd = c->lastcmd = cmd;
c->cmd = c->lastcmd = c->realcmd = cmd;
/* Basic arity checks. */
if ((cmd->arity > 0 && cmd->arity != argc) || (argc < -cmd->arity)) {
......@@ -5135,7 +5742,8 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch
proto = sdscatlen(proto,o->buf,o->used);
listDelNode(c->reply,listFirst(c->reply));
}
reply = callReplyCreate(proto, ctx);
reply = callReplyCreate(proto, c->deferred_reply_errors, ctx);
c->deferred_reply_errors = NULL; /* now the responsibility of the reply object. */
autoMemoryAdd(ctx,REDISMODULE_AM_REPLY,reply);
cleanup:
......@@ -6572,6 +7180,7 @@ void moduleHandleBlockedClients(void) {
* was blocked on keys (RM_BlockClientOnKeys()), because we already
* called such callback in moduleTryServeClientBlockedOnKey() when
* the key was signaled as ready. */
long long prev_error_replies = server.stat_total_error_replies;
uint64_t reply_us = 0;
if (c && !bc->blocked_on_keys && bc->reply_callback) {
RedisModuleCtx ctx;
......@@ -6586,13 +7195,6 @@ void moduleHandleBlockedClients(void) {
reply_us = elapsedUs(replyTimer);
moduleFreeContext(&ctx);
}
/* Update stats now that we've finished the blocking operation.
* This needs to be out of the reply callback above given that a
* module might not define any callback and still do blocking ops.
*/
if (c && !bc->blocked_on_keys) {
updateStatsOnUnblock(c, bc->background_duration, reply_us);
}
/* Free privdata if any. */
if (bc->privdata && bc->free_privdata) {
......@@ -6613,6 +7215,14 @@ void moduleHandleBlockedClients(void) {
moduleReleaseTempClient(bc->reply_client);
moduleReleaseTempClient(bc->thread_safe_ctx_client);
/* Update stats now that we've finished the blocking operation.
* This needs to be out of the reply callback above given that a
* module might not define any callback and still do blocking ops.
*/
if (c && !bc->blocked_on_keys) {
updateStatsOnUnblock(c, bc->background_duration, reply_us, server.stat_total_error_replies != prev_error_replies);
}
if (c != NULL) {
/* Before unblocking the client, set the disconnect callback
* to NULL, because if we reached this point, the client was
......@@ -6670,10 +7280,11 @@ void moduleBlockedClientTimedOut(client *c) {
ctx.client = bc->client;
ctx.blocked_client = bc;
ctx.blocked_privdata = bc->privdata;
long long prev_error_replies = server.stat_total_error_replies;
bc->timeout_callback(&ctx,(void**)c->argv,c->argc);
moduleFreeContext(&ctx);
if (!bc->blocked_on_keys) {
updateStatsOnUnblock(c, bc->background_duration, 0);
updateStatsOnUnblock(c, bc->background_duration, 0, server.stat_total_error_replies != prev_error_replies);
}
/* For timeout events, we do not want to call the disconnect callback,
* because the blocked client will be automatically disconnected in
......@@ -7427,6 +8038,24 @@ int RM_GetTimerInfo(RedisModuleCtx *ctx, RedisModuleTimerID id, uint64_t *remain
return REDISMODULE_OK;
}
/* Query timers to see if any timer belongs to the module.
* Return 1 if any timer was found, otherwise 0 would be returned. */
int moduleHoldsTimer(struct RedisModule *module) {
raxIterator iter;
int found = 0;
raxStart(&iter,Timers);
raxSeek(&iter,"^",NULL,0);
while (raxNext(&iter)) {
RedisModuleTimer *timer = iter.data;
if (timer->module == module) {
found = 1;
break;
}
}
raxStop(&iter);
return found;
}
/* --------------------------------------------------------------------------
* ## Modules EventLoop API
* --------------------------------------------------------------------------*/
......@@ -7808,28 +8437,34 @@ int RM_ACLCheckCommandPermissions(RedisModuleUser *user, RedisModuleString **arg
return REDISMODULE_OK;
}
/* Check if the key can be accessed by the user, according to the ACLs associated with it
* and the flags used. The supported flags are:
*
* REDISMODULE_KEY_PERMISSION_READ: Can the module read data from the key.
* REDISMODULE_KEY_PERMISSION_WRITE: Can the module write data to the key.
/* Check if the key can be accessed by the user according to the ACLs attached to the user
* and the flags representing the key access. The flags are the same that are used in the
* keyspec for logical operations. These flags are documented in RedisModule_SetCommandInfo as
* the REDISMODULE_CMD_KEY_ACCESS, REDISMODULE_CMD_KEY_UPDATE, REDISMODULE_CMD_KEY_INSERT,
* and REDISMODULE_CMD_KEY_DELETE flags.
*
* If no flags are supplied, the user is still required to have some access to the key for
* this command to return successfully.
*
* On success a REDISMODULE_OK is returned, otherwise
* REDISMODULE_ERR is returned and errno is set to the following values:
* If the user is able to access the key then REDISMODULE_OK is returned, otherwise
* REDISMODULE_ERR is returned and errno is set to one of the following values:
*
* * EINVAL: The provided flags are invalid.
* * EACCESS: The user does not have permission to access the key.
*/
int RM_ACLCheckKeyPermissions(RedisModuleUser *user, RedisModuleString *key, int flags) {
int acl_flags = 0;
if (flags & REDISMODULE_KEY_PERMISSION_READ) acl_flags |= ACL_READ_PERMISSION;
if (flags & REDISMODULE_KEY_PERMISSION_WRITE) acl_flags |= ACL_WRITE_PERMISSION;
if (!acl_flags || ((flags & REDISMODULE_KEY_PERMISSION_ALL) != flags)) {
const int allow_mask = (REDISMODULE_CMD_KEY_ACCESS
| REDISMODULE_CMD_KEY_INSERT
| REDISMODULE_CMD_KEY_DELETE
| REDISMODULE_CMD_KEY_UPDATE);
if ((flags & allow_mask) != flags) {
errno = EINVAL;
return REDISMODULE_ERR;
}
if (ACLUserCheckKeyPerm(user->user, key->ptr, sdslen(key->ptr), acl_flags) != ACL_OK) {
int keyspec_flags = moduleConvertKeySpecsFlags(flags, 0);
if (ACLUserCheckKeyPerm(user->user, key->ptr, sdslen(key->ptr), keyspec_flags) != ACL_OK) {
errno = EACCES;
return REDISMODULE_ERR;
}
......@@ -7837,14 +8472,34 @@ int RM_ACLCheckKeyPermissions(RedisModuleUser *user, RedisModuleString *key, int
return REDISMODULE_OK;
}
/* Check if the pubsub channel can be accessed by the user, according to the ACLs associated with it.
* Glob-style pattern matching is employed, unless the literal flag is
* set.
/* Check if the pubsub channel can be accessed by the user based off of the given
* access flags. See RM_ChannelAtPosWithFlags for more information about the
* possible flags that can be passed in.
*
* If the user can access the pubsub channel, REDISMODULE_OK is returned, otherwise
* REDISMODULE_ERR is returned. */
int RM_ACLCheckChannelPermissions(RedisModuleUser *user, RedisModuleString *ch, int literal) {
if (ACLUserCheckChannelPerm(user->user, ch->ptr, literal) != ACL_OK)
* If the user is able to acecss the pubsub channel then REDISMODULE_OK is returned, otherwise
* REDISMODULE_ERR is returned and errno is set to one of the following values:
*
* * EINVAL: The provided flags are invalid.
* * EACCESS: The user does not have permission to access the pubsub channel.
*/
int RM_ACLCheckChannelPermissions(RedisModuleUser *user, RedisModuleString *ch, int flags) {
const int allow_mask = (REDISMODULE_CMD_CHANNEL_PUBLISH
| REDISMODULE_CMD_CHANNEL_SUBSCRIBE
| REDISMODULE_CMD_CHANNEL_UNSUBSCRIBE
| REDISMODULE_CMD_CHANNEL_PATTERN);
if ((flags & allow_mask) != flags) {
errno = EINVAL;
return REDISMODULE_ERR;
}
/* Unsubscribe permissions are currently always allowed. */
if (flags & REDISMODULE_CMD_CHANNEL_UNSUBSCRIBE){
return REDISMODULE_OK;
}
int is_pattern = flags & REDISMODULE_CMD_CHANNEL_PATTERN;
if (ACLUserCheckChannelPerm(user->user, ch->ptr, is_pattern) != ACL_OK)
return REDISMODULE_ERR;
return REDISMODULE_OK;
......@@ -8254,9 +8909,10 @@ int RM_InfoAddSection(RedisModuleInfoCtx *ctx, const char *name) {
* 1) no section was requested (emit all)
* 2) the module name was requested (emit all)
* 3) this specific section was requested. */
if (ctx->requested_section) {
if (strcasecmp(ctx->requested_section, full_name) &&
strcasecmp(ctx->requested_section, ctx->module->name)) {
if (ctx->requested_sections) {
if ((!full_name || !dictFind(ctx->requested_sections, full_name)) &&
(!dictFind(ctx->requested_sections, ctx->module->name)))
{
sdsfree(full_name);
ctx->in_section = 0;
return REDISMODULE_ERR;
......@@ -8405,7 +9061,7 @@ int RM_RegisterInfoFunc(RedisModuleCtx *ctx, RedisModuleInfoFunc cb) {
return REDISMODULE_OK;
}
sds modulesCollectInfo(sds info, const char *section, int for_crash_report, int sections) {
sds modulesCollectInfo(sds info, dict *sections_dict, int for_crash_report, int sections) {
dictIterator *di = dictGetIterator(modules);
dictEntry *de;
......@@ -8413,7 +9069,7 @@ sds modulesCollectInfo(sds info, const char *section, int for_crash_report, int
struct RedisModule *module = dictGetVal(de);
if (!module->info_cb)
continue;
RedisModuleInfoCtx info_ctx = {module, section, info, sections, 0, 0};
RedisModuleInfoCtx info_ctx = {module, sections_dict, info, sections, 0, 0};
module->info_cb(&info_ctx, for_crash_report);
/* Implicitly end dicts (no way to handle errors, and we must add the newline). */
if (info_ctx.in_dict_field)
......@@ -8435,7 +9091,11 @@ RedisModuleServerInfoData *RM_GetServerInfo(RedisModuleCtx *ctx, const char *sec
struct RedisModuleServerInfoData *d = zmalloc(sizeof(*d));
d->rax = raxNew();
if (ctx != NULL) autoMemoryAdd(ctx,REDISMODULE_AM_INFO,d);
sds info = genRedisInfoString(section);
int all = 0, everything = 0;
robj *argv[1];
argv[0] = section ? createStringObject(section, strlen(section)) : NULL;
dict *section_dict = genInfoSectionDict(argv, section ? 1 : 0, NULL, &all, &everything);
sds info = genRedisInfoString(section_dict, all, everything);
int totlines, i;
sds *lines = sdssplitlen(info, sdslen(info), "\r\n", 2, &totlines);
for(i=0; i<totlines; i++) {
......@@ -8451,6 +9111,8 @@ RedisModuleServerInfoData *RM_GetServerInfo(RedisModuleCtx *ctx, const char *sec
}
sdsfree(info);
sdsfreesplitres(lines,totlines);
releaseInfoSectionDict(section_dict);
if(argv[0]) decrRefCount(argv[0]);
return d;
}
......@@ -9995,17 +10657,23 @@ int moduleFreeCommand(struct RedisModule *module, struct redisCommand *cmd) {
return C_ERR;
/* Free everything except cmd->fullname and cmd itself. */
for (int j = 0; j < cmd->key_specs_num; j++) {
if (cmd->key_specs[j].notes)
zfree((char *)cmd->key_specs[j].notes);
if (cmd->key_specs[j].begin_search_type == KSPEC_BS_KEYWORD)
zfree((char *)cmd->key_specs[j].bs.keyword.keyword);
}
if (cmd->key_specs != cmd->key_specs_static)
zfree(cmd->key_specs);
for (int j = 0; cmd->tips && cmd->tips[j]; j++)
sdsfree((sds)cmd->tips[j]);
zfree((char *)cmd->tips[j]);
for (int j = 0; cmd->history && cmd->history[j].since; j++) {
sdsfree((sds)cmd->history[j].since);
sdsfree((sds)cmd->history[j].changes);
zfree((char *)cmd->history[j].since);
zfree((char *)cmd->history[j].changes);
}
sdsfree((sds)cmd->summary);
sdsfree((sds)cmd->since);
sdsfree((sds)cmd->complexity);
zfree((char *)cmd->summary);
zfree((char *)cmd->since);
zfree((char *)cmd->complexity);
if (cmd->latency_histogram) {
hdr_close(cmd->latency_histogram);
cmd->latency_histogram = NULL;
......@@ -10125,6 +10793,7 @@ int moduleLoad(const char *path, void **module_argv, int module_argc) {
* * EBUSY: The module exports a new data type and can only be reloaded.
* * EPERM: The module exports APIs which are used by other module.
* * EAGAIN: The module has blocked clients.
* * EINPROGRESS: The module holds timer not fired.
* * ECANCELED: Unload module error. */
int moduleUnload(sds name) {
struct RedisModule *module = dictFetchValue(modules,name);
......@@ -10141,6 +10810,9 @@ int moduleUnload(sds name) {
} else if (module->blocked_clients) {
errno = EAGAIN;
return C_ERR;
} else if (moduleHoldsTimer(module)) {
errno = EINPROGRESS;
return C_ERR;
}
/* Give module a chance to clean up. */
......@@ -10255,6 +10927,8 @@ sds genModulesInfoStringRenderModuleOptions(struct RedisModule *module) {
output = sdscat(output,"handle-io-errors|");
if (module->options & REDISMODULE_OPTIONS_HANDLE_REPL_ASYNC_LOAD)
output = sdscat(output,"handle-repl-async-load|");
if (module->options & REDISMODULE_OPTION_NO_IMPLICIT_SIGNAL_MODIFIED)
output = sdscat(output,"no-implicit-signal-modified|");
output = sdstrim(output,"|");
output = sdscat(output,"]");
return output;
......@@ -10345,6 +11019,10 @@ NULL
errmsg = "the module has blocked clients. "
"Please wait them unblocked and try again";
break;
case EINPROGRESS:
errmsg = "the module holds timer that is not fired. "
"Please stop the timer or wait until it fires.";
break;
default:
errmsg = "operation not possible.";
break;
......@@ -10511,6 +11189,10 @@ int RM_ModuleTypeReplaceValue(RedisModuleKey *key, moduleType *mt, void *new_val
* contains the indexes of all key name arguments. This function is
* essentially a more efficient way to do `COMMAND GETKEYS`.
*
* The out_flags argument is optional, and can be set to NULL.
* When provided it is filled with REDISMODULE_CMD_KEY_ flags in matching
* indexes with the key indexes of the returned array.
*
* A NULL return value indicates the specified command has no keys, or
* an error condition. Error conditions are indicated by setting errno
* as follows:
......@@ -10520,9 +11202,10 @@ int RM_ModuleTypeReplaceValue(RedisModuleKey *key, moduleType *mt, void *new_val
*
* NOTE: The returned array is not a Redis Module object so it does not
* get automatically freed even when auto-memory is used. The caller
* must explicitly call RM_Free() to free it.
* must explicitly call RM_Free() to free it, same as the out_flags pointer if
* used.
*/
int *RM_GetCommandKeys(RedisModuleCtx *ctx, RedisModuleString **argv, int argc, int *num_keys) {
int *RM_GetCommandKeysWithFlags(RedisModuleCtx *ctx, RedisModuleString **argv, int argc, int *num_keys, int **out_flags) {
UNUSED(ctx);
struct redisCommand *cmd;
int *res = NULL;
......@@ -10557,13 +11240,22 @@ int *RM_GetCommandKeys(RedisModuleCtx *ctx, RedisModuleString **argv, int argc,
/* The return value here expects an array of key positions */
unsigned long int size = sizeof(int) * result.numkeys;
res = zmalloc(size);
if (out_flags)
*out_flags = zmalloc(size);
for (int i = 0; i < result.numkeys; i++) {
res[i] = result.keys[i].pos;
if (out_flags)
(*out_flags)[i] = moduleConvertKeySpecsFlags(result.keys[i].flags, 0);
}
return res;
}
/* Identinal to RM_GetCommandKeysWithFlags when flags are not needed. */
int *RM_GetCommandKeys(RedisModuleCtx *ctx, RedisModuleString **argv, int argc, int *num_keys) {
return RM_GetCommandKeysWithFlags(ctx, argv, argc, num_keys, NULL);
}
/* Return the name of the command currently running */
const char *RM_GetCurrentCommandName(RedisModuleCtx *ctx) {
if (!ctx || !ctx->client || !ctx->client->cmd)
......@@ -10803,6 +11495,7 @@ void moduleRegisterCoreAPI(void) {
REGISTER_API(CreateCommand);
REGISTER_API(GetCommand);
REGISTER_API(CreateSubcommand);
REGISTER_API(SetCommandInfo);
REGISTER_API(SetModuleAttribs);
REGISTER_API(IsModuleNameBusy);
REGISTER_API(WrongArity);
......@@ -10913,6 +11606,9 @@ void moduleRegisterCoreAPI(void) {
REGISTER_API(StreamTrimByID);
REGISTER_API(IsKeysPositionRequest);
REGISTER_API(KeyAtPos);
REGISTER_API(KeyAtPosWithFlags);
REGISTER_API(IsChannelsPositionRequest);
REGISTER_API(ChannelAtPosWithFlags);
REGISTER_API(GetClientId);
REGISTER_API(GetClientUserNameById);
REGISTER_API(GetContextFlags);
......@@ -11090,6 +11786,7 @@ void moduleRegisterCoreAPI(void) {
REGISTER_API(GetServerVersion);
REGISTER_API(GetClientCertificate);
REGISTER_API(GetCommandKeys);
REGISTER_API(GetCommandKeysWithFlags);
REGISTER_API(GetCurrentCommandName);
REGISTER_API(GetTypeMethodVersion);
REGISTER_API(RegisterDefragFunc);
......@@ -11098,13 +11795,6 @@ void moduleRegisterCoreAPI(void) {
REGISTER_API(DefragShouldStop);
REGISTER_API(DefragCursorSet);
REGISTER_API(DefragCursorGet);
#ifdef INCLUDE_UNRELEASED_KEYSPEC_API
REGISTER_API(AddCommandKeySpec);
REGISTER_API(SetCommandKeySpecBeginSearchIndex);
REGISTER_API(SetCommandKeySpecBeginSearchKeyword);
REGISTER_API(SetCommandKeySpecFindKeysRange);
REGISTER_API(SetCommandKeySpecFindKeysKeynum);
#endif
REGISTER_API(EventLoopAdd);
REGISTER_API(EventLoopDel);
REGISTER_API(EventLoopAddOneShot);
......
......@@ -11,6 +11,13 @@ else
SHOBJ_LDFLAGS ?= -bundle -undefined dynamic_lookup
endif
# OS X 11.x doesn't have /usr/lib/libSystem.dylib and needs an explicit setting.
ifeq ($(uname_S),Darwin)
ifeq ("$(wildcard /usr/lib/libSystem.dylib)","")
LIBS = -L /Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/lib -lsystem
endif
endif
.SUFFIXES: .c .so .xo .o
all: helloworld.so hellotype.so helloblock.so hellocluster.so hellotimer.so hellodict.so hellohook.so helloacl.so
......
......@@ -76,7 +76,7 @@ int ListCommand_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int
void PingReceiver(RedisModuleCtx *ctx, const char *sender_id, uint8_t type, const unsigned char *payload, uint32_t len) {
RedisModule_Log(ctx,"notice","PING (type %d) RECEIVED from %.*s: '%.*s'",
type,REDISMODULE_NODE_ID_LEN,sender_id,(int)len, payload);
RedisModule_SendClusterMessage(ctx,NULL,MSGTYPE_PONG,(unsigned char*)"Ohi!",4);
RedisModule_SendClusterMessage(ctx,NULL,MSGTYPE_PONG,"Ohi!",4);
RedisModuleCallReply *reply = RedisModule_Call(ctx, "INCR", "c", "pings_received");
RedisModule_FreeCallReply(reply);
}
......
......@@ -189,7 +189,7 @@ void execCommand(client *c) {
c->argc = c->mstate.commands[j].argc;
c->argv = c->mstate.commands[j].argv;
c->argv_len = c->mstate.commands[j].argv_len;
c->cmd = c->mstate.commands[j].cmd;
c->cmd = c->realcmd = c->mstate.commands[j].cmd;
/* ACL permissions are also checked at the time of execution in case
* they were changed after the commands were queued. */
......@@ -240,7 +240,7 @@ void execCommand(client *c) {
c->argv = orig_argv;
c->argv_len = orig_argv_len;
c->argc = orig_argc;
c->cmd = orig_cmd;
c->cmd = c->realcmd = orig_cmd;
discardTransaction(c);
server.in_exec = 0;
......@@ -257,10 +257,13 @@ void execCommand(client *c) {
/* In the client->watched_keys list we need to use watchedKey structures
* as in order to identify a key in Redis we need both the key name and the
* DB */
* DB. This struct is also referenced from db->watched_keys dict, where the
* values are lists of watchedKey pointers. */
typedef struct watchedKey {
robj *key;
redisDb *db;
client *client;
unsigned expired:1; /* Flag that we're watching an already expired key. */
} watchedKey;
/* Watch for the specified key */
......@@ -284,13 +287,15 @@ void watchForKey(client *c, robj *key) {
dictAdd(c->db->watched_keys,key,clients);
incrRefCount(key);
}
listAddNodeTail(clients,c);
/* Add the new key to the list of keys watched by this client */
wk = zmalloc(sizeof(*wk));
wk->key = key;
wk->client = c;
wk->db = c->db;
wk->expired = keyIsExpired(c->db, key);
incrRefCount(key);
listAddNodeTail(c->watched_keys,wk);
listAddNodeTail(clients,wk);
}
/* Unwatch all the keys watched by this client. To clean the EXEC dirty
......@@ -305,12 +310,12 @@ void unwatchAllKeys(client *c) {
list *clients;
watchedKey *wk;
/* Lookup the watched key -> clients list and remove the client
/* Lookup the watched key -> clients list and remove the client's wk
* from the list */
wk = listNodeValue(ln);
clients = dictFetchValue(wk->db->watched_keys, wk->key);
serverAssertWithInfo(c,NULL,clients != NULL);
listDelNode(clients,listSearchKey(clients,c));
listDelNode(clients,listSearchKey(clients,wk));
/* Kill the entry at all if this was the only client */
if (listLength(clients) == 0)
dictDelete(wk->db->watched_keys, wk->key);
......@@ -321,8 +326,8 @@ void unwatchAllKeys(client *c) {
}
}
/* iterates over the watched_keys list and
* look for an expired key . */
/* Iterates over the watched_keys list and looks for an expired key. Keys which
* were expired already when WATCH was called are ignored. */
int isWatchedKeyExpired(client *c) {
listIter li;
listNode *ln;
......@@ -331,6 +336,7 @@ int isWatchedKeyExpired(client *c) {
listRewind(c->watched_keys,&li);
while ((ln = listNext(&li))) {
wk = listNodeValue(ln);
if (wk->expired) continue; /* was expired when WATCH was called */
if (keyIsExpired(wk->db, wk->key)) return 1;
}
......@@ -352,13 +358,31 @@ void touchWatchedKey(redisDb *db, robj *key) {
/* Check if we are already watching for this key */
listRewind(clients,&li);
while((ln = listNext(&li))) {
client *c = listNodeValue(ln);
watchedKey *wk = listNodeValue(ln);
client *c = wk->client;
if (wk->expired) {
/* The key was already expired when WATCH was called. */
if (db == wk->db &&
equalStringObjects(key, wk->key) &&
dictFind(db->dict, key->ptr) == NULL)
{
/* Already expired key is deleted, so logically no change. Clear
* the flag. Deleted keys are not flagged as expired. */
wk->expired = 0;
goto skip_client;
}
break;
}
c->flags |= CLIENT_DIRTY_CAS;
/* As the client is marked as dirty, there is no point in getting here
* again in case that key (or others) are modified again (or keep the
* memory overhead till EXEC). */
unwatchAllKeys(c);
skip_client:
continue;
}
}
......@@ -379,14 +403,31 @@ void touchAllWatchedKeysInDb(redisDb *emptied, redisDb *replaced_with) {
dictIterator *di = dictGetSafeIterator(emptied->watched_keys);
while((de = dictNext(di)) != NULL) {
robj *key = dictGetKey(de);
if (dictFind(emptied->dict, key->ptr) ||
int exists_in_emptied = dictFind(emptied->dict, key->ptr) != NULL;
if (exists_in_emptied ||
(replaced_with && dictFind(replaced_with->dict, key->ptr)))
{
list *clients = dictGetVal(de);
if (!clients) continue;
listRewind(clients,&li);
while((ln = listNext(&li))) {
client *c = listNodeValue(ln);
watchedKey *wk = listNodeValue(ln);
if (wk->expired) {
if (!replaced_with || !dictFind(replaced_with->dict, key->ptr)) {
/* Expired key now deleted. No logical change. Clear the
* flag. Deleted keys are not flagged as expired. */
wk->expired = 0;
continue;
} else if (keyIsExpired(replaced_with, key)) {
/* Expired key remains expired. */
continue;
}
} else if (!exists_in_emptied && keyIsExpired(replaced_with, key)) {
/* Non-existing key is replaced with an expired key. */
wk->expired = 1;
continue;
}
client *c = wk->client;
c->flags |= CLIENT_DIRTY_CAS;
/* As the client is marked as dirty, there is no point in getting here
* again for others keys (or keep the memory overhead till EXEC). */
......
......@@ -131,7 +131,7 @@ client *createClient(connection *conn) {
connSetReadHandler(conn, readQueryFromClient);
connSetPrivateData(conn, c);
}
c->buf = zmalloc(PROTO_REPLY_CHUNK_BYTES);
selectDb(c,0);
uint64_t client_id;
atomicGetIncr(server.next_client_id, client_id, 1);
......@@ -140,7 +140,9 @@ client *createClient(connection *conn) {
c->conn = conn;
c->name = NULL;
c->bufpos = 0;
c->buf_usable_size = zmalloc_usable_size(c)-offsetof(client,buf);
c->buf_usable_size = zmalloc_usable_size(c->buf);
c->buf_peak = c->buf_usable_size;
c->buf_peak_last_reset_time = server.unixtime;
c->ref_repl_buf_node = NULL;
c->ref_block_pos = 0;
c->qb_pos = 0;
......@@ -154,7 +156,7 @@ client *createClient(connection *conn) {
c->argv_len_sum = 0;
c->original_argc = 0;
c->original_argv = NULL;
c->cmd = c->lastcmd = NULL;
c->cmd = c->lastcmd = c->realcmd = NULL;
c->multibulklen = 0;
c->bulklen = -1;
c->sentlen = 0;
......@@ -173,6 +175,7 @@ client *createClient(connection *conn) {
c->slave_capa = SLAVE_CAPA_NONE;
c->slave_req = SLAVE_REQ_NONE;
c->reply = listCreate();
c->deferred_reply_errors = NULL;
c->reply_bytes = 0;
c->obuf_soft_limit_reached_time = 0;
listSetFreeMethod(c->reply,freeClientReplyValue);
......@@ -313,6 +316,9 @@ size_t _addReplyToBuffer(client *c, const char *s, size_t len) {
size_t reply_len = len > available ? available : len;
memcpy(c->buf+c->bufpos,s,reply_len);
c->bufpos+=reply_len;
/* We update the buffer peak after appending the reply to the buffer */
if(c->buf_peak < (size_t)c->bufpos)
c->buf_peak = (size_t)c->bufpos;
return reply_len;
}
......@@ -437,24 +443,46 @@ void addReplyErrorLength(client *c, const char *s, size_t len) {
addReplyProto(c,"\r\n",2);
}
/* Do some actions after an error reply was sent (Log if needed, updates stats, etc.) */
void afterErrorReply(client *c, const char *s, size_t len) {
/* Increment the global error counter */
server.stat_total_error_replies++;
/* Increment the error stats
* If the string already starts with "-..." then the error prefix
* is provided by the caller ( we limit the search to 32 chars). Otherwise we use "-ERR". */
if (s[0] != '-') {
incrementErrorCount("ERR", 3);
} else {
char *spaceloc = memchr(s, ' ', len < 32 ? len : 32);
if (spaceloc) {
const size_t errEndPos = (size_t)(spaceloc - s);
incrementErrorCount(s+1, errEndPos-1);
} else {
/* Fallback to ERR if we can't retrieve the error prefix */
/* Do some actions after an error reply was sent (Log if needed, updates stats, etc.)
* Possible flags:
* * ERR_REPLY_FLAG_NO_STATS_UPDATE - indicate not to update any error stats. */
void afterErrorReply(client *c, const char *s, size_t len, int flags) {
/* Module clients fall into two categories:
* Calls to RM_Call, in which case the error isn't being returned to a client, so should not be counted.
* Module thread safe context calls to RM_ReplyWithError, which will be added to a real client by the main thread later. */
if (c->flags & CLIENT_MODULE) {
if (!c->deferred_reply_errors) {
c->deferred_reply_errors = listCreate();
listSetFreeMethod(c->deferred_reply_errors, (void (*)(void*))sdsfree);
}
listAddNodeTail(c->deferred_reply_errors, sdsnewlen(s, len));
return;
}
if (!(flags & ERR_REPLY_FLAG_NO_STATS_UPDATE)) {
/* Increment the global error counter */
server.stat_total_error_replies++;
/* Increment the error stats
* If the string already starts with "-..." then the error prefix
* is provided by the caller ( we limit the search to 32 chars). Otherwise we use "-ERR". */
if (s[0] != '-') {
incrementErrorCount("ERR", 3);
} else {
char *spaceloc = memchr(s, ' ', len < 32 ? len : 32);
if (spaceloc) {
const size_t errEndPos = (size_t)(spaceloc - s);
incrementErrorCount(s+1, errEndPos-1);
} else {
/* Fallback to ERR if we can't retrieve the error prefix */
incrementErrorCount("ERR", 3);
}
}
} else {
/* stat_total_error_replies will not be updated, which means that
* the cmd stats will not be updated as well, we still want this command
* to be counted as failed so we update it here. We update c->realcmd in
* case c->cmd was changed (like in GEOADD). */
c->realcmd->failed_calls++;
}
/* Sometimes it could be normal that a slave replies to a master with
......@@ -500,7 +528,7 @@ void afterErrorReply(client *c, const char *s, size_t len) {
* Unlike addReplyErrorSds and others alike which rely on addReplyErrorLength. */
void addReplyErrorObject(client *c, robj *err) {
addReply(c, err);
afterErrorReply(c, err->ptr, sdslen(err->ptr)-2); /* Ignore trailing \r\n */
afterErrorReply(c, err->ptr, sdslen(err->ptr)-2, 0); /* Ignore trailing \r\n */
}
/* Sends either a reply or an error reply by checking the first char.
......@@ -521,34 +549,57 @@ void addReplyOrErrorObject(client *c, robj *reply) {
/* See addReplyErrorLength for expectations from the input string. */
void addReplyError(client *c, const char *err) {
addReplyErrorLength(c,err,strlen(err));
afterErrorReply(c,err,strlen(err));
afterErrorReply(c,err,strlen(err),0);
}
/* Add error reply to the given client.
* Supported flags:
* * ERR_REPLY_FLAG_NO_STATS_UPDATE - indicate not to perform any error stats updates */
void addReplyErrorSdsEx(client *c, sds err, int flags) {
addReplyErrorLength(c,err,sdslen(err));
afterErrorReply(c,err,sdslen(err),flags);
sdsfree(err);
}
/* See addReplyErrorLength for expectations from the input string. */
/* As a side effect the SDS string is freed. */
void addReplyErrorSds(client *c, sds err) {
addReplyErrorLength(c,err,sdslen(err));
afterErrorReply(c,err,sdslen(err));
sdsfree(err);
addReplyErrorSdsEx(c, err, 0);
}
/* See addReplyErrorLength for expectations from the formatted string.
* The formatted string is safe to contain \r and \n anywhere. */
void addReplyErrorFormat(client *c, const char *fmt, ...) {
va_list ap;
va_start(ap,fmt);
sds s = sdscatvprintf(sdsempty(),fmt,ap);
va_end(ap);
/* Internal function used by addReplyErrorFormat and addReplyErrorFormatEx.
* Refer to afterErrorReply for more information about the flags. */
static void addReplyErrorFormatInternal(client *c, int flags, const char *fmt, va_list ap) {
va_list cpy;
va_copy(cpy,ap);
sds s = sdscatvprintf(sdsempty(),fmt,cpy);
va_end(cpy);
/* Trim any newlines at the end (ones will be added by addReplyErrorLength) */
s = sdstrim(s, "\r\n");
/* Make sure there are no newlines in the middle of the string, otherwise
* invalid protocol is emitted. */
s = sdsmapchars(s, "\r\n", " ", 2);
addReplyErrorLength(c,s,sdslen(s));
afterErrorReply(c,s,sdslen(s));
afterErrorReply(c,s,sdslen(s),flags);
sdsfree(s);
}
void addReplyErrorFormatEx(client *c, int flags, const char *fmt, ...) {
va_list ap;
va_start(ap,fmt);
addReplyErrorFormatInternal(c, flags, fmt, ap);
va_end(ap);
}
/* See addReplyErrorLength for expectations from the formatted string.
* The formatted string is safe to contain \r and \n anywhere. */
void addReplyErrorFormat(client *c, const char *fmt, ...) {
va_list ap;
va_start(ap,fmt);
addReplyErrorFormatInternal(c, 0, fmt, ap);
va_end(ap);
}
void addReplyErrorArity(client *c) {
addReplyErrorFormat(c, "wrong number of arguments for '%s' command",
c->cmd->fullname);
......@@ -696,6 +747,24 @@ void setDeferredAggregateLen(client *c, void *node, long length, char prefix) {
* we return NULL in addReplyDeferredLen() */
if (node == NULL) return;
/* Things like *2\r\n, %3\r\n or ~4\r\n are emitted very often by the protocol
* so we have a few shared objects to use if the integer is small
* like it is most of the times. */
const size_t hdr_len = OBJ_SHARED_HDR_STRLEN(length);
const int opt_hdr = length < OBJ_SHARED_BULKHDR_LEN;
if (prefix == '*' && opt_hdr) {
setDeferredReply(c, node, shared.mbulkhdr[length]->ptr, hdr_len);
return;
}
if (prefix == '%' && opt_hdr) {
setDeferredReply(c, node, shared.maphdr[length]->ptr, hdr_len);
return;
}
if (prefix == '~' && opt_hdr) {
setDeferredReply(c, node, shared.sethdr[length]->ptr, hdr_len);
return;
}
char lenstr[128];
size_t lenstr_len = sprintf(lenstr, "%c%ld\r\n", prefix, length);
setDeferredReply(c, node, lenstr, lenstr_len);
......@@ -788,11 +857,19 @@ void addReplyLongLongWithPrefix(client *c, long long ll, char prefix) {
/* Things like $3\r\n or *2\r\n are emitted very often by the protocol
* so we have a few shared objects to use if the integer is small
* like it is most of the times. */
if (prefix == '*' && ll < OBJ_SHARED_BULKHDR_LEN && ll >= 0) {
addReply(c,shared.mbulkhdr[ll]);
const int opt_hdr = ll < OBJ_SHARED_BULKHDR_LEN && ll >= 0;
const size_t hdr_len = OBJ_SHARED_HDR_STRLEN(ll);
if (prefix == '*' && opt_hdr) {
addReplyProto(c,shared.mbulkhdr[ll]->ptr,hdr_len);
return;
} else if (prefix == '$' && opt_hdr) {
addReplyProto(c,shared.bulkhdr[ll]->ptr,hdr_len);
return;
} else if (prefix == '%' && opt_hdr) {
addReplyProto(c,shared.maphdr[ll]->ptr,hdr_len);
return;
} else if (prefix == '$' && ll < OBJ_SHARED_BULKHDR_LEN && ll >= 0) {
addReply(c,shared.bulkhdr[ll]);
} else if (prefix == '~' && opt_hdr) {
addReplyProto(c,shared.sethdr[ll]->ptr,hdr_len);
return;
}
......@@ -1024,10 +1101,28 @@ void AddReplyFromClient(client *dst, client *src) {
src->reply_bytes = 0;
src->bufpos = 0;
if (src->deferred_reply_errors) {
deferredAfterErrorReply(dst, src->deferred_reply_errors);
listRelease(src->deferred_reply_errors);
src->deferred_reply_errors = NULL;
}
/* Check output buffer limits */
closeClientOnOutputBufferLimitReached(dst, 1);
}
/* Append the listed errors to the server error statistics. the input
* list is not modified and remains the responsibility of the caller. */
void deferredAfterErrorReply(client *c, list *errors) {
listIter li;
listNode *ln;
listRewind(errors,&li);
while((ln = listNext(&li))) {
sds err = ln->value;
afterErrorReply(c, err, sdslen(err), 0);
}
}
/* Logically copy 'src' replica client buffers info to 'dst' replica.
* Basically increase referenced buffer block node reference count. */
void copyReplicaOutputBuffer(client *dst, client *src) {
......@@ -1494,9 +1589,12 @@ void freeClient(client *c) {
/* Free data structures. */
listRelease(c->reply);
zfree(c->buf);
freeReplicaReferencedReplBuffer(c);
freeClientArgv(c);
freeClientOriginalArgv(c);
if (c->deferred_reply_errors)
listRelease(c->deferred_reply_errors);
/* Unlink the client: this will close the socket, remove the I/O
* handlers, and remove references of the client from different
......@@ -1658,10 +1756,82 @@ client *lookupClientByID(uint64_t id) {
return (c == raxNotFound) ? NULL : c;
}
/* This function should be called from _writeToClient when the reply list is not empty,
* it gathers the scattered buffers from reply list and sends them away with connWritev.
* If we write successfully, it returns C_OK, otherwise, C_ERR is returned,
* and 'nwritten' is an output parameter, it means how many bytes server write
* to client. */
static int _writevToClient(client *c, ssize_t *nwritten) {
struct iovec iov[IOV_MAX];
int iovcnt = 0;
size_t iov_bytes_len = 0;
/* If the static reply buffer is not empty,
* add it to the iov array for writev() as well. */
if (c->bufpos > 0) {
iov[iovcnt].iov_base = c->buf + c->sentlen;
iov[iovcnt].iov_len = c->bufpos - c->sentlen;
iov_bytes_len += iov[iovcnt++].iov_len;
}
/* The first node of reply list might be incomplete from the last call,
* thus it needs to be calibrated to get the actual data address and length. */
size_t offset = c->bufpos > 0 ? 0 : c->sentlen;
listIter iter;
listNode *next;
clientReplyBlock *o;
listRewind(c->reply, &iter);
while ((next = listNext(&iter)) && iovcnt < IOV_MAX && iov_bytes_len < NET_MAX_WRITES_PER_EVENT) {
o = listNodeValue(next);
if (o->used == 0) { /* empty node, just release it and skip. */
c->reply_bytes -= o->size;
listDelNode(c->reply, next);
offset = 0;
continue;
}
iov[iovcnt].iov_base = o->buf + offset;
iov[iovcnt].iov_len = o->used - offset;
iov_bytes_len += iov[iovcnt++].iov_len;
offset = 0;
}
if (iovcnt == 0) return C_OK;
*nwritten = connWritev(c->conn, iov, iovcnt);
if (*nwritten <= 0) return C_ERR;
/* Locate the new node which has leftover data and
* release all nodes in front of it. */
ssize_t remaining = *nwritten;
if (c->bufpos > 0) { /* deal with static reply buffer first. */
int buf_len = c->bufpos - c->sentlen;
c->sentlen += remaining;
/* If the buffer was sent, set bufpos to zero to continue with
* the remainder of the reply. */
if (remaining >= buf_len) {
c->bufpos = 0;
c->sentlen = 0;
}
remaining -= buf_len;
}
listRewind(c->reply, &iter);
while (remaining > 0) {
next = listNext(&iter);
o = listNodeValue(next);
if (remaining < (ssize_t)(o->used - c->sentlen)) {
c->sentlen += remaining;
break;
}
remaining -= (ssize_t)(o->used - c->sentlen);
c->reply_bytes -= o->size;
listDelNode(c->reply, next);
c->sentlen = 0;
}
return C_OK;
}
/* This function does actual writing output buffers to different types of
* clients, it is called by writeToClient.
* If we write successfully, it return C_OK, otherwise, C_ERR is returned,
* And 'nwritten' is a output parameter, it means how many bytes server write
* If we write successfully, it returns C_OK, otherwise, C_ERR is returned,
* and 'nwritten' is an output parameter, it means how many bytes server write
* to client. */
int _writeToClient(client *c, ssize_t *nwritten) {
*nwritten = 0;
......@@ -1690,8 +1860,18 @@ int _writeToClient(client *c, ssize_t *nwritten) {
return C_OK;
}
if (c->bufpos > 0) {
*nwritten = connWrite(c->conn,c->buf+c->sentlen,c->bufpos-c->sentlen);
/* When the reply list is not empty, it's better to use writev to save us some
* system calls and TCP packets. */
if (listLength(c->reply) > 0) {
int ret = _writevToClient(c, nwritten);
if (ret != C_OK) return ret;
/* If there are no longer objects in the list, we expect
* the count of reply bytes to be exactly zero. */
if (listLength(c->reply) == 0)
serverAssert(c->reply_bytes == 0);
} else if (c->bufpos > 0) {
*nwritten = connWrite(c->conn, c->buf + c->sentlen, c->bufpos - c->sentlen);
if (*nwritten <= 0) return C_ERR;
c->sentlen += *nwritten;
......@@ -1701,31 +1881,8 @@ int _writeToClient(client *c, ssize_t *nwritten) {
c->bufpos = 0;
c->sentlen = 0;
}
} else {
clientReplyBlock *o = listNodeValue(listFirst(c->reply));
size_t objlen = o->used;
if (objlen == 0) {
c->reply_bytes -= o->size;
listDelNode(c->reply,listFirst(c->reply));
return C_OK;
}
*nwritten = connWrite(c->conn, o->buf + c->sentlen, objlen - c->sentlen);
if (*nwritten <= 0) return C_ERR;
c->sentlen += *nwritten;
}
/* If we fully sent the object on head go to the next one */
if (c->sentlen == objlen) {
c->reply_bytes -= o->size;
listDelNode(c->reply,listFirst(c->reply));
c->sentlen = 0;
/* If there are no longer objects in the list, we expect
* the count of reply bytes to be exactly zero. */
if (listLength(c->reply) == 0)
serverAssert(c->reply_bytes == 0);
}
}
return C_OK;
}
......@@ -1863,6 +2020,10 @@ void resetClient(client *c) {
c->multibulklen = 0;
c->bulklen = -1;
if (c->deferred_reply_errors)
listRelease(c->deferred_reply_errors);
c->deferred_reply_errors = NULL;
/* We clear the ASKING flag as well if we are not inside a MULTI, and
* if what we just executed is not the ASKING command itself. */
if (!(c->flags & CLIENT_MULTI) && prevcmd != askingCommand)
......@@ -2556,7 +2717,7 @@ sds catClientInfoString(sds s, client *client) {
}
sds ret = sdscatfmt(s,
"id=%U addr=%s laddr=%s %s name=%s age=%I idle=%I flags=%s db=%i sub=%i psub=%i multi=%i qbuf=%U qbuf-free=%U argv-mem=%U multi-mem=%U obl=%U oll=%U omem=%U tot-mem=%U events=%s cmd=%s user=%s redir=%I resp=%i",
"id=%U addr=%s laddr=%s %s name=%s age=%I idle=%I flags=%s db=%i sub=%i psub=%i multi=%i qbuf=%U qbuf-free=%U argv-mem=%U multi-mem=%U rbs=%U rbp=%U obl=%U oll=%U omem=%U tot-mem=%U events=%s cmd=%s user=%s redir=%I resp=%i",
(unsigned long long) client->id,
getClientPeerId(client),
getClientSockname(client),
......@@ -2573,6 +2734,8 @@ sds catClientInfoString(sds s, client *client) {
(unsigned long long) sdsavail(client->querybuf),
(unsigned long long) client->argv_len_sum,
(unsigned long long) client->mstate.argv_len_sums,
(unsigned long long) client->buf_usable_size,
(unsigned long long) client->buf_peak,
(unsigned long long) client->bufpos,
(unsigned long long) listLength(client->reply) + used_blocks_of_repl_buf,
(unsigned long long) obufmem, /* should not include client->buf since we want to see 0 for static clients. */
......@@ -2919,6 +3082,7 @@ NULL
else
replyToBlockedClientTimedOut(target);
unblockClient(target);
updateStatsOnUnblock(target, 0, 0, 1);
addReply(c,shared.cone);
} else {
addReply(c,shared.czero);
......@@ -3414,6 +3578,7 @@ size_t getClientMemoryUsage(client *c, size_t *output_buffer_mem_usage) {
*output_buffer_mem_usage = mem;
mem += sdsZmallocSize(c->querybuf);
mem += zmalloc_size(c);
mem += c->buf_usable_size;
/* For efficiency (less work keeping track of the argv memory), it doesn't include the used memory
* i.e. unused sds space and internal fragmentation, just the string length. but this is enough to
* spot problematic clients. */
......
......@@ -692,7 +692,7 @@ int rdbSaveObjectType(rio *rdb, robj *o) {
else
serverPanic("Unknown hash encoding");
case OBJ_STREAM:
return rdbSaveType(rdb,RDB_TYPE_STREAM_LISTPACKS);
return rdbSaveType(rdb,RDB_TYPE_STREAM_LISTPACKS_2);
case OBJ_MODULE:
return rdbSaveType(rdb,RDB_TYPE_MODULE_2);
default:
......@@ -986,6 +986,19 @@ ssize_t rdbSaveObject(rio *rdb, robj *o, robj *key, int dbid) {
nwritten += n;
if ((n = rdbSaveLen(rdb,s->last_id.seq)) == -1) return -1;
nwritten += n;
/* Save the first entry ID. */
if ((n = rdbSaveLen(rdb,s->first_id.ms)) == -1) return -1;
nwritten += n;
if ((n = rdbSaveLen(rdb,s->first_id.seq)) == -1) return -1;
nwritten += n;
/* Save the maximal tombstone ID. */
if ((n = rdbSaveLen(rdb,s->max_deleted_entry_id.ms)) == -1) return -1;
nwritten += n;
if ((n = rdbSaveLen(rdb,s->max_deleted_entry_id.seq)) == -1) return -1;
nwritten += n;
/* Save the offset. */
if ((n = rdbSaveLen(rdb,s->entries_added)) == -1) return -1;
nwritten += n;
/* The consumer groups and their clients are part of the stream
* type, so serialize every consumer group. */
......@@ -1020,6 +1033,13 @@ ssize_t rdbSaveObject(rio *rdb, robj *o, robj *key, int dbid) {
return -1;
}
nwritten += n;
/* Save the group's logical reads counter. */
if ((n = rdbSaveLen(rdb,cg->entries_read)) == -1) {
raxStop(&ri);
return -1;
}
nwritten += n;
/* Save the global PEL. */
if ((n = rdbSaveStreamPEL(rdb,cg->pel,1)) == -1) {
......@@ -1151,8 +1171,9 @@ ssize_t rdbSaveAuxFieldStrInt(rio *rdb, char *key, long long val) {
/* Save a few default AUX fields with information about the RDB generated. */
int rdbSaveInfoAuxFields(rio *rdb, int rdbflags, rdbSaveInfo *rsi) {
UNUSED(rdbflags);
int redis_bits = (sizeof(void*) == 8) ? 64 : 32;
int aof_preamble = (rdbflags & RDBFLAGS_AOF_PREAMBLE) != 0;
int aof_base = (rdbflags & RDBFLAGS_AOF_PREAMBLE) != 0;
/* Add a few fields about the state when the RDB was created. */
if (rdbSaveAuxFieldStrStr(rdb,"redis-ver",REDIS_VERSION) == -1) return -1;
......@@ -1169,7 +1190,7 @@ int rdbSaveInfoAuxFields(rio *rdb, int rdbflags, rdbSaveInfo *rsi) {
if (rdbSaveAuxFieldStrInt(rdb,"repl-offset",server.master_repl_offset)
== -1) return -1;
}
if (rdbSaveAuxFieldStrInt(rdb,"aof-preamble",aof_preamble) == -1) return -1;
if (rdbSaveAuxFieldStrInt(rdb, "aof-base", aof_base) == -1) return -1;
return 1;
}
......@@ -1470,6 +1491,7 @@ int rdbSaveBackground(int req, char *filename, rdbSaveInfo *rsi) {
pid_t childpid;
if (hasActiveChildProcess()) return C_ERR;
server.stat_rdb_saves++;
server.dirty_before_bgsave = server.dirty;
server.lastbgsave_try = time(NULL);
......@@ -2319,7 +2341,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) {
rdbReportCorruptRDB("Unknown RDB encoding type %d",rdbtype);
break;
}
} else if (rdbtype == RDB_TYPE_STREAM_LISTPACKS) {
} else if (rdbtype == RDB_TYPE_STREAM_LISTPACKS || rdbtype == RDB_TYPE_STREAM_LISTPACKS_2) {
o = createStreamObject();
stream *s = o->ptr;
uint64_t listpacks = rdbLoadLen(rdb,NULL);
......@@ -2395,6 +2417,30 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) {
/* Load the last entry ID. */
s->last_id.ms = rdbLoadLen(rdb,NULL);
s->last_id.seq = rdbLoadLen(rdb,NULL);
if (rdbtype == RDB_TYPE_STREAM_LISTPACKS_2) {
/* Load the first entry ID. */
s->first_id.ms = rdbLoadLen(rdb,NULL);
s->first_id.seq = rdbLoadLen(rdb,NULL);
/* Load the maximal deleted entry ID. */
s->max_deleted_entry_id.ms = rdbLoadLen(rdb,NULL);
s->max_deleted_entry_id.seq = rdbLoadLen(rdb,NULL);
/* Load the offset. */
s->entries_added = rdbLoadLen(rdb,NULL);
} else {
/* During migration the offset can be initialized to the stream's
* length. At this point, we also don't care about tombstones
* because CG offsets will be later initialized as well. */
s->max_deleted_entry_id.ms = 0;
s->max_deleted_entry_id.seq = 0;
s->entries_added = s->length;
/* Since the rax is already loaded, we can find the first entry's
* ID. */
streamGetEdgeID(s,1,1,&s->first_id);
}
if (rioGetReadError(rdb)) {
rdbReportReadError("Stream object metadata loading failed.");
......@@ -2430,8 +2476,22 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) {
decrRefCount(o);
return NULL;
}
/* Load group offset. */
uint64_t cg_offset;
if (rdbtype == RDB_TYPE_STREAM_LISTPACKS_2) {
cg_offset = rdbLoadLen(rdb,NULL);
if (rioGetReadError(rdb)) {
rdbReportReadError("Stream cgroup offset loading failed.");
sdsfree(cgname);
decrRefCount(o);
return NULL;
}
} else {
cg_offset = streamEstimateDistanceFromFirstEverEntry(s,&cg_id);
}
streamCG *cgroup = streamCreateCG(s,cgname,sdslen(cgname),&cg_id);
streamCG *cgroup = streamCreateCG(s,cgname,sdslen(cgname),&cg_id,cg_offset);
if (cgroup == NULL) {
rdbReportCorruptRDB("Duplicated consumer group name %s",
cgname);
......@@ -2962,6 +3022,9 @@ int rdbLoadRioWithLoadingCtx(rio *rdb, int rdbflags, rdbSaveInfo *rsi, rdbLoadin
} else if (!strcasecmp(auxkey->ptr,"aof-preamble")) {
long long haspreamble = strtoll(auxval->ptr,NULL,10);
if (haspreamble) serverLog(LL_NOTICE,"RDB has an AOF tail");
} else if (!strcasecmp(auxkey->ptr, "aof-base")) {
long long isbase = strtoll(auxval->ptr, NULL, 10);
if (isbase) serverLog(LL_NOTICE, "RDB is base AOF");
} else if (!strcasecmp(auxkey->ptr,"redis-bits")) {
/* Just ignored. */
} else {
......@@ -3049,9 +3112,9 @@ int rdbLoadRioWithLoadingCtx(rio *rdb, int rdbflags, rdbSaveInfo *rsi, rdbLoadin
* received from the master. In the latter case, the master is
* responsible for key expiry. If we would expire keys here, the
* snapshot taken by the master may not be reflected on the slave.
* Similarly if the RDB is the preamble of an AOF file, we want to
* load all the keys as they are, since the log of operations later
* assume to work in an exact keyspace state. */
* Similarly, if the base AOF is RDB format, we want to load all
* the keys they are, since the log of operations in the incr AOF
* is assumed to work in the exact keyspace state. */
if (val == NULL) {
/* Since we used to have bug that could lead to empty keys
* (See #8453), we rather not fail when empty key is encountered
......
......@@ -94,10 +94,11 @@
#define RDB_TYPE_HASH_LISTPACK 16
#define RDB_TYPE_ZSET_LISTPACK 17
#define RDB_TYPE_LIST_QUICKLIST_2 18
#define RDB_TYPE_STREAM_LISTPACKS_2 19
/* NOTE: WHEN ADDING NEW RDB TYPE, UPDATE rdbIsObjectType() BELOW */
/* Test if a type is an object type. */
#define rdbIsObjectType(t) ((t >= 0 && t <= 7) || (t >= 9 && t <= 18))
#define rdbIsObjectType(t) ((t >= 0 && t <= 7) || (t >= 9 && t <= 19))
/* Special RDB opcodes (saved/loaded with rdbSaveType/rdbLoadType). */
#define RDB_OPCODE_FUNCTION 246 /* engine data */
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment