Commit 72935b9d authored by Vitaly Arbuzov's avatar Vitaly Arbuzov
Browse files

Merge branch 'unstable' into dict-split-by-slot

parents 6baf20af 6948daca
......@@ -32,6 +32,10 @@
}
}
],
"reply_schema": {
"type": "integer",
"description": "Number of elements removed."
},
"arguments": [
{
"name": "key",
......
......@@ -32,6 +32,10 @@
}
}
],
"reply_schema": {
"type": "integer",
"description": "Number of elements removed."
},
"arguments": [
{
"name": "key",
......
......@@ -32,6 +32,10 @@
}
}
],
"reply_schema": {
"type": "integer",
"description": "Number of elements removed."
},
"arguments": [
{
"name": "key",
......
......@@ -37,6 +37,38 @@
}
}
],
"reply_schema": {
"anyOf": [
{
"description": "List of member elements.",
"type": "array",
"uniqueItems": true,
"items": {
"type": "string"
}
},
{
"description": "List of the members and their scores. Returned in case `WITHSCORES` was used.",
"type": "array",
"uniqueItems": true,
"items": {
"type": "array",
"minItems": 2,
"maxItems": 2,
"items": [
{
"description": "member",
"type": "string"
},
{
"description": "score",
"type": "number"
}
]
}
}
]
},
"arguments": [
{
"name": "key",
......
......@@ -37,6 +37,14 @@
}
}
],
"reply_schema": {
"type": "array",
"description": "List of the elements in the specified score range.",
"uniqueItems": true,
"items": {
"type": "string"
}
},
"arguments": [
{
"name": "key",
......
......@@ -43,6 +43,40 @@
}
}
],
"reply_schema": {
"anyOf": [
{
"type": "array",
"description": "List of the elements in the specified score range, as not WITHSCORES",
"uniqueItems": true,
"items": {
"type": "string",
"description": "Element"
}
},
{
"type": "array",
"description": "List of the elements and their scores in the specified score range, as WITHSCORES used",
"uniqueItems": true,
"items": {
"type": "array",
"description": "Tuple of element and its score",
"minItems": 2,
"maxItems": 2,
"items": [
{
"type": "string",
"description": "element"
},
{
"type": "number",
"description": "score"
}
]
}
}
]
},
"arguments": [
{
"name": "key",
......
......@@ -39,6 +39,32 @@
}
}
],
"reply_schema": {
"oneOf": [
{
"type": "null",
"description": "Key does not exist or the member does not exist in the sorted set."
},
{
"type": "integer",
"description": "The rank of the member when 'WITHSCORES' is not used."
},
{
"type": "array",
"description": "The rank and score of the member when 'WITHSCORES' is used.",
"minItems": 2,
"maxItems": 2,
"items": [
{
"type": "integer"
},
{
"type": "number"
}
]
}
]
},
"arguments": [
{
"name": "key",
......
......@@ -57,6 +57,25 @@
"type": "integer",
"optional": true
}
],
"reply_schema": {
"description": "cursor and scan response in array form",
"type": "array",
"minItems": 2,
"maxItems": 2,
"items": [
{
"description": "cursor",
"type": "string"
},
{
"description": "list of elements of the sorted set, where each even element is the member, and each odd value is its associated score",
"type": "array",
"items": {
"type": "string"
}
}
]
}
}
}
......@@ -33,6 +33,18 @@
}
}
],
"reply_schema": {
"oneOf": [
{
"type": "number",
"description": "The score of the member (a double precision floating point number). In RESP2, this is returned as string."
},
{
"type": "null",
"description": "Member does not exist in the sorted set, or key does not exist."
}
]
},
"arguments": [
{
"name": "key",
......
......@@ -33,6 +33,36 @@
}
}
],
"reply_schema": {
"anyOf": [
{
"description": "The result of union when 'WITHSCORES' is not used.",
"type": "array",
"uniqueItems": true,
"items": {
"type": "string"
}
},
{
"description": "The result of union when 'WITHSCORES' is used.",
"type": "array",
"uniqueItems": true,
"items": {
"type": "array",
"minItems": 2,
"maxItems": 2,
"items": [
{
"type": "string"
},
{
"type": "number"
}
]
}
}
]
},
"arguments": [
{
"name": "numkeys",
......
......@@ -52,6 +52,10 @@
}
}
],
"reply_schema": {
"description": "The number of elements in the resulting sorted set.",
"type": "integer"
},
"arguments": [
{
"name": "destination",
......
......@@ -31,6 +31,7 @@
#include "server.h"
#include "cluster.h"
#include "connection.h"
#include "bio.h"
#include <fcntl.h>
#include <sys/stat.h>
......@@ -2558,6 +2559,17 @@ int updateRequirePass(const char **err) {
return 1;
}
int updateAppendFsync(const char **err) {
UNUSED(err);
if (server.aof_fsync == AOF_FSYNC_ALWAYS) {
/* Wait for all bio jobs related to AOF to drain before proceeding. This prevents a race
* between updates to `fsynced_reploff_pending` done in the main thread and those done on the
* worker thread. */
bioDrainWorker(BIO_AOF_FSYNC);
}
return 1;
}
/* applyBind affects both TCP and TLS (if enabled) together */
static int applyBind(const char **err) {
connListener *tcp_listener = listenerByType(CONN_TYPE_SOCKET);
......@@ -3083,6 +3095,9 @@ standardConfig static_configs[] = {
createStringConfig("proc-title-template", NULL, MODIFIABLE_CONFIG, ALLOW_EMPTY_STRING, server.proc_title_template, CONFIG_DEFAULT_PROC_TITLE_TEMPLATE, isValidProcTitleTemplate, updateProcTitleTemplate),
createStringConfig("bind-source-addr", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.bind_source_addr, NULL, NULL, NULL),
createStringConfig("logfile", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, server.logfile, "", NULL, NULL),
#ifdef LOG_REQ_RES
createStringConfig("req-res-logfile", NULL, IMMUTABLE_CONFIG | HIDDEN_CONFIG, EMPTY_STRING_IS_NULL, server.req_res_logfile, NULL, NULL, NULL),
#endif
createStringConfig("locale-collate", NULL, MODIFIABLE_CONFIG, ALLOW_EMPTY_STRING, server.locale_collate, "", NULL, updateLocaleCollate),
/* SDS Configs */
......@@ -3095,7 +3110,7 @@ standardConfig static_configs[] = {
createEnumConfig("repl-diskless-load", NULL, DEBUG_CONFIG | MODIFIABLE_CONFIG | DENY_LOADING_CONFIG, repl_diskless_load_enum, server.repl_diskless_load, REPL_DISKLESS_LOAD_DISABLED, NULL, NULL),
createEnumConfig("loglevel", NULL, MODIFIABLE_CONFIG, loglevel_enum, server.verbosity, LL_NOTICE, NULL, NULL),
createEnumConfig("maxmemory-policy", NULL, MODIFIABLE_CONFIG, maxmemory_policy_enum, server.maxmemory_policy, MAXMEMORY_NO_EVICTION, NULL, NULL),
createEnumConfig("appendfsync", NULL, MODIFIABLE_CONFIG, aof_fsync_enum, server.aof_fsync, AOF_FSYNC_EVERYSEC, NULL, NULL),
createEnumConfig("appendfsync", NULL, MODIFIABLE_CONFIG, aof_fsync_enum, server.aof_fsync, AOF_FSYNC_EVERYSEC, NULL, updateAppendFsync),
createEnumConfig("oom-score-adj", NULL, MODIFIABLE_CONFIG, oom_score_adj_enum, server.oom_score_adj, OOM_SCORE_ADJ_NO, NULL, updateOOMScoreAdj),
createEnumConfig("acl-pubsub-default", NULL, MODIFIABLE_CONFIG, acl_pubsub_default_enum, server.acl_pubsub_default, 0, NULL, NULL),
createEnumConfig("sanitize-dump-payload", NULL, DEBUG_CONFIG | MODIFIABLE_CONFIG, sanitize_dump_payload_enum, server.sanitize_dump_payload, SANITIZE_DUMP_NO, NULL, NULL),
......@@ -3150,6 +3165,9 @@ standardConfig static_configs[] = {
createUIntConfig("maxclients", NULL, MODIFIABLE_CONFIG, 1, UINT_MAX, server.maxclients, 10000, INTEGER_CONFIG, NULL, updateMaxclients),
createUIntConfig("unixsocketperm", NULL, IMMUTABLE_CONFIG, 0, 0777, server.unixsocketperm, 0, OCTAL_CONFIG, NULL, NULL),
createUIntConfig("socket-mark-id", NULL, IMMUTABLE_CONFIG, 0, UINT_MAX, server.socket_mark_id, 0, INTEGER_CONFIG, NULL, NULL),
#ifdef LOG_REQ_RES
createUIntConfig("client-default-resp", NULL, IMMUTABLE_CONFIG | HIDDEN_CONFIG, 2, 3, server.client_default_resp, 2, INTEGER_CONFIG, NULL, NULL),
#endif
/* Unsigned Long configs */
createULongConfig("active-defrag-max-scan-fields", NULL, MODIFIABLE_CONFIG, 1, LONG_MAX, server.active_defrag_max_scan_fields, 1000, INTEGER_CONFIG, NULL, NULL), /* Default: keys with more than 1000 fields will be processed separately */
......
......@@ -814,9 +814,12 @@ NULL
addReplyError(c,"RESP2 is not supported by this command");
return;
}
uint64_t old_flags = c->flags;
c->flags |= CLIENT_PUSHING;
addReplyPushLen(c,2);
addReplyBulkCString(c,"server-cpu-usage");
addReplyLongLong(c,42);
if (!(old_flags & CLIENT_PUSHING)) c->flags &= ~CLIENT_PUSHING;
/* Push replies are not synchronous replies, so we emit also a
* normal reply in order for blocking clients just discarding the
* push reply, to actually consume the reply and continue. */
......@@ -1863,11 +1866,17 @@ void logCurrentClient(client *cc, const char *title) {
client = catClientInfoString(sdsempty(),cc);
serverLog(LL_WARNING|LL_RAW,"%s\n", client);
sdsfree(client);
serverLog(LL_WARNING|LL_RAW,"argc: '%d'\n", cc->argc);
for (j = 0; j < cc->argc; j++) {
robj *decoded;
decoded = getDecodedObject(cc->argv[j]);
sds repr = sdscatrepr(sdsempty(),decoded->ptr, min(sdslen(decoded->ptr), 128));
serverLog(LL_WARNING|LL_RAW,"argv[%d]: '%s'\n", j, (char*)repr);
if (!strcasecmp(decoded->ptr, "auth") || !strcasecmp(decoded->ptr, "auth2")) {
sdsfree(repr);
decrRefCount(decoded);
break;
}
sdsfree(repr);
decrRefCount(decoded);
}
......
......@@ -144,7 +144,7 @@
* In the example the sparse representation used just 7 bytes instead
* of 12k in order to represent the HLL registers. In general for low
* cardinality there is a big win in terms of space efficiency, traded
* with CPU time since the sparse representation is slower to access:
* with CPU time since the sparse representation is slower to access.
*
* The following table shows average cardinality vs bytes used, 100
* samples per cardinality (when the set was not representable because
......
......@@ -732,7 +732,7 @@ unsigned char *lpFind(unsigned char *lp, unsigned char *p, unsigned char *s,
/* Skip entry */
skipcnt--;
/* Move to next entry, avoid use `lpNext` due to `ASSERT_INTEGRITY` in
/* Move to next entry, avoid use `lpNext` due to `lpAssertValidEntry` in
* `lpNext` will call `lpBytes`, will cause performance degradation */
p = lpSkip(p);
}
......
/*
* Copyright (c) 2021, Redis Ltd.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Redis nor the names of its contributors may be used
* to endorse or promote products derived from this software without
* specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
/* This file implements the interface of logging clients' requests and
* responses into a file.
* This feature needs the LOG_REQ_RES macro to be compiled and is turned
* on by the req-res-logfile config."
*
* Some examples:
*
* PING:
*
* 4
* ping
* 12
* __argv_end__
* +PONG
*
* LRANGE:
*
* 6
* lrange
* 4
* list
* 1
* 0
* 2
* -1
* 12
* __argv_end__
* *1
* $3
* ele
*
* The request is everything up until the __argv_end__ marker.
* The format is:
* <number of characters>
* <the argument>
*
* After __argv_end__ the response appears, and the format is
* RESP (2 or 3, depending on what the client has configured)
*/
#include "server.h"
#include <ctype.h>
#ifdef LOG_REQ_RES
/* ----- Helpers ----- */
static int reqresShouldLog(client *c) {
if (!server.req_res_logfile)
return 0;
/* Ignore client with streaming non-standard response */
if (c->flags & (CLIENT_PUBSUB|CLIENT_MONITOR|CLIENT_SLAVE))
return 0;
/* We only work on masters (didn't implement reqresAppendResponse to work on shared slave buffers) */
if (getClientType(c) == CLIENT_TYPE_MASTER)
return 0;
return 1;
}
static size_t reqresAppendBuffer(client *c, void *buf, size_t len) {
if (!c->reqres.buf) {
c->reqres.capacity = max(len, 1024);
c->reqres.buf = zmalloc(c->reqres.capacity);
} else if (c->reqres.capacity - c->reqres.used < len) {
c->reqres.capacity += len;
c->reqres.buf = zrealloc(c->reqres.buf, c->reqres.capacity);
}
memcpy(c->reqres.buf + c->reqres.used, buf, len);
c->reqres.used += len;
return len;
}
/* Functions for requests */
static size_t reqresAppendArg(client *c, char *arg, size_t arg_len) {
char argv_len_buf[LONG_STR_SIZE];
size_t argv_len_buf_len = ll2string(argv_len_buf,sizeof(argv_len_buf),(long)arg_len);
size_t ret = reqresAppendBuffer(c, argv_len_buf, argv_len_buf_len);
ret += reqresAppendBuffer(c, "\r\n", 2);
ret += reqresAppendBuffer(c, arg, arg_len);
ret += reqresAppendBuffer(c, "\r\n", 2);
return ret;
}
/* ----- API ----- */
/* Zero out the clientReqResInfo struct inside the client,
* and free the buffer if needed */
void reqresReset(client *c, int free_buf) {
if (free_buf && c->reqres.buf)
zfree(c->reqres.buf);
memset(&c->reqres, 0, sizeof(c->reqres));
}
/* Save the offset of the reply buffer (or the reply list).
* Should be called when adding a reply (but it will only save the offset
* on the very first time it's called, because of c->reqres.offset.saved)
* The idea is:
* 1. When a client is executing a command, we save the reply offset.
* 2. During the execution, the reply offset may grow, as addReply* functions are called.
* 3. When client is done with the command (commandProcessed), reqresAppendResponse
* is called.
* 4. reqresAppendResponse will append the diff between the current offset and the one from step (1)
* 5. When client is reset before the next command, we clear c->reqres.offset.saved and start again
*
* We cannot reply on c->sentlen to keep track because it depends on the network
* (reqresAppendResponse will always write the whole buffer, unlike writeToClient)
*
* Ideally, we would just have this code inside reqresAppendRequest, which is called
* from processCommand, but we cannot save the reply offset inside processCommand
* because of the following pipe-lining scenario:
* set rd [redis_deferring_client]
* set buf ""
* append buf "SET key vale\r\n"
* append buf "BLPOP mylist 0\r\n"
* $rd write $buf
* $rd flush
*
* Let's assume we save the reply offset in processCommand
* When BLPOP is processed the offset is 5 (+OK\r\n from the SET)
* Then beforeSleep is called, the +OK is written to network, and bufpos is 0
* When the client is finally unblocked, the cached offset is 5, but bufpos is already
* 0, so we would miss the first 5 bytes of the reply.
**/
void reqresSaveClientReplyOffset(client *c) {
if (!reqresShouldLog(c))
return;
if (c->reqres.offset.saved)
return;
c->reqres.offset.saved = 1;
c->reqres.offset.bufpos = c->bufpos;
if (listLength(c->reply) && listNodeValue(listLast(c->reply))) {
c->reqres.offset.last_node.index = listLength(c->reply) - 1;
c->reqres.offset.last_node.used = ((clientReplyBlock *)listNodeValue(listLast(c->reply)))->used;
} else {
c->reqres.offset.last_node.index = 0;
c->reqres.offset.last_node.used = 0;
}
}
size_t reqresAppendRequest(client *c) {
robj **argv = c->argv;
int argc = c->argc;
serverAssert(argc);
if (!reqresShouldLog(c))
return 0;
/* Ignore commands that have streaming non-standard response */
sds cmd = argv[0]->ptr;
if (!strcasecmp(cmd,"sync") ||
!strcasecmp(cmd,"psync") ||
!strcasecmp(cmd,"monitor") ||
!strcasecmp(cmd,"subscribe") ||
!strcasecmp(cmd,"unsubscribe") ||
!strcasecmp(cmd,"ssubscribe") ||
!strcasecmp(cmd,"sunsubscribe") ||
!strcasecmp(cmd,"psubscribe") ||
!strcasecmp(cmd,"punsubscribe") ||
!strcasecmp(cmd,"debug") ||
!strcasecmp(cmd,"pfdebug") ||
!strcasecmp(cmd,"lolwut") ||
(!strcasecmp(cmd,"sentinel") && argc > 1 && !strcasecmp(argv[1]->ptr,"debug")))
{
return 0;
}
c->reqres.argv_logged = 1;
size_t ret = 0;
for (int i = 0; i < argc; i++) {
if (sdsEncodedObject(argv[i])) {
ret += reqresAppendArg(c, argv[i]->ptr, sdslen(argv[i]->ptr));
} else if (argv[i]->encoding == OBJ_ENCODING_INT) {
char buf[LONG_STR_SIZE];
size_t len = ll2string(buf,sizeof(buf),(long)argv[i]->ptr);
ret += reqresAppendArg(c, buf, len);
} else {
serverPanic("Wrong encoding in reqresAppendRequest()");
}
}
return ret + reqresAppendArg(c, "__argv_end__", 12);
}
size_t reqresAppendResponse(client *c) {
size_t ret = 0;
if (!reqresShouldLog(c))
return 0;
if (!c->reqres.argv_logged) /* Example: UNSUBSCRIBE */
return 0;
if (!c->reqres.offset.saved) /* Example: module client blocked on keys + CLIENT KILL */
return 0;
/* First append the static reply buffer */
if (c->bufpos > c->reqres.offset.bufpos) {
size_t written = reqresAppendBuffer(c, c->buf + c->reqres.offset.bufpos, c->bufpos - c->reqres.offset.bufpos);
ret += written;
}
int curr_index = 0;
size_t curr_used = 0;
if (listLength(c->reply)) {
curr_index = listLength(c->reply) - 1;
curr_used = ((clientReplyBlock *)listNodeValue(listLast(c->reply)))->used;
}
/* Now, append reply bytes from the reply list */
if (curr_index > c->reqres.offset.last_node.index ||
curr_used > c->reqres.offset.last_node.used)
{
int i = 0;
listIter iter;
listNode *curr;
clientReplyBlock *o;
listRewind(c->reply, &iter);
while ((curr = listNext(&iter)) != NULL) {
size_t written;
/* Skip nodes we had already processed */
if (i < c->reqres.offset.last_node.index) {
i++;
continue;
}
o = listNodeValue(curr);
if (o->used == 0) {
i++;
continue;
}
if (i == c->reqres.offset.last_node.index) {
/* Write the potentially incomplete node, which had data from
* before the current command started */
written = reqresAppendBuffer(c,
o->buf + c->reqres.offset.last_node.used,
o->used - c->reqres.offset.last_node.used);
} else {
/* New node */
written = reqresAppendBuffer(c, o->buf, o->used);
}
ret += written;
i++;
}
}
serverAssert(ret);
/* Flush both request and response to file */
FILE *fp = fopen(server.req_res_logfile, "a");
serverAssert(fp);
fwrite(c->reqres.buf, c->reqres.used, 1, fp);
fclose(fp);
return ret;
}
#else /* #ifdef LOG_REQ_RES */
/* Just mimic the API without doing anything */
void reqresReset(client *c, int free_buf) {
UNUSED(c);
UNUSED(free_buf);
}
inline void reqresSaveClientReplyOffset(client *c) {
UNUSED(c);
}
inline size_t reqresAppendRequest(client *c) {
UNUSED(c);
return 0;
}
inline size_t reqresAppendResponse(client *c) {
UNUSED(c);
return 0;
}
#endif /* #ifdef LOG_REQ_RES */
......@@ -229,6 +229,7 @@ struct RedisModuleKey {
* a Redis module. */
struct RedisModuleBlockedClient;
typedef int (*RedisModuleCmdFunc) (RedisModuleCtx *ctx, void **argv, int argc);
typedef int (*RedisModuleAuthCallback)(RedisModuleCtx *ctx, void *username, void *password, RedisModuleString **err);
typedef void (*RedisModuleDisconnectFunc) (RedisModuleCtx *ctx, struct RedisModuleBlockedClient *bc);
 
/* This struct holds the information about a command registered by a module.*/
......@@ -249,6 +250,12 @@ typedef struct RedisModuleCommand RedisModuleCommand;
* only the type, proto and protolen are filled. */
typedef struct CallReply RedisModuleCallReply;
 
/* Structure to hold the module auth callback & the Module implementing it. */
typedef struct RedisModuleAuthCtx {
struct RedisModule *module;
RedisModuleAuthCallback auth_cb;
} RedisModuleAuthCtx;
/* Structure representing a blocked client. We get a pointer to such
* an object when blocking from modules. */
typedef struct RedisModuleBlockedClient {
......@@ -256,6 +263,8 @@ typedef struct RedisModuleBlockedClient {
was destroyed during the life of this object. */
RedisModule *module; /* Module blocking the client. */
RedisModuleCmdFunc reply_callback; /* Reply callback on normal completion.*/
RedisModuleAuthCallback auth_reply_cb; /* Reply callback on completing blocking
module authentication. */
RedisModuleCmdFunc timeout_callback; /* Reply callback on timeout. */
RedisModuleDisconnectFunc disconnect_callback; /* Called on disconnection.*/
void (*free_privdata)(RedisModuleCtx*,void*);/* privdata cleanup callback.*/
......@@ -274,6 +283,11 @@ typedef struct RedisModuleBlockedClient {
Used for measuring latency of blocking cmds */
} RedisModuleBlockedClient;
 
/* This is a list of Module Auth Contexts. Each time a Module registers a callback, a new ctx is
* added to this list. Multiple modules can register auth callbacks and the same Module can have
* multiple auth callbacks. */
static list *moduleAuthCallbacks;
static pthread_mutex_t moduleUnblockedClientsMutex = PTHREAD_MUTEX_INITIALIZER;
static list *moduleUnblockedClients;
 
......@@ -380,6 +394,7 @@ typedef struct RedisModuleServerInfoData {
#define REDISMODULE_ARGV_CALL_REPLIES_AS_ERRORS (1<<8)
#define REDISMODULE_ARGV_RESPECT_DENY_OOM (1<<9)
#define REDISMODULE_ARGV_DRY_RUN (1<<10)
#define REDISMODULE_ARGV_ALLOW_BLOCK (1<<11)
 
/* Determine whether Redis should signalModifiedKey implicitly.
* In case 'ctx' has no 'module' member (and therefore no module->options),
......@@ -455,6 +470,15 @@ struct ModuleConfig {
RedisModule *module;
};
 
typedef struct RedisModuleAsyncRMCallPromise{
size_t ref_count;
void *private_data;
RedisModule *module;
RedisModuleOnUnblocked on_unblocked;
client *c;
RedisModuleCtx *ctx;
} RedisModuleAsyncRMCallPromise;
/* --------------------------------------------------------------------------
* Prototypes
* -------------------------------------------------------------------------- */
......@@ -478,7 +502,7 @@ static struct redisCommandArg *moduleCopyCommandArgs(RedisModuleCommandArg *args
const RedisModuleCommandInfoVersion *version);
static redisCommandArgType moduleConvertArgType(RedisModuleCommandArgType type, int *error);
static int moduleConvertArgFlags(int flags);
void moduleCreateContext(RedisModuleCtx *out_ctx, RedisModule *module, int ctx_flags);
/* --------------------------------------------------------------------------
* ## Heap allocation raw functions
*
......@@ -605,6 +629,16 @@ client *moduleAllocTempClient(user *user) {
return c;
}
 
static void freeRedisModuleAsyncRMCallPromise(RedisModuleAsyncRMCallPromise *promise) {
if (--promise->ref_count > 0) {
return;
}
/* When the promise is finally freed it can not have a client attached to it.
* Either releasing the client or RM_CallReplyPromiseAbort would have removed it. */
serverAssert(!promise->c);
zfree(promise);
}
void moduleReleaseTempClient(client *c) {
if (moduleTempClientCount == moduleTempClientCap) {
moduleTempClientCap = moduleTempClientCap ? moduleTempClientCap*2 : 32;
......@@ -618,6 +652,12 @@ void moduleReleaseTempClient(client *c) {
c->flags = CLIENT_MODULE;
c->user = NULL; /* Root user */
c->cmd = c->lastcmd = c->realcmd = NULL;
if (c->bstate.async_rm_call_handle) {
RedisModuleAsyncRMCallPromise *promise = c->bstate.async_rm_call_handle;
promise->c = NULL; /* Remove the client from the promise so it will no longer be possible to abort it. */
freeRedisModuleAsyncRMCallPromise(promise);
c->bstate.async_rm_call_handle = NULL;
}
moduleTempClients[moduleTempClientCount++] = c;
}
 
......@@ -757,7 +797,7 @@ void modulePostExecutionUnitOperations() {
void moduleFreeContext(RedisModuleCtx *ctx) {
/* See comment in moduleCreateContext */
if (!(ctx->flags & (REDISMODULE_CTX_THREAD_SAFE|REDISMODULE_CTX_COMMAND))) {
server.execution_nesting--;
exitExecutionUnit();
postExecutionUnitOperations();
}
autoMemoryCollect(ctx);
......@@ -782,6 +822,42 @@ void moduleFreeContext(RedisModuleCtx *ctx) {
freeClient(ctx->client);
}
 
static CallReply *moduleParseReply(client *c, RedisModuleCtx *ctx) {
/* Convert the result of the Redis command into a module reply. */
sds proto = sdsnewlen(c->buf,c->bufpos);
c->bufpos = 0;
while(listLength(c->reply)) {
clientReplyBlock *o = listNodeValue(listFirst(c->reply));
proto = sdscatlen(proto,o->buf,o->used);
listDelNode(c->reply,listFirst(c->reply));
}
CallReply *reply = callReplyCreate(proto, c->deferred_reply_errors, ctx);
c->deferred_reply_errors = NULL; /* now the responsibility of the reply object. */
return reply;
}
void moduleCallCommandUnblockedHandler(client *c) {
RedisModuleCtx ctx;
RedisModuleAsyncRMCallPromise *promise = c->bstate.async_rm_call_handle;
serverAssert(promise);
RedisModule *module = promise->module;
if (!promise->on_unblocked) {
moduleReleaseTempClient(c);
return; /* module did not set any unblock callback. */
}
moduleCreateContext(&ctx, module, REDISMODULE_CTX_TEMP_CLIENT);
selectDb(ctx.client, c->db->id);
CallReply *reply = moduleParseReply(c, &ctx);
module->in_call++;
promise->on_unblocked(&ctx, reply, promise->private_data);
module->in_call--;
moduleFreeContext(&ctx);
moduleReleaseTempClient(c);
}
/* Create a module ctx and keep track of the nesting level.
*
* Note: When creating ctx for threads (RM_GetThreadSafeContext and
......@@ -818,7 +894,7 @@ void moduleCreateContext(RedisModuleCtx *out_ctx, RedisModule *module, int ctx_f
* 2. If we are running in a thread (execution_nesting will be dealt with
* when locking/unlocking the GIL) */
if (!(ctx_flags & (REDISMODULE_CTX_THREAD_SAFE|REDISMODULE_CTX_COMMAND))) {
server.execution_nesting++;
enterExecutionUnit(1, 0);
}
}
 
......@@ -1076,6 +1152,7 @@ RedisModuleCommand *moduleCreateCommandProxy(struct RedisModule *module, sds dec
* convention.
*
* The function returns REDISMODULE_ERR in these cases:
* - If creation of module command is called outside the RedisModule_OnLoad.
* - The specified command is already busy.
* - The command name contains some chars that are not allowed.
* - A set of invalid flags were passed.
......@@ -1164,8 +1241,11 @@ RedisModuleCommand *moduleCreateCommandProxy(struct RedisModule *module, sds dec
* NOTE: The scheme described above serves a limited purpose and can
* only be used to find keys that exist at constant indices.
* For non-trivial key arguments, you may pass 0,0,0 and use
* RedisModule_SetCommandInfo to set key specs using a more advanced scheme. */
* RedisModule_SetCommandInfo to set key specs using a more advanced scheme and use
* RedisModule_SetCommandACLCategories to set Redis ACL categories of the commands. */
int RM_CreateCommand(RedisModuleCtx *ctx, const char *name, RedisModuleCmdFunc cmdfunc, const char *strflags, int firstkey, int lastkey, int keystep) {
if (!ctx->module->onload)
return REDISMODULE_ERR;
int64_t flags = strflags ? commandFlagsFromString((char*)strflags) : 0;
if (flags == -1) return REDISMODULE_ERR;
if ((flags & CMD_MODULE_NO_CLUSTER) && server.cluster_enabled)
......@@ -1285,8 +1365,11 @@ RedisModuleCommand *RM_GetCommand(RedisModuleCtx *ctx, const char *name) {
* * `parent` is already a subcommand (we do not allow more than one level of command nesting)
* * `parent` is a command with an implementation (RedisModuleCmdFunc) (A parent command should be a pure container of subcommands)
* * `parent` already has a subcommand called `name`
* * Creating a subcommand is called outside of RedisModule_OnLoad.
*/
int RM_CreateSubcommand(RedisModuleCommand *parent, const char *name, RedisModuleCmdFunc cmdfunc, const char *strflags, int firstkey, int lastkey, int keystep) {
if (!parent->module->onload)
return REDISMODULE_ERR;
int64_t flags = strflags ? commandFlagsFromString((char*)strflags) : 0;
if (flags == -1) return REDISMODULE_ERR;
if ((flags & CMD_MODULE_NO_CLUSTER) && server.cluster_enabled)
......@@ -1341,6 +1424,63 @@ moduleCmdArgAt(const RedisModuleCommandInfoVersion *version,
return (RedisModuleCommandArg *)((char *)(args) + offset);
}
 
/* Helper for categoryFlagsFromString(). Attempts to find an acl flag representing the provided flag string
* and adds that flag to acl_categories_flags if a match is found.
*
* Returns '1' if acl category flag is recognized or
* returns '0' if not recognized */
int matchAclCategoryFlag(char *flag, int64_t *acl_categories_flags) {
uint64_t this_flag = ACLGetCommandCategoryFlagByName(flag);
if (this_flag) {
*acl_categories_flags |= (int64_t) this_flag;
return 1;
}
return 0; /* Unrecognized */
}
/* Helper for RM_SetCommandACLCategories(). Turns a string representing acl category
* flags into the acl category flags used by Redis ACL which allows users to access
* the module commands by acl categories.
*
* It returns the set of acl flags, or -1 if unknown flags are found. */
int64_t categoryFlagsFromString(char *aclflags) {
int count, j;
int64_t acl_categories_flags = 0;
sds *tokens = sdssplitlen(aclflags,strlen(aclflags)," ",1,&count);
for (j = 0; j < count; j++) {
char *t = tokens[j];
if (!matchAclCategoryFlag(t, &acl_categories_flags)) {
serverLog(LL_WARNING,"Unrecognized categories flag %s on module load", t);
break;
}
}
sdsfreesplitres(tokens,count);
if (j != count) return -1; /* Some token not processed correctly. */
return acl_categories_flags;
}
/* RedisModule_SetCommandACLCategories can be used to set ACL categories to module
* commands and subcommands. The set of ACL categories should be passed as
* a space separated C string 'aclflags'.
*
* Example, the acl flags 'write slow' marks the command as part of the write and
* slow ACL categories.
*
* On success REDISMODULE_OK is returned. On error REDISMODULE_ERR is returned.
*
* This function can only be called during the RedisModule_OnLoad function. If called
* outside of this function, an error is returned.
*/
int RM_SetCommandACLCategories(RedisModuleCommand *command, const char *aclflags) {
if (!command || !command->module || !command->module->onload) return REDISMODULE_ERR;
int64_t categories_flags = aclflags ? categoryFlagsFromString((char*)aclflags) : 0;
if (categories_flags == -1) return REDISMODULE_ERR;
struct redisCommand *rcmd = command->rediscmd;
rcmd->acl_categories = categories_flags; /* ACL categories flags for module command */
command->module->num_commands_with_acl_categories++;
return REDISMODULE_OK;
}
/* Set additional command information.
*
* Affects the output of `COMMAND`, `COMMAND INFO` and `COMMAND DOCS`, Cluster,
......@@ -2092,6 +2232,8 @@ void RM_SetModuleAttribs(RedisModuleCtx *ctx, const char *name, int ver, int api
module->info_cb = 0;
module->defrag_cb = 0;
module->loadmod = NULL;
module->num_commands_with_acl_categories = 0;
module->onload = 1;
ctx->module = module;
}
 
......@@ -2576,30 +2718,30 @@ RedisModuleString* RM_HoldString(RedisModuleCtx *ctx, RedisModuleString *str) {
if (ctx != NULL) {
/*
* Put the str in the auto memory management of the ctx.
         * It might already be there, in this case, the ref count will
         * be 2 and we will decrease the ref count twice and free the
         * object in the auto memory free function.
         *
         * Why we can not do the same trick of just remove the object
         * from the auto memory (like in RM_RetainString)?
         * This code shows the issue:
         *
         * RM_AutoMemory(ctx);
         * str1 = RM_CreateString(ctx, "test", 4);
         * str2 = RM_HoldString(ctx, str1);
         * RM_FreeString(str1);
         * RM_FreeString(str2);
         *
         * If after the RM_HoldString we would just remove the string from
         * the auto memory, this example will cause access to a freed memory
         * on 'RM_FreeString(str2);' because the String will be free
         * on 'RM_FreeString(str1);'.
         *
         * So it's safer to just increase the ref count
         * and add the String to auto memory again.
         *
         * The limitation is that it is not possible to use RedisModule_StringAppendBuffer
         * on the String.
* It might already be there, in this case, the ref count will
* be 2 and we will decrease the ref count twice and free the
* object in the auto memory free function.
*
* Why we can not do the same trick of just remove the object
* from the auto memory (like in RM_RetainString)?
* This code shows the issue:
*
* RM_AutoMemory(ctx);
* str1 = RM_CreateString(ctx, "test", 4);
* str2 = RM_HoldString(ctx, str1);
* RM_FreeString(str1);
* RM_FreeString(str2);
*
* If after the RM_HoldString we would just remove the string from
* the auto memory, this example will cause access to a freed memory
* on 'RM_FreeString(str2);' because the String will be free
* on 'RM_FreeString(str1);'.
*
* So it's safer to just increase the ref count
* and add the String to auto memory again.
*
* The limitation is that it is not possible to use RedisModule_StringAppendBuffer
* on the String.
*/
autoMemoryAdd(ctx,REDISMODULE_AM_STRING,str);
}
......@@ -3521,7 +3663,7 @@ int RM_SetClientNameById(uint64_t id, RedisModuleString *name) {
errno = ENOENT;
return REDISMODULE_ERR;
}
if (clientSetName(client, name) == C_ERR) {
if (clientSetName(client, name, NULL) == C_ERR) {
errno = EINVAL;
return REDISMODULE_ERR;
}
......@@ -5618,9 +5760,20 @@ void RM_FreeCallReply(RedisModuleCallReply *reply) {
/* This is a wrapper for the recursive free reply function. This is needed
* in order to have the first level function to return on nested replies,
* but only if called by the module API. */
RedisModuleCtx *ctx = callReplyGetPrivateData(reply);
RedisModuleCtx *ctx = NULL;
if(callReplyType(reply) == REDISMODULE_REPLY_PROMISE) {
RedisModuleAsyncRMCallPromise *promise = callReplyGetPrivateData(reply);
ctx = promise->ctx;
freeRedisModuleAsyncRMCallPromise(promise);
} else {
ctx = callReplyGetPrivateData(reply);
}
freeCallReply(reply);
if (ctx) {
autoMemoryFreed(ctx,REDISMODULE_AM_REPLY,reply);
}
}
 
/* Return the reply type as one of the following:
......@@ -5637,7 +5790,8 @@ void RM_FreeCallReply(RedisModuleCallReply *reply) {
* - REDISMODULE_REPLY_DOUBLE
* - REDISMODULE_REPLY_BIG_NUMBER
* - REDISMODULE_REPLY_VERBATIM_STRING
* - REDISMODULE_REPLY_ATTRIBUTE */
* - REDISMODULE_REPLY_ATTRIBUTE
* - REDISMODULE_REPLY_PROMISE */
int RM_CallReplyType(RedisModuleCallReply *reply) {
return callReplyType(reply);
}
......@@ -5720,6 +5874,39 @@ int RM_CallReplyAttributeElement(RedisModuleCallReply *reply, size_t idx, RedisM
return REDISMODULE_ERR;
}
 
/* Set unblock handler (callback and private data) on the given promise RedisModuleCallReply.
* The given reply must be of promise type (REDISMODULE_REPLY_PROMISE). */
void RM_CallReplyPromiseSetUnblockHandler(RedisModuleCallReply *reply, RedisModuleOnUnblocked on_unblock, void *private_data) {
RedisModuleAsyncRMCallPromise *promise = callReplyGetPrivateData(reply);
promise->on_unblocked = on_unblock;
promise->private_data = private_data;
}
/* Abort the execution of a given promise RedisModuleCallReply.
* return REDMODULE_OK in case the abort was done successfully and REDISMODULE_ERR
* if its not possible to abort the execution (execution already finished).
* In case the execution was aborted (REDMODULE_OK was returned), the private_data out parameter
* will be set with the value of the private data that was given on 'RM_CallReplyPromiseSetUnblockHandler'
* so the caller will be able to release the private data.
*
* If the execution was aborted successfully, it is promised that the unblock handler will not be called.
* That said, it is possible that the abort operation will successes but the operation will still continue.
* This can happened if, for example, a module implements some blocking command and does not respect the
* disconnect callback. For pure Redis commands this can not happened.*/
int RM_CallReplyPromiseAbort(RedisModuleCallReply *reply, void **private_data) {
RedisModuleAsyncRMCallPromise *promise = callReplyGetPrivateData(reply);
if (!promise->c) return REDISMODULE_ERR; /* Promise can not be aborted, either already aborted or already finished. */
if (!(promise->c->flags & CLIENT_BLOCKED)) return REDISMODULE_ERR; /* Client is not blocked anymore, can not abort it. */
/* Client is still blocked, remove it from any blocking state and release it. */
if (private_data) *private_data = promise->private_data;
promise->private_data = NULL;
promise->on_unblocked = NULL;
unblockClient(promise->c, 0);
moduleReleaseTempClient(promise->c);
return REDISMODULE_OK;
}
/* Return the pointer and length of a string or error reply. */
const char *RM_CallReplyStringPtr(RedisModuleCallReply *reply, size_t *len) {
size_t private_len;
......@@ -5767,6 +5954,7 @@ void RM_SetContextUser(RedisModuleCtx *ctx, const RedisModuleUser *user) {
* "0" -> REDISMODULE_ARGV_RESP_AUTO
* "C" -> REDISMODULE_ARGV_RUN_AS_USER
* "M" -> REDISMODULE_ARGV_RESPECT_DENY_OOM
* "K" -> REDISMODULE_ARGV_ALLOW_BLOCK
*
* On error (format specifier error) NULL is returned and nothing is
* allocated. On success the argument vector is returned. */
......@@ -5841,6 +6029,8 @@ robj **moduleCreateArgvFromUserFormat(const char *cmdname, const char *fmt, int
if (flags) (*flags) |= REDISMODULE_ARGV_CALL_REPLIES_AS_ERRORS;
} else if (*p == 'D') {
if (flags) (*flags) |= (REDISMODULE_ARGV_DRY_RUN | REDISMODULE_ARGV_CALL_REPLIES_AS_ERRORS);
} else if (*p == 'K') {
if (flags) (*flags) |= REDISMODULE_ARGV_ALLOW_BLOCK;
} else {
goto fmterr;
}
......@@ -5905,6 +6095,34 @@ fmterr:
* If everything succeeded, it will return with a NULL, otherwise it will
* return with a CallReply object denoting the error, as if it was called with
* the 'E' code.
* * 'K' -- Allow running blocking commands. If enabled and the command gets blocked, a
* special REDISMODULE_REPLY_PROMISE will be returned. This reply type
* indicates that the command was blocked and the reply will be given asynchronously.
* The module can use this reply object to set a handler which will be called when
* the command gets unblocked using RedisModule_CallReplyPromiseSetUnblockHandler.
* The handler must be set immediately after the command invocation (without releasing
* the Redis lock in between). If the handler is not set, the blocking command will
* still continue its execution but the reply will be ignored (fire and forget),
* notice that this is dangerous in case of role change, as explained below.
* The module can use RedisModule_CallReplyPromiseAbort to abort the command invocation
* if it was not yet finished (see RedisModule_CallReplyPromiseAbort documentation for more
* details). It is also the module's responsibility to abort the execution on role change, either by using
* server event (to get notified when the instance becomes a replica) or relying on the disconnect
* callback of the original client. Failing to do so can result in a write operation on a replica.
* Unlike other call replies, promise call reply **must** be freed while the Redis GIL is locked.
* Notice that on unblocking, the only promise is that the unblock handler will be called,
* If the blocking RM_Call caused the module to also block some real client (using RM_BlockClient),
* it is the module responsibility to unblock this client on the unblock handler.
* On the unblock handler it is only allowed to perform the following:
* * Calling additional Redis commands using RM_Call
* * Open keys using RM_OpenKey
* * Replicate data to the replica or AOF
*
* Specifically, it is not allowed to call any Redis module API which are client related such as:
* * RM_Reply* API's
* * RM_BlockClient
* * RM_GetCurrentUserName
*
* * **...**: The actual arguments to the Redis command.
*
* On success a RedisModuleCallReply object is returned, otherwise
......@@ -5964,8 +6182,10 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch
 
c = moduleAllocTempClient(user);
 
if (!(flags & REDISMODULE_ARGV_ALLOW_BLOCK)) {
/* We do not want to allow block, the module do not expect it */
c->flags |= CLIENT_DENY_BLOCKING;
}
c->db = ctx->client->db;
c->argv = argv;
/* We have to assign argv_len, which is equal to argc in that case (RM_Call)
......@@ -6198,24 +6418,39 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch
call(c,call_flags);
server.replication_allowed = prev_replication_allowed;
 
serverAssert((c->flags & CLIENT_BLOCKED) == 0);
/* Convert the result of the Redis command into a module reply. */
sds proto = sdsnewlen(c->buf,c->bufpos);
c->bufpos = 0;
while(listLength(c->reply)) {
clientReplyBlock *o = listNodeValue(listFirst(c->reply));
proto = sdscatlen(proto,o->buf,o->used);
listDelNode(c->reply,listFirst(c->reply));
if (c->flags & CLIENT_BLOCKED) {
serverAssert(flags & REDISMODULE_ARGV_ALLOW_BLOCK);
serverAssert(ctx->module);
RedisModuleAsyncRMCallPromise *promise = zmalloc(sizeof(RedisModuleAsyncRMCallPromise));
*promise = (RedisModuleAsyncRMCallPromise) {
/* We start with ref_count value of 2 because this object is held
* by the promise CallReply and the fake client that was used to execute the command. */
.ref_count = 2,
.module = ctx->module,
.on_unblocked = NULL,
.private_data = NULL,
.c = c,
.ctx = (ctx->flags & REDISMODULE_CTX_AUTO_MEMORY) ? ctx : NULL,
};
reply = callReplyCreatePromise(promise);
c->bstate.async_rm_call_handle = promise;
if (!(call_flags & CMD_CALL_PROPAGATE_AOF)) {
/* No need for AOF propagation, set the relevant flags of the client */
c->flags |= CLIENT_MODULE_PREVENT_AOF_PROP;
}
if (!(call_flags & CMD_CALL_PROPAGATE_REPL)) {
/* No need for replication propagation, set the relevant flags of the client */
c->flags |= CLIENT_MODULE_PREVENT_REPL_PROP;
}
c = NULL; /* Make sure not to free the client */
} else {
reply = moduleParseReply(c, (ctx->flags & REDISMODULE_CTX_AUTO_MEMORY) ? ctx : NULL);
}
reply = callReplyCreate(proto, c->deferred_reply_errors, ctx);
c->deferred_reply_errors = NULL; /* now the responsibility of the reply object. */
 
cleanup:
if (reply) autoMemoryAdd(ctx,REDISMODULE_AM_REPLY,reply);
if (ctx->module) ctx->module->in_call--;
moduleReleaseTempClient(c);
if (c) moduleReleaseTempClient(c);
return reply;
}
 
......@@ -6513,8 +6748,9 @@ robj *moduleTypeDupOrReply(client *c, robj *fromkey, robj *tokey, int todb, robj
* Note: the module name "AAAAAAAAA" is reserved and produces an error, it
* happens to be pretty lame as well.
*
* If there is already a module registering a type with the same name,
* and if the module name or encver is invalid, NULL is returned.
* If RedisModule_CreateDataType() is called outside of RedisModule_OnLoad() function,
* there is already a module registering a type with the same name,
* or if the module name or encver is invalid, NULL is returned.
* Otherwise the new type is registered into Redis, and a reference of
* type RedisModuleType is returned: the caller of the function should store
* this reference into a global variable to make future use of it in the
......@@ -6529,6 +6765,8 @@ robj *moduleTypeDupOrReply(client *c, robj *fromkey, robj *tokey, int todb, robj
* }
*/
moduleType *RM_CreateDataType(RedisModuleCtx *ctx, const char *name, int encver, void *typemethods_ptr) {
if (!ctx->module->onload)
return NULL;
uint64_t id = moduleTypeEncodeId(name,encver);
if (id == 0) return NULL;
if (moduleTypeLookupModuleByName(name) != NULL) return NULL;
......@@ -7369,6 +7607,7 @@ void unblockClientFromModule(client *c) {
*
*/
RedisModuleBlockedClient *moduleBlockClient(RedisModuleCtx *ctx, RedisModuleCmdFunc reply_callback,
RedisModuleAuthCallback auth_reply_callback,
RedisModuleCmdFunc timeout_callback, void (*free_privdata)(RedisModuleCtx*,void*),
long long timeout_ms, RedisModuleString **keys, int numkeys, void *privdata,
int flags) {
......@@ -7388,6 +7627,7 @@ RedisModuleBlockedClient *moduleBlockClient(RedisModuleCtx *ctx, RedisModuleCmdF
bc->client = (islua || ismulti) ? NULL : c;
bc->module = ctx->module;
bc->reply_callback = reply_callback;
bc->auth_reply_cb = auth_reply_callback;
bc->timeout_callback = timeout_callback;
bc->disconnect_callback = NULL; /* Set by RM_SetDisconnectCallback() */
bc->free_privdata = free_privdata;
......@@ -7408,6 +7648,13 @@ RedisModuleBlockedClient *moduleBlockClient(RedisModuleCtx *ctx, RedisModuleCmdF
addReplyError(c, islua ?
"Blocking module command called from Lua script" :
"Blocking module command called from transaction");
} else if (ctx->flags & REDISMODULE_CTX_BLOCKED_REPLY) {
c->bstate.module_blocked_handle = NULL;
addReplyError(c, "Blocking module command called from a Reply callback context");
}
else if (!auth_reply_callback && clientHasModuleAuthInProgress(c)) {
c->bstate.module_blocked_handle = NULL;
addReplyError(c, "Clients undergoing module based authentication can only be blocked on auth");
} else {
if (keys) {
blockForKeys(c,BLOCKED_MODULE,keys,numkeys,timeout,flags&REDISMODULE_BLOCK_UNBLOCK_DELETED);
......@@ -7418,6 +7665,185 @@ RedisModuleBlockedClient *moduleBlockClient(RedisModuleCtx *ctx, RedisModuleCmdF
return bc;
}
 
/* This API registers a callback to execute in addition to normal password based authentication.
* Multiple callbacks can be registered across different modules. When a Module is unloaded, all the
* auth callbacks registered by it are unregistered.
* The callbacks are attempted (in the order of most recently registered first) when the AUTH/HELLO
* (with AUTH field provided) commands are called.
* The callbacks will be called with a module context along with a username and a password, and are
* expected to take one of the following actions:
* (1) Authenticate - Use the RM_AuthenticateClient* API and return REDISMODULE_AUTH_HANDLED.
* This will immediately end the auth chain as successful and add the OK reply.
* (2) Deny Authentication - Return REDISMODULE_AUTH_HANDLED without authenticating or blocking the
* client. Optionally, `err` can be set to a custom error message and `err` will be automatically
* freed by the server.
* This will immediately end the auth chain as unsuccessful and add the ERR reply.
* (3) Block a client on authentication - Use the RM_BlockClientOnAuth API and return
* REDISMODULE_AUTH_HANDLED. Here, the client will be blocked until the RM_UnblockClient API is used
* which will trigger the auth reply callback (provided through the RM_BlockClientOnAuth).
* In this reply callback, the Module should authenticate, deny or skip handling authentication.
* (4) Skip handling Authentication - Return REDISMODULE_AUTH_NOT_HANDLED without blocking the
* client. This will allow the engine to attempt the next module auth callback.
* If none of the callbacks authenticate or deny auth, then password based auth is attempted and
* will authenticate or add failure logs and reply to the clients accordingly.
*
* Note: If a client is disconnected while it was in the middle of blocking module auth, that
* occurrence of the AUTH or HELLO command will not be tracked in the INFO command stats.
*
* The following is an example of how non-blocking module based authentication can be used:
*
* int auth_cb(RedisModuleCtx *ctx, RedisModuleString *username, RedisModuleString *password, RedisModuleString **err) {
* const char *user = RedisModule_StringPtrLen(username, NULL);
* const char *pwd = RedisModule_StringPtrLen(password, NULL);
* if (!strcmp(user,"foo") && !strcmp(pwd,"valid_password")) {
* RedisModule_AuthenticateClientWithACLUser(ctx, "foo", 3, NULL, NULL, NULL);
* return REDISMODULE_AUTH_HANDLED;
* }
*
* else if (!strcmp(user,"foo") && !strcmp(pwd,"wrong_password")) {
* RedisModuleString *log = RedisModule_CreateString(ctx, "Module Auth", 11);
* RedisModule_ACLAddLogEntryByUserName(ctx, username, log, REDISMODULE_ACL_LOG_AUTH);
* RedisModule_FreeString(ctx, log);
* const char *err_msg = "Auth denied by Misc Module.";
* *err = RedisModule_CreateString(ctx, err_msg, strlen(err_msg));
* return REDISMODULE_AUTH_HANDLED;
* }
* return REDISMODULE_AUTH_NOT_HANDLED;
* }
*
* int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
* if (RedisModule_Init(ctx,"authmodule",1,REDISMODULE_APIVER_1)== REDISMODULE_ERR)
* return REDISMODULE_ERR;
* RedisModule_RegisterAuthCallback(ctx, auth_cb);
* return REDISMODULE_OK;
* }
*/
void RM_RegisterAuthCallback(RedisModuleCtx *ctx, RedisModuleAuthCallback cb) {
RedisModuleAuthCtx *auth_ctx = zmalloc(sizeof(RedisModuleAuthCtx));
auth_ctx->module = ctx->module;
auth_ctx->auth_cb = cb;
listAddNodeHead(moduleAuthCallbacks, auth_ctx);
}
/* Helper function to invoke the free private data callback of a Module blocked client. */
void moduleInvokeFreePrivDataCallback(client *c, RedisModuleBlockedClient *bc) {
if (bc->privdata && bc->free_privdata) {
RedisModuleCtx ctx;
int ctx_flags = c == NULL ? REDISMODULE_CTX_BLOCKED_DISCONNECTED : REDISMODULE_CTX_NONE;
moduleCreateContext(&ctx, bc->module, ctx_flags);
ctx.blocked_privdata = bc->privdata;
ctx.client = bc->client;
bc->free_privdata(&ctx,bc->privdata);
moduleFreeContext(&ctx);
}
}
/* Unregisters all the module auth callbacks that have been registered by this Module. */
void moduleUnregisterAuthCBs(RedisModule *module) {
listIter li;
listNode *ln;
listRewind(moduleAuthCallbacks, &li);
while ((ln = listNext(&li))) {
RedisModuleAuthCtx *ctx = listNodeValue(ln);
if (ctx->module == module) {
listDelNode(moduleAuthCallbacks, ln);
zfree(ctx);
}
}
}
/* Search for & attempt next module auth callback after skipping the ones already attempted.
* Returns the result of the module auth callback. */
int attemptNextAuthCb(client *c, robj *username, robj *password, robj **err) {
int handle_next_callback = c->module_auth_ctx == NULL;
RedisModuleAuthCtx *cur_auth_ctx = NULL;
listNode *ln;
listIter li;
listRewind(moduleAuthCallbacks, &li);
int result = REDISMODULE_AUTH_NOT_HANDLED;
while((ln = listNext(&li))) {
cur_auth_ctx = listNodeValue(ln);
/* Skip over the previously attempted auth contexts. */
if (!handle_next_callback) {
handle_next_callback = cur_auth_ctx == c->module_auth_ctx;
continue;
}
/* Remove the module auth complete flag before we attempt the next cb. */
c->flags &= ~CLIENT_MODULE_AUTH_HAS_RESULT;
RedisModuleCtx ctx;
moduleCreateContext(&ctx, cur_auth_ctx->module, REDISMODULE_CTX_NONE);
ctx.client = c;
*err = NULL;
c->module_auth_ctx = cur_auth_ctx;
result = cur_auth_ctx->auth_cb(&ctx, username, password, err);
moduleFreeContext(&ctx);
if (result == REDISMODULE_AUTH_HANDLED) break;
/* If Auth was not handled (allowed/denied/blocked) by the Module, try the next auth cb. */
}
return result;
}
/* Helper function to handle a reprocessed unblocked auth client.
* Returns REDISMODULE_AUTH_NOT_HANDLED if the client was not reprocessed after a blocking module
* auth operation.
* Otherwise, we attempt the auth reply callback & the free priv data callback, update fields and
* return the result of the reply callback. */
int attemptBlockedAuthReplyCallback(client *c, robj *username, robj *password, robj **err) {
int result = REDISMODULE_AUTH_NOT_HANDLED;
if (!c->module_blocked_client) return result;
RedisModuleBlockedClient *bc = (RedisModuleBlockedClient *) c->module_blocked_client;
bc->client = c;
if (bc->auth_reply_cb) {
RedisModuleCtx ctx;
moduleCreateContext(&ctx, bc->module, REDISMODULE_CTX_BLOCKED_REPLY);
ctx.blocked_privdata = bc->privdata;
ctx.blocked_ready_key = NULL;
ctx.client = bc->client;
ctx.blocked_client = bc;
result = bc->auth_reply_cb(&ctx, username, password, err);
moduleFreeContext(&ctx);
}
moduleInvokeFreePrivDataCallback(c, bc);
c->module_blocked_client = NULL;
c->lastcmd->microseconds += bc->background_duration;
bc->module->blocked_clients--;
zfree(bc);
return result;
}
/* Helper function to attempt Module based authentication through module auth callbacks.
* Here, the Module is expected to authenticate the client using the RedisModule APIs and to add ACL
* logs in case of errors.
* Returns one of the following codes:
* AUTH_OK - Indicates that a module handled and authenticated the client.
* AUTH_ERR - Indicates that a module handled and denied authentication for this client.
* AUTH_NOT_HANDLED - Indicates that authentication was not handled by any Module and that
* normal password based authentication can be attempted next.
* AUTH_BLOCKED - Indicates module authentication is in progress through a blocking implementation.
* In this case, authentication is handled here again after the client is unblocked / reprocessed. */
int checkModuleAuthentication(client *c, robj *username, robj *password, robj **err) {
if (!listLength(moduleAuthCallbacks)) return AUTH_NOT_HANDLED;
int result = attemptBlockedAuthReplyCallback(c, username, password, err);
if (result == REDISMODULE_AUTH_NOT_HANDLED) {
result = attemptNextAuthCb(c, username, password, err);
}
if (c->flags & CLIENT_BLOCKED) {
/* Modules are expected to return REDISMODULE_AUTH_HANDLED when blocking clients. */
serverAssert(result == REDISMODULE_AUTH_HANDLED);
return AUTH_BLOCKED;
}
c->module_auth_ctx = NULL;
if (result == REDISMODULE_AUTH_NOT_HANDLED) {
c->flags &= ~CLIENT_MODULE_AUTH_HAS_RESULT;
return AUTH_NOT_HANDLED;
}
if (c->flags & CLIENT_MODULE_AUTH_HAS_RESULT) {
c->flags &= ~CLIENT_MODULE_AUTH_HAS_RESULT;
if (c->authenticated) return AUTH_OK;
}
return AUTH_ERR;
}
/* This function is called from module.c in order to check if a module
* blocked for BLOCKED_MODULE and subtype 'on keys' (bc->blocked_on_keys true)
* can really be unblocked, since the module was able to serve the client.
......@@ -7488,7 +7914,34 @@ int moduleTryServeClientBlockedOnKey(client *c, robj *key) {
RedisModuleBlockedClient *RM_BlockClient(RedisModuleCtx *ctx, RedisModuleCmdFunc reply_callback,
RedisModuleCmdFunc timeout_callback, void (*free_privdata)(RedisModuleCtx*,void*),
long long timeout_ms) {
return moduleBlockClient(ctx,reply_callback,timeout_callback,free_privdata,timeout_ms, NULL,0,NULL,0);
return moduleBlockClient(ctx,reply_callback,NULL,timeout_callback,free_privdata,timeout_ms, NULL,0,NULL,0);
}
/* Block the current client for module authentication in the background. If module auth is not in
* progress on the client, the API returns NULL. Otherwise, the client is blocked and the RM_BlockedClient
* is returned similar to the RM_BlockClient API.
* Note: Only use this API from the context of a module auth callback. */
RedisModuleBlockedClient *RM_BlockClientOnAuth(RedisModuleCtx *ctx, RedisModuleAuthCallback reply_callback,
void (*free_privdata)(RedisModuleCtx*,void*)) {
if (!clientHasModuleAuthInProgress(ctx->client)) {
addReplyError(ctx->client, "Module blocking client on auth when not currently undergoing module authentication");
return NULL;
}
RedisModuleBlockedClient *bc = moduleBlockClient(ctx,NULL,reply_callback,NULL,free_privdata,0, NULL,0,NULL,0);
if (ctx->client->flags & CLIENT_BLOCKED) {
ctx->client->flags |= CLIENT_PENDING_COMMAND;
}
return bc;
}
/* Get the private data that was previusely set on a blocked client */
void *RM_BlockClientGetPrivateData(RedisModuleBlockedClient *blocked_client) {
return blocked_client->privdata;
}
/* Set private data on a blocked client */
void RM_BlockClientSetPrivateData(RedisModuleBlockedClient *blocked_client, void *private_data) {
blocked_client->privdata = private_data;
}
 
/* This call is similar to RedisModule_BlockClient(), however in this case we
......@@ -7552,7 +8005,7 @@ RedisModuleBlockedClient *RM_BlockClient(RedisModuleCtx *ctx, RedisModuleCmdFunc
RedisModuleBlockedClient *RM_BlockClientOnKeys(RedisModuleCtx *ctx, RedisModuleCmdFunc reply_callback,
RedisModuleCmdFunc timeout_callback, void (*free_privdata)(RedisModuleCtx*,void*),
long long timeout_ms, RedisModuleString **keys, int numkeys, void *privdata) {
return moduleBlockClient(ctx,reply_callback,timeout_callback,free_privdata,timeout_ms, keys,numkeys,privdata,0);
return moduleBlockClient(ctx,reply_callback,NULL,timeout_callback,free_privdata,timeout_ms, keys,numkeys,privdata,0);
}
 
/* Same as RedisModule_BlockClientOnKeys, but can take REDISMODULE_BLOCK_* flags
......@@ -7568,7 +8021,7 @@ RedisModuleBlockedClient *RM_BlockClientOnKeysWithFlags(RedisModuleCtx *ctx, Red
RedisModuleCmdFunc timeout_callback, void (*free_privdata)(RedisModuleCtx*,void*),
long long timeout_ms, RedisModuleString **keys, int numkeys, void *privdata,
int flags) {
return moduleBlockClient(ctx,reply_callback,timeout_callback,free_privdata,timeout_ms, keys,numkeys,privdata,flags);
return moduleBlockClient(ctx,reply_callback,NULL,timeout_callback,free_privdata,timeout_ms, keys,numkeys,privdata,flags);
}
 
/* This function is used in order to potentially unblock a client blocked
......@@ -7643,6 +8096,7 @@ int RM_UnblockClient(RedisModuleBlockedClient *bc, void *privdata) {
int RM_AbortBlock(RedisModuleBlockedClient *bc) {
bc->reply_callback = NULL;
bc->disconnect_callback = NULL;
bc->auth_reply_cb = NULL;
return RM_UnblockClient(bc,NULL);
}
 
......@@ -7709,16 +8163,13 @@ void moduleHandleBlockedClients(void) {
reply_us = elapsedUs(replyTimer);
moduleFreeContext(&ctx);
}
/* Hold onto the blocked client if module auth is in progress. The reply callback is invoked
* when the client is reprocessed. */
if (c && clientHasModuleAuthInProgress(c)) {
c->module_blocked_client = bc;
} else {
/* Free privdata if any. */
if (bc->privdata && bc->free_privdata) {
RedisModuleCtx ctx;
int ctx_flags = c == NULL ? REDISMODULE_CTX_BLOCKED_DISCONNECTED : REDISMODULE_CTX_NONE;
moduleCreateContext(&ctx, bc->module, ctx_flags);
ctx.blocked_privdata = bc->privdata;
ctx.client = bc->client;
bc->free_privdata(&ctx,bc->privdata);
moduleFreeContext(&ctx);
moduleInvokeFreePrivDataCallback(c, bc);
}
 
/* It is possible that this blocked client object accumulated
......@@ -7733,7 +8184,7 @@ void moduleHandleBlockedClients(void) {
* This needs to be out of the reply callback above given that a
* module might not define any callback and still do blocking ops.
*/
if (c && !bc->blocked_on_keys) {
if (c && !clientHasModuleAuthInProgress(c) && !bc->blocked_on_keys) {
updateStatsOnUnblock(c, bc->background_duration, reply_us, server.stat_total_error_replies != prev_error_replies);
}
 
......@@ -7742,11 +8193,11 @@ void moduleHandleBlockedClients(void) {
* to NULL, because if we reached this point, the client was
* properly unblocked by the module. */
bc->disconnect_callback = NULL;
unblockClient(c);
unblockClient(c, 1);
/* Put the client in the list of clients that need to write
* if there are pending replies here. This is needed since
* during a non blocking command the client may receive output. */
if (clientHasPendingReplies(c) &&
if (!clientHasModuleAuthInProgress(c) && clientHasPendingReplies(c) &&
!(c->flags & CLIENT_PENDING_WRITE))
{
c->flags |= CLIENT_PENDING_WRITE;
......@@ -7757,8 +8208,10 @@ void moduleHandleBlockedClients(void) {
/* Free 'bc' only after unblocking the client, since it is
* referenced in the client blocking context, and must be valid
* when calling unblockClient(). */
if (!(c && clientHasModuleAuthInProgress(c))) {
bc->module->blocked_clients--;
zfree(bc);
}
 
/* Lock again before to iterate the loop. */
pthread_mutex_lock(&moduleUnblockedClientsMutex);
......@@ -7926,8 +8379,7 @@ void moduleGILAfterLock() {
serverAssert(server.execution_nesting == 0);
/* Bump up the nesting level to prevent immediate propagation
* of possible RM_Call from th thread */
server.execution_nesting++;
updateCachedTime(0);
enterExecutionUnit(1, 0);
}
 
/* Acquire the server lock before executing a thread safe API call.
......@@ -7965,7 +8417,7 @@ void moduleGILBeforeUnlock() {
/* Restore nesting level and propagate pending commands
* (because it's unclear when thread safe contexts are
* released we have to propagate here). */
server.execution_nesting--;
exitExecutionUnit();
postExecutionUnitOperations();
}
 
......@@ -8075,8 +8527,10 @@ int RM_SubscribeToKeyspaceEvents(RedisModuleCtx *ctx, int types, RedisModuleNoti
void firePostExecutionUnitJobs() {
/* Avoid propagation of commands.
* In that way, postExecutionUnitOperations will prevent
* recursive calls to firePostExecutionUnitJobs. */
server.execution_nesting++;
* recursive calls to firePostExecutionUnitJobs.
* This is a special case where we need to increase 'execution_nesting'
* but we do not want to update the cached time */
enterExecutionUnit(0, 0);
while (listLength(modulePostExecUnitJobs) > 0) {
listNode *ln = listFirst(modulePostExecUnitJobs);
RedisModulePostExecUnitJob *job = listNodeValue(ln);
......@@ -8092,7 +8546,7 @@ void firePostExecutionUnitJobs() {
moduleFreeContext(&ctx);
zfree(job);
}
server.execution_nesting--;
exitExecutionUnit();
}
 
/* When running inside a key space notification callback, it is dangerous and highly discouraged to perform any write
......@@ -8155,8 +8609,11 @@ void moduleNotifyKeyspaceEvent(int type, const char *event, robj *key, int dbid)
*
* In order to do that we increment the execution_nesting counter, thus
* preventing postExecutionUnitOperations (from within moduleFreeContext)
* from propagating commands from CB. */
server.execution_nesting++;
* from propagating commands from CB.
*
* This is a special case where we need to increase 'execution_nesting'
* but we do not want to update the cached time */
enterExecutionUnit(0, 0);
 
listIter li;
listNode *ln;
......@@ -8187,7 +8644,7 @@ void moduleNotifyKeyspaceEvent(int type, const char *event, robj *key, int dbid)
}
}
 
server.execution_nesting--;
exitExecutionUnit();
}
 
/* Unsubscribe any notification subscribers this module has upon unloading */
......@@ -9135,24 +9592,41 @@ int RM_ACLCheckChannelPermissions(RedisModuleUser *user, RedisModuleString *ch,
return REDISMODULE_OK;
}
 
/* Adds a new entry in the ACL log.
* Returns REDISMODULE_OK on success and REDISMODULE_ERR on error.
*
* For more information about ACL log, please refer to https://redis.io/commands/acl-log */
int RM_ACLAddLogEntry(RedisModuleCtx *ctx, RedisModuleUser *user, RedisModuleString *object, RedisModuleACLLogEntryReason reason) {
int acl_reason;
/* Helper function to map a RedisModuleACLLogEntryReason to ACL Log entry reason. */
int moduleGetACLLogEntryReason(RedisModuleACLLogEntryReason reason) {
int acl_reason = 0;
switch (reason) {
case REDISMODULE_ACL_LOG_AUTH: acl_reason = ACL_DENIED_AUTH; break;
case REDISMODULE_ACL_LOG_KEY: acl_reason = ACL_DENIED_KEY; break;
case REDISMODULE_ACL_LOG_CHANNEL: acl_reason = ACL_DENIED_CHANNEL; break;
case REDISMODULE_ACL_LOG_CMD: acl_reason = ACL_DENIED_CMD; break;
default: return REDISMODULE_ERR;
default: break;
}
return acl_reason;
}
 
/* Adds a new entry in the ACL log.
* Returns REDISMODULE_OK on success and REDISMODULE_ERR on error.
*
* For more information about ACL log, please refer to https://redis.io/commands/acl-log */
int RM_ACLAddLogEntry(RedisModuleCtx *ctx, RedisModuleUser *user, RedisModuleString *object, RedisModuleACLLogEntryReason reason) {
int acl_reason = moduleGetACLLogEntryReason(reason);
if (!acl_reason) return REDISMODULE_ERR;
addACLLogEntry(ctx->client, acl_reason, ACL_LOG_CTX_MODULE, -1, user->user->name, sdsdup(object->ptr));
return REDISMODULE_OK;
}
 
/* Adds a new entry in the ACL log with the `username` RedisModuleString provided.
* Returns REDISMODULE_OK on success and REDISMODULE_ERR on error.
*
* For more information about ACL log, please refer to https://redis.io/commands/acl-log */
int RM_ACLAddLogEntryByUserName(RedisModuleCtx *ctx, RedisModuleString *username, RedisModuleString *object, RedisModuleACLLogEntryReason reason) {
int acl_reason = moduleGetACLLogEntryReason(reason);
if (!acl_reason) return REDISMODULE_ERR;
addACLLogEntry(ctx->client, acl_reason, ACL_LOG_CTX_MODULE, -1, username->ptr, sdsdup(object->ptr));
return REDISMODULE_OK;
}
/* Authenticate the client associated with the context with
* the provided user. Returns REDISMODULE_OK on success and
* REDISMODULE_ERR on error.
......@@ -9188,6 +9662,10 @@ static int authenticateClientWithUser(RedisModuleCtx *ctx, user *user, RedisModu
ctx->client->user = user;
ctx->client->authenticated = 1;
 
if (clientHasModuleAuthInProgress(ctx->client)) {
ctx->client->flags |= CLIENT_MODULE_AUTH_HAS_RESULT;
}
if (callback) {
ctx->client->auth_callback = callback;
ctx->client->auth_callback_privdata = privdata;
......@@ -11284,6 +11762,7 @@ void moduleInitModulesSystem(void) {
server.loadmodule_queue = listCreate();
server.module_configs_queue = dictCreate(&sdsKeyValueHashDictType);
modules = dictCreate(&modulesDictType);
moduleAuthCallbacks = listCreate();
 
/* Set up the keyspace notification subscriber list and static client */
moduleKeyspaceSubscribers = listCreate();
......@@ -11588,6 +12067,7 @@ int moduleLoad(const char *path, void **module_argv, int module_argc, int is_loa
moduleUnregisterSharedAPI(ctx.module);
moduleUnregisterUsedAPI(ctx.module);
moduleRemoveConfigs(ctx.module);
moduleUnregisterAuthCBs(ctx.module);
moduleFreeModuleStructure(ctx.module);
}
moduleFreeContext(&ctx);
......@@ -11608,18 +12088,29 @@ int moduleLoad(const char *path, void **module_argv, int module_argc, int is_loa
incrRefCount(ctx.module->loadmod->argv[i]);
}
 
/* If module commands have ACL categories, recompute command bits
* for all existing users once the modules has been registered. */
if (ctx.module->num_commands_with_acl_categories) {
ACLRecomputeCommandBitsFromCommandRulesAllUsers();
}
serverLog(LL_NOTICE,"Module '%s' loaded from %s",ctx.module->name,path);
ctx.module->onload = 0;
 
int post_load_err = 0;
if (listLength(ctx.module->module_configs) && !ctx.module->configs_initialized) {
serverLogRaw(LL_WARNING, "Module Configurations were not set, likely a missing LoadConfigs call. Unloading the module.");
moduleUnload(ctx.module->name);
moduleFreeContext(&ctx);
return C_ERR;
post_load_err = 1;
}
 
if (is_loadex && dictSize(server.module_configs_queue)) {
serverLogRaw(LL_WARNING, "Loadex configurations were not applied, likely due to invalid arguments. Unloading the module.");
moduleUnload(ctx.module->name);
post_load_err = 1;
}
if (post_load_err) {
/* Unregister module auth callbacks (if any exist) that this Module registered onload. */
moduleUnregisterAuthCBs(ctx.module);
moduleUnload(ctx.module->name, NULL);
moduleFreeContext(&ctx);
return C_ERR;
}
......@@ -11634,32 +12125,29 @@ int moduleLoad(const char *path, void **module_argv, int module_argc, int is_loa
}
 
/* Unload the module registered with the specified name. On success
* C_OK is returned, otherwise C_ERR is returned and errno is set
* to the following values depending on the type of error:
*
* * ENONET: No such module having the specified name.
* * EBUSY: The module exports a new data type and can only be reloaded.
* * EPERM: The module exports APIs which are used by other module.
* * EAGAIN: The module has blocked clients.
* * EINPROGRESS: The module holds timer not fired.
* * ECANCELED: Unload module error. */
int moduleUnload(sds name) {
* C_OK is returned, otherwise C_ERR is returned and errmsg is set
* with an appropriate message. */
int moduleUnload(sds name, const char **errmsg) {
struct RedisModule *module = dictFetchValue(modules,name);
 
if (module == NULL) {
errno = ENOENT;
*errmsg = "no such module with that name";
return C_ERR;
} else if (listLength(module->types)) {
errno = EBUSY;
*errmsg = "the module exports one or more module-side data "
"types, can't unload";
return C_ERR;
} else if (listLength(module->usedby)) {
errno = EPERM;
*errmsg = "the module exports APIs used by other modules. "
"Please unload them first and try again";
return C_ERR;
} else if (module->blocked_clients) {
errno = EAGAIN;
*errmsg = "the module has blocked clients. "
"Please wait for them to be unblocked and try again";
return C_ERR;
} else if (moduleHoldsTimer(module)) {
errno = EINPROGRESS;
*errmsg = "the module holds timer that is not fired. "
"Please stop the timer or wait until it fires.";
return C_ERR;
}
 
......@@ -11684,6 +12172,7 @@ int moduleUnload(sds name) {
moduleUnregisterSharedAPI(module);
moduleUnregisterUsedAPI(module);
moduleUnregisterFilters(module);
moduleUnregisterAuthCBs(module);
moduleRemoveConfigs(module);
 
/* Remove any notification subscribers this module might have */
......@@ -11709,6 +12198,8 @@ int moduleUnload(sds name) {
module->name = NULL; /* The name was already freed by dictDelete(). */
moduleFreeModuleStructure(module);
 
/* Recompute command bits for all users once the modules has been completely unloaded. */
ACLRecomputeCommandBitsFromCommandRulesAllUsers();
return C_OK;
}
 
......@@ -12016,6 +12507,10 @@ ModuleConfig *createModuleConfig(sds name, RedisModuleConfigApplyFunc apply_fn,
}
 
int moduleConfigValidityCheck(RedisModule *module, sds name, unsigned int flags, configType type) {
if (!module->onload) {
errno = EBUSY;
return REDISMODULE_ERR;
}
if (moduleVerifyConfigFlags(flags, type) || moduleVerifyConfigName(name)) {
errno = EINVAL;
return REDISMODULE_ERR;
......@@ -12122,6 +12617,7 @@ unsigned int maskModuleEnumConfigFlags(unsigned int flags) {
*
* If the registration fails, REDISMODULE_ERR is returned and one of the following
* errno is set:
* * EBUSY: Registering the Config outside of RedisModule_OnLoad.
* * EINVAL: The provided flags are invalid for the registration or the name of the config contains invalid characters.
* * EALREADY: The provided configuration name is already used. */
int RM_RegisterStringConfig(RedisModuleCtx *ctx, const char *name, const char *default_val, unsigned int flags, RedisModuleConfigGetStringFunc getfn, RedisModuleConfigSetStringFunc setfn, RedisModuleConfigApplyFunc applyfn, void *privdata) {
......@@ -12238,10 +12734,11 @@ int RM_RegisterNumericConfig(RedisModuleCtx *ctx, const char *name, long long de
 
/* Applies all pending configurations on the module load. This should be called
* after all of the configurations have been registered for the module inside of RedisModule_OnLoad.
* This will return REDISMODULE_ERR if it is called outside RedisModule_OnLoad.
* This API needs to be called when configurations are provided in either `MODULE LOADEX`
* or provided as startup arguments. */
int RM_LoadConfigs(RedisModuleCtx *ctx) {
if (!ctx || !ctx->module) {
if (!ctx || !ctx->module || !ctx->module->onload) {
return REDISMODULE_ERR;
}
RedisModule *module = ctx->module;
......@@ -12307,35 +12804,13 @@ NULL
}
 
} else if (!strcasecmp(subcmd,"unload") && c->argc == 3) {
if (moduleUnload(c->argv[2]->ptr) == C_OK)
const char *errmsg = NULL;
if (moduleUnload(c->argv[2]->ptr, &errmsg) == C_OK)
addReply(c,shared.ok);
else {
char *errmsg;
switch(errno) {
case ENOENT:
errmsg = "no such module with that name";
break;
case EBUSY:
errmsg = "the module exports one or more module-side data "
"types, can't unload";
break;
case EPERM:
errmsg = "the module exports APIs used by other modules. "
"Please unload them first and try again";
break;
case EAGAIN:
errmsg = "the module has blocked clients. "
"Please wait them unblocked and try again";
break;
case EINPROGRESS:
errmsg = "the module holds timer that is not fired. "
"Please stop the timer or wait until it fires.";
break;
default:
errmsg = "operation not possible.";
break;
}
addReplyErrorFormat(c,"Error unloading module: %s",errmsg);
if (errmsg == NULL) errmsg = "operation not possible.";
addReplyErrorFormat(c, "Error unloading module: %s", errmsg);
serverLog(LL_WARNING, "Error unloading module %s: %s", (sds) c->argv[2]->ptr, errmsg);
}
} else if (!strcasecmp(subcmd,"list") && c->argc == 2) {
addReplyLoadedModules(c);
......@@ -12812,6 +13287,7 @@ void moduleRegisterCoreAPI(void) {
REGISTER_API(GetCommand);
REGISTER_API(CreateSubcommand);
REGISTER_API(SetCommandInfo);
REGISTER_API(SetCommandACLCategories);
REGISTER_API(SetModuleAttribs);
REGISTER_API(IsModuleNameBusy);
REGISTER_API(WrongArity);
......@@ -12870,6 +13346,8 @@ void moduleRegisterCoreAPI(void) {
REGISTER_API(CallReplySetElement);
REGISTER_API(CallReplyMapElement);
REGISTER_API(CallReplyAttributeElement);
REGISTER_API(CallReplyPromiseSetUnblockHandler);
REGISTER_API(CallReplyPromiseAbort);
REGISTER_API(CallReplyAttribute);
REGISTER_API(CallReplyType);
REGISTER_API(CallReplyLength);
......@@ -12984,6 +13462,9 @@ void moduleRegisterCoreAPI(void) {
REGISTER_API(GetKeyNameFromDigest);
REGISTER_API(GetDbIdFromDigest);
REGISTER_API(BlockClient);
REGISTER_API(BlockClientGetPrivateData);
REGISTER_API(BlockClientSetPrivateData);
REGISTER_API(BlockClientOnAuth);
REGISTER_API(UnblockClient);
REGISTER_API(IsBlockedReplyRequest);
REGISTER_API(IsBlockedTimeoutRequest);
......@@ -13110,6 +13591,7 @@ void moduleRegisterCoreAPI(void) {
REGISTER_API(ACLCheckKeyPermissions);
REGISTER_API(ACLCheckChannelPermissions);
REGISTER_API(ACLAddLogEntry);
REGISTER_API(ACLAddLogEntryByUserName);
REGISTER_API(FreeModuleUser);
REGISTER_API(DeauthenticateAndCloseClient);
REGISTER_API(AuthenticateClientWithACLUser);
......@@ -13140,4 +13622,5 @@ void moduleRegisterCoreAPI(void) {
REGISTER_API(RegisterStringConfig);
REGISTER_API(RegisterEnumConfig);
REGISTER_API(LoadConfigs);
REGISTER_API(RegisterAuthCallback);
}
......@@ -139,7 +139,12 @@ client *createClient(connection *conn) {
uint64_t client_id;
atomicGetIncr(server.next_client_id, client_id, 1);
c->id = client_id;
#ifdef LOG_REQ_RES
reqresReset(c, 0);
c->resp = server.client_default_resp;
#else
c->resp = 2;
#endif
c->conn = conn;
c->name = NULL;
c->bufpos = 0;
......@@ -175,6 +180,7 @@ client *createClient(connection *conn) {
c->repl_applied = 0;
c->repl_ack_off = 0;
c->repl_ack_time = 0;
c->repl_aof_off = 0;
c->repl_last_partial_write = 0;
c->slave_listening_port = 0;
c->slave_addr = NULL;
......@@ -201,6 +207,8 @@ client *createClient(connection *conn) {
c->client_tracking_prefixes = NULL;
c->last_memory_usage = 0;
c->last_memory_type = CLIENT_TYPE_NORMAL;
c->module_blocked_client = NULL;
c->module_auth_ctx = NULL;
c->auth_callback = NULL;
c->auth_callback_privdata = NULL;
c->auth_module = NULL;
......@@ -287,8 +295,10 @@ int prepareClientToWrite(client *c) {
/* If CLIENT_CLOSE_ASAP flag is set, we need not write anything. */
if (c->flags & CLIENT_CLOSE_ASAP) return C_ERR;
/* CLIENT REPLY OFF / SKIP handling: don't send replies. */
if (c->flags & (CLIENT_REPLY_OFF|CLIENT_REPLY_SKIP)) return C_ERR;
/* CLIENT REPLY OFF / SKIP handling: don't send replies.
* CLIENT_PUSHING handling: disables the reply silencing flags. */
if ((c->flags & (CLIENT_REPLY_OFF|CLIENT_REPLY_SKIP)) &&
!(c->flags & CLIENT_PUSHING)) return C_ERR;
/* Masters don't receive replies, unless CLIENT_MASTER_FORCE_REPLY flag
* is set. */
......@@ -390,6 +400,10 @@ void _addReplyToBufferOrList(client *c, const char *s, size_t len) {
return;
}
/* We call it here because this function may affect the reply
* buffer offset (see function comment) */
reqresSaveClientReplyOffset(c);
size_t reply_len = _addReplyToBuffer(c,s,len);
if (len > reply_len) _addReplyProtoToList(c,s+reply_len,len-reply_len);
}
......@@ -563,8 +577,8 @@ void addReplyErrorObject(client *c, robj *err) {
}
/* Sends either a reply or an error reply by checking the first char.
 * If the first char is '-' the reply is considered an error.
 * In any case the given reply is sent, if the reply is also recognize
* If the first char is '-' the reply is considered an error.
* In any case the given reply is sent, if the reply is also recognize
* as an error we also perform some post reply operations such as
* logging and stats update. */
void addReplyOrErrorObject(client *c, robj *reply) {
......@@ -714,6 +728,10 @@ void *addReplyDeferredLen(client *c) {
return NULL;
}
/* We call it here because this function conceptually affects the reply
* buffer offset (see function comment) */
reqresSaveClientReplyOffset(c);
trimReplyUnusedTailSpace(c);
listAddNodeTail(c->reply,NULL); /* NULL is our placeholder. */
return listLast(c->reply);
......@@ -963,6 +981,7 @@ void addReplyAttributeLen(client *c, long length) {
void addReplyPushLen(client *c, long length) {
serverAssert(c->resp >= 3);
serverAssertWithInfo(c, NULL, c->flags & CLIENT_PUSHING);
addReplyAggregateLen(c,length,'>');
}
......@@ -1517,6 +1536,9 @@ void freeClient(client *c) {
/* Notify module system that this client auth status changed. */
moduleNotifyUserChanged(c);
/* Free the RedisModuleBlockedClient held onto for reprocessing if not already freed. */
zfree(c->module_blocked_client);
/* If this client was scheduled for async freeing we need to remove it
* from the queue. Note that we need to do this here, because later
* we may call replicationCacheMaster() and the client should already
......@@ -1552,7 +1574,7 @@ void freeClient(client *c) {
c->querybuf = NULL;
/* Deallocate structures used to block on blocking ops. */
if (c->flags & CLIENT_BLOCKED) unblockClient(c);
if (c->flags & CLIENT_BLOCKED) unblockClient(c, 1);
dictRelease(c->bstate.keys);
/* UNWATCH all the keys */
......@@ -1575,6 +1597,9 @@ void freeClient(client *c) {
freeClientOriginalArgv(c);
if (c->deferred_reply_errors)
listRelease(c->deferred_reply_errors);
#ifdef LOG_REQ_RES
reqresReset(c, 1);
#endif
/* Unlink the client: this will close the socket, remove the I/O
* handlers, and remove references of the client from different
......@@ -1690,6 +1715,11 @@ void logInvalidUseAndFreeClientAsync(client *c, const char *fmt, ...) {
* The input client argument: c, may be NULL in case the previous client was
* freed before the call. */
int beforeNextClient(client *c) {
/* Notice, this code is also called from 'processUnblockedClients'.
* But in case of a module blocked client (see RM_Call 'K' flag) we do not reach this code path.
* So whenever we change the code here we need to consider if we need this change on module
* blocked client as well */
/* Skip the client processing if we're in an IO thread, in that case we'll perform
this operation later (this function is called again) in the fan-in stage of the threading mechanism */
if (io_threads_op != IO_THREADS_OP_IDLE)
......@@ -2000,6 +2030,9 @@ void resetClient(client *c) {
c->slot = -1;
c->duration = 0;
c->flags &= ~CLIENT_EXECUTING_COMMAND;
#ifdef LOG_REQ_RES
reqresReset(c, 1);
#endif
if (c->deferred_reply_errors)
listRelease(c->deferred_reply_errors);
......@@ -2357,6 +2390,7 @@ void commandProcessed(client *c) {
* since we have not applied the command. */
if (c->flags & CLIENT_BLOCKED) return;
reqresAppendResponse(c);
resetClient(c);
long long prev_offset = c->reploff;
......@@ -2419,6 +2453,10 @@ int processCommandAndResetClient(client *c) {
* the client. Returns C_ERR if the client is no longer valid after executing
* the command, and C_OK for all other cases. */
int processPendingCommandAndInputBuffer(client *c) {
/* Notice, this code is also called from 'processUnblockedClients'.
* But in case of a module blocked client (see RM_Call 'K' flag) we do not reach this code path.
* So whenever we change the code here we need to consider if we need this change on module
* blocked client as well */
if (c->flags & CLIENT_PENDING_COMMAND) {
c->flags &= ~CLIENT_PENDING_COMMAND;
if (processCommandAndResetClient(c) == C_ERR) {
......@@ -2429,7 +2467,7 @@ int processPendingCommandAndInputBuffer(client *c) {
/* Now process client if it has more data in it's buffer.
*
* Note: when a master client steps into this function,
* it can always satisfy this condition, because its querbuf
* it can always satisfy this condition, because its querybuf
* contains data not applied. */
if (c->querybuf && sdslen(c->querybuf) > 0) {
return processInputBuffer(c);
......@@ -2785,26 +2823,38 @@ sds getAllClientsInfoString(int type) {
return o;
}
/* Returns C_OK if the name has been set or C_ERR if the name is invalid. */
int clientSetName(client *c, robj *name) {
/* Returns C_OK if the name is valid. Returns C_ERR & sets `err` (when provided) otherwise. */
int validateClientName(robj *name, const char **err) {
const char *err_msg = "Client names cannot contain spaces, newlines or special characters.";
int len = (name != NULL) ? sdslen(name->ptr) : 0;
/* Setting the client name to an empty string actually removes
* the current name. */
if (len == 0) {
if (c->name) decrRefCount(c->name);
c->name = NULL;
/* We allow setting the client name to an empty string. */
if (len == 0)
return C_OK;
}
/* Otherwise check if the charset is ok. We need to do this otherwise
* CLIENT LIST format will break. You should always be able to
* split by space to get the different fields. */
char *p = name->ptr;
for (int j = 0; j < len; j++) {
if (p[j] < '!' || p[j] > '~') { /* ASCII is assumed. */
if (err) *err = err_msg;
return C_ERR;
}
}
return C_OK;
}
/* Returns C_OK if the name has been set or C_ERR if the name is invalid. */
int clientSetName(client *c, robj *name, const char **err) {
if (validateClientName(name, err) == C_ERR) {
return C_ERR;
}
int len = (name != NULL) ? sdslen(name->ptr) : 0;
/* Setting the client name to an empty string actually removes
* the current name. */
if (len == 0) {
if (c->name) decrRefCount(c->name);
c->name = NULL;
return C_OK;
}
if (c->name) decrRefCount(c->name);
c->name = name;
......@@ -2822,11 +2872,10 @@ int clientSetName(client *c, robj *name) {
*
* This function is also used to implement the HELLO SETNAME option. */
int clientSetNameOrReply(client *c, robj *name) {
int result = clientSetName(c, name);
const char *err = NULL;
int result = clientSetName(c, name, &err);
if (result == C_ERR) {
addReplyError(c,
"Client names cannot contain spaces, "
"newlines or special characters.");
addReplyError(c, err);
}
return result;
}
......@@ -3410,19 +3459,25 @@ void helloCommand(client *c) {
}
}
robj *username = NULL;
robj *password = NULL;
robj *clientname = NULL;
for (int j = next_arg; j < c->argc; j++) {
int moreargs = (c->argc-1) - j;
const char *opt = c->argv[j]->ptr;
if (!strcasecmp(opt,"AUTH") && moreargs >= 2) {
redactClientCommandArgument(c, j+1);
redactClientCommandArgument(c, j+2);
if (ACLAuthenticateUser(c, c->argv[j+1], c->argv[j+2]) == C_ERR) {
addReplyError(c,"-WRONGPASS invalid username-password pair or user is disabled.");
return;
}
username = c->argv[j+1];
password = c->argv[j+2];
j += 2;
} else if (!strcasecmp(opt,"SETNAME") && moreargs) {
if (clientSetNameOrReply(c, c->argv[j+1]) == C_ERR) return;
clientname = c->argv[j+1];
const char *err = NULL;
if (validateClientName(clientname, &err) == C_ERR) {
addReplyError(c, err);
return;
}
j++;
} else {
addReplyErrorFormat(c,"Syntax error in HELLO option '%s'",opt);
......@@ -3430,6 +3485,20 @@ void helloCommand(client *c) {
}
}
if (username && password) {
robj *err = NULL;
int auth_result = ACLAuthenticateUser(c, username, password, &err);
if (auth_result == AUTH_ERR) {
addAuthErrReply(c, err);
}
if (err) decrRefCount(err);
/* In case of auth errors, return early since we already replied with an ERR.
* In case of blocking module auth, we reply to the client/setname later upon unblocking. */
if (auth_result == AUTH_ERR || auth_result == AUTH_BLOCKED) {
return;
}
}
/* At this point we need to be authenticated to continue. */
if (!c->authenticated) {
addReplyError(c,"-NOAUTH HELLO must be called with the client already "
......@@ -3439,6 +3508,9 @@ void helloCommand(client *c) {
return;
}
/* Now that we're authenticated, set the client name. */
if (clientname) clientSetName(c, clientname, NULL);
/* Let's switch to the specified RESP mode. */
if (ver) c->resp = ver;
addReplyMapLen(c,6 + !server.sentinel_mode);
......@@ -3842,7 +3914,7 @@ void unblockPostponedClients() {
listRewind(server.postponed_clients, &li);
while ((ln = listNext(&li)) != NULL) {
client *c = listNodeValue(ln);
unblockClient(c);
unblockClient(c, 1);
}
}
......@@ -4008,7 +4080,7 @@ static inline void setIOPendingCount(int i, unsigned long count) {
}
void *IOThreadMain(void *myid) {
/* The ID is the thread number (from 0 to server.iothreads_num-1), and is
/* The ID is the thread number (from 0 to server.io_threads_num-1), and is
* used by the thread to just manipulate a single sub-array of clients. */
long id = (unsigned long)myid;
char thdname[16];
......
......@@ -105,6 +105,8 @@ pubsubtype pubSubShardType = {
* to send a special message (for instance an Array type) by using the
* addReply*() API family. */
void addReplyPubsubMessage(client *c, robj *channel, robj *msg, robj *message_bulk) {
uint64_t old_flags = c->flags;
c->flags |= CLIENT_PUSHING;
if (c->resp == 2)
addReply(c,shared.mbulkhdr[3]);
else
......@@ -112,12 +114,15 @@ void addReplyPubsubMessage(client *c, robj *channel, robj *msg, robj *message_bu
addReply(c,message_bulk);
addReplyBulk(c,channel);
if (msg) addReplyBulk(c,msg);
if (!(old_flags & CLIENT_PUSHING)) c->flags &= ~CLIENT_PUSHING;
}
/* Send a pubsub message of type "pmessage" to the client. The difference
* with the "message" type delivered by addReplyPubsubMessage() is that
* this message format also includes the pattern that matched the message. */
void addReplyPubsubPatMessage(client *c, robj *pat, robj *channel, robj *msg) {
uint64_t old_flags = c->flags;
c->flags |= CLIENT_PUSHING;
if (c->resp == 2)
addReply(c,shared.mbulkhdr[4]);
else
......@@ -126,10 +131,13 @@ void addReplyPubsubPatMessage(client *c, robj *pat, robj *channel, robj *msg) {
addReplyBulk(c,pat);
addReplyBulk(c,channel);
addReplyBulk(c,msg);
if (!(old_flags & CLIENT_PUSHING)) c->flags &= ~CLIENT_PUSHING;
}
/* Send the pubsub subscription notification to the client. */
void addReplyPubsubSubscribed(client *c, robj *channel, pubsubtype type) {
uint64_t old_flags = c->flags;
c->flags |= CLIENT_PUSHING;
if (c->resp == 2)
addReply(c,shared.mbulkhdr[3]);
else
......@@ -137,6 +145,7 @@ void addReplyPubsubSubscribed(client *c, robj *channel, pubsubtype type) {
addReply(c,*type.subscribeMsg);
addReplyBulk(c,channel);
addReplyLongLong(c,type.subscriptionCount(c));
if (!(old_flags & CLIENT_PUSHING)) c->flags &= ~CLIENT_PUSHING;
}
/* Send the pubsub unsubscription notification to the client.
......@@ -144,6 +153,8 @@ void addReplyPubsubSubscribed(client *c, robj *channel, pubsubtype type) {
* unsubscribe command but there are no channels to unsubscribe from: we
* still send a notification. */
void addReplyPubsubUnsubscribed(client *c, robj *channel, pubsubtype type) {
uint64_t old_flags = c->flags;
c->flags |= CLIENT_PUSHING;
if (c->resp == 2)
addReply(c,shared.mbulkhdr[3]);
else
......@@ -154,10 +165,13 @@ void addReplyPubsubUnsubscribed(client *c, robj *channel, pubsubtype type) {
else
addReplyNull(c);
addReplyLongLong(c,type.subscriptionCount(c));
if (!(old_flags & CLIENT_PUSHING)) c->flags &= ~CLIENT_PUSHING;
}
/* Send the pubsub pattern subscription notification to the client. */
void addReplyPubsubPatSubscribed(client *c, robj *pattern) {
uint64_t old_flags = c->flags;
c->flags |= CLIENT_PUSHING;
if (c->resp == 2)
addReply(c,shared.mbulkhdr[3]);
else
......@@ -165,6 +179,7 @@ void addReplyPubsubPatSubscribed(client *c, robj *pattern) {
addReply(c,shared.psubscribebulk);
addReplyBulk(c,pattern);
addReplyLongLong(c,clientSubscriptionsCount(c));
if (!(old_flags & CLIENT_PUSHING)) c->flags &= ~CLIENT_PUSHING;
}
/* Send the pubsub pattern unsubscription notification to the client.
......@@ -172,6 +187,8 @@ void addReplyPubsubPatSubscribed(client *c, robj *pattern) {
* punsubscribe command but there are no pattern to unsubscribe from: we
* still send a notification. */
void addReplyPubsubPatUnsubscribed(client *c, robj *pattern) {
uint64_t old_flags = c->flags;
c->flags |= CLIENT_PUSHING;
if (c->resp == 2)
addReply(c,shared.mbulkhdr[3]);
else
......@@ -182,6 +199,7 @@ void addReplyPubsubPatUnsubscribed(client *c, robj *pattern) {
else
addReplyNull(c);
addReplyLongLong(c,clientSubscriptionsCount(c));
if (!(old_flags & CLIENT_PUSHING)) c->flags &= ~CLIENT_PUSHING;
}
/*-----------------------------------------------------------------------------
......
......@@ -45,6 +45,7 @@
#include <fcntl.h>
#include <limits.h>
#include <math.h>
#include <termios.h>
#include <hiredis.h>
#ifdef USE_OPENSSL
......@@ -172,6 +173,9 @@ int spectrum_palette_mono[] = {0,233,234,235,237,239,241,243,245,247,249,251,253
int *spectrum_palette;
int spectrum_palette_size;
static int orig_termios_saved = 0;
static struct termios orig_termios; /* To restore terminal at exit.*/
/* Dict Helpers */
static uint64_t dictSdsHash(const void *key);
static int dictSdsKeyCompare(dict *d, const void *key1,
......@@ -267,12 +271,14 @@ static struct config {
int eval_ldb_end; /* Lua debugging session ended. */
int enable_ldb_on_eval; /* Handle manual SCRIPT DEBUG + EVAL commands. */
int last_cmd_type;
redisReply *last_reply;
int verbose;
int set_errcode;
clusterManagerCommand cluster_manager_command;
int no_auth_warning;
int resp2;
int resp2; /* value of 1: specified explicitly with option -2 */
int resp3; /* value of 1: specified explicitly, value of 2: implicit like --json option */
int current_resp3; /* 1 if we have RESP3 right now in the current connection. */
int in_multi;
int pre_multi_dbnum;
} config;
......@@ -335,6 +341,9 @@ static void cliRefreshPrompt(void) {
if (config.in_multi)
prompt = sdscatlen(prompt,"(TX)",4);
if (config.pubsub_mode)
prompt = sdscatfmt(prompt,"(subscribed mode)");
/* Copy the prompt in the static buffer. */
prompt = sdscatlen(prompt,"> ",2);
snprintf(config.prompt,sizeof(config.prompt),"%s",prompt);
......@@ -1016,6 +1025,29 @@ static void freeHintsCallback(void *ptr) {
sdsfree(ptr);
}
/*------------------------------------------------------------------------------
* TTY manipulation
*--------------------------------------------------------------------------- */
/* Restore terminal if we've changed it. */
void cliRestoreTTY(void) {
if (orig_termios_saved)
tcsetattr(STDIN_FILENO, TCSANOW, &orig_termios);
}
/* Put the terminal in "press any key" mode */
static void cliPressAnyKeyTTY(void) {
if (!isatty(STDIN_FILENO)) return;
if (!orig_termios_saved) {
if (tcgetattr(STDIN_FILENO, &orig_termios) == -1) return;
atexit(cliRestoreTTY);
orig_termios_saved = 1;
}
struct termios mode = orig_termios;
mode.c_lflag &= ~(ECHO | ICANON); /* echoing off, canonical off */
tcsetattr(STDIN_FILENO, TCSANOW, &mode);
}
/*------------------------------------------------------------------------------
* Networking / parsing
*--------------------------------------------------------------------------- */
......@@ -1088,6 +1120,7 @@ static int cliSwitchProto(void) {
}
}
freeReplyObject(reply);
config.current_resp3 = 1;
return result;
}
......@@ -1147,6 +1180,9 @@ static int cliConnect(int flags) {
* errors. */
anetKeepAlive(NULL, context->fd, REDIS_CLI_KEEPALIVE_INTERVAL);
/* State of the current connection. */
config.current_resp3 = 0;
/* Do AUTH, select the right DB, switch to RESP3 if needed. */
if (cliAuth(context, config.conn_info.user, config.conn_info.auth) != REDIS_OK)
return REDIS_ERR;
......@@ -1309,6 +1345,8 @@ static sds cliFormatReplyTTY(redisReply *r, char *prefix) {
char numsep;
if (r->type == REDIS_REPLY_SET) numsep = '~';
else if (r->type == REDIS_REPLY_MAP) numsep = '#';
/* TODO: this would be a breaking change for scripts, do that in a major version. */
/* else if (r->type == REDIS_REPLY_PUSH) numsep = '>'; */
else numsep = ')';
snprintf(_prefixfmt,sizeof(_prefixfmt),"%%s%%%ud%c ",idxlen,numsep);
......@@ -1351,6 +1389,25 @@ static sds cliFormatReplyTTY(redisReply *r, char *prefix) {
return out;
}
/* Returns 1 if the reply is a pubsub pushed reply. */
int isPubsubPush(redisReply *r) {
if (r == NULL ||
r->type != (config.current_resp3 ? REDIS_REPLY_PUSH : REDIS_REPLY_ARRAY) ||
r->elements < 3 ||
r->element[0]->type != REDIS_REPLY_STRING)
{
return 0;
}
char *str = r->element[0]->str;
size_t len = r->element[0]->len;
/* Check if it is [p|s][un]subscribe or [p|s]message, but even simpler, we
* just check that it ends with "message" or "subscribe". */
return ((len >= strlen("message") &&
!strcmp(str + len - strlen("message"), "message")) ||
(len >= strlen("subscribe") &&
!strcmp(str + len - strlen("subscribe"), "subscribe")));
}
int isColorTerm(void) {
char *t = getenv("TERM");
return t != NULL && strstr(t,"xterm") != NULL;
......@@ -1656,6 +1713,11 @@ static int cliReadReply(int output_raw_strings) {
sds out = NULL;
int output = 1;
if (config.last_reply) {
freeReplyObject(config.last_reply);
config.last_reply = NULL;
}
if (redisGetReply(context,&_reply) != REDIS_OK) {
if (config.blocking_state_aborted) {
config.blocking_state_aborted = 0;
......@@ -1682,7 +1744,7 @@ static int cliReadReply(int output_raw_strings) {
return REDIS_ERR; /* avoid compiler warning */
}
reply = (redisReply*)_reply;
config.last_reply = reply = (redisReply*)_reply;
config.last_cmd_type = reply->type;
......@@ -1731,10 +1793,74 @@ static int cliReadReply(int output_raw_strings) {
fflush(stdout);
sdsfree(out);
}
freeReplyObject(reply);
return REDIS_OK;
}
/* Simultaneously wait for pubsub messages from redis and input on stdin. */
static void cliWaitForMessagesOrStdin() {
int show_info = config.output != OUTPUT_RAW && (isatty(STDOUT_FILENO) ||
getenv("FAKETTY"));
int use_color = show_info && isColorTerm();
cliPressAnyKeyTTY();
while (config.pubsub_mode) {
/* First check if there are any buffered replies. */
redisReply *reply;
do {
if (redisGetReplyFromReader(context, (void **)&reply) != REDIS_OK) {
cliPrintContextError();
exit(1);
}
if (reply) {
sds out = cliFormatReply(reply, config.output, 0);
fwrite(out,sdslen(out),1,stdout);
fflush(stdout);
sdsfree(out);
}
} while(reply);
/* Wait for input, either on the Redis socket or on stdin. */
struct timeval tv;
fd_set readfds;
FD_ZERO(&readfds);
FD_SET(context->fd, &readfds);
FD_SET(STDIN_FILENO, &readfds);
tv.tv_sec = 5;
tv.tv_usec = 0;
if (show_info) {
if (use_color) printf("\033[1;90m"); /* Bold, bright color. */
printf("Reading messages... (press Ctrl-C to quit or any key to type command)\r");
if (use_color) printf("\033[0m"); /* Reset color. */
fflush(stdout);
}
select(context->fd + 1, &readfds, NULL, NULL, &tv);
if (show_info) {
printf("\033[K"); /* Erase current line */
fflush(stdout);
}
if (config.blocking_state_aborted) {
/* Ctrl-C pressed */
config.blocking_state_aborted = 0;
config.pubsub_mode = 0;
if (cliConnect(CC_FORCE) != REDIS_OK) {
cliPrintContextError();
exit(1);
}
break;
} else if (FD_ISSET(context->fd, &readfds)) {
/* Message from Redis */
if (cliReadReply(0) != REDIS_OK) {
cliPrintContextError();
exit(1);
}
fflush(stdout);
} else if (FD_ISSET(STDIN_FILENO, &readfds)) {
/* Any key pressed */
break;
}
}
cliRestoreTTY();
}
static int cliSendCommand(int argc, char **argv, long repeat) {
char *command = argv[0];
size_t *argvlen;
......@@ -1774,9 +1900,12 @@ static int cliSendCommand(int argc, char **argv, long repeat) {
if (!strcasecmp(command,"shutdown")) config.shutdown = 1;
if (!strcasecmp(command,"monitor")) config.monitor_mode = 1;
if (!strcasecmp(command,"subscribe") ||
!strcasecmp(command,"psubscribe") ||
!strcasecmp(command,"ssubscribe")) config.pubsub_mode = 1;
int is_subscribe = (!strcasecmp(command, "subscribe") ||
!strcasecmp(command, "psubscribe") ||
!strcasecmp(command, "ssubscribe"));
int is_unsubscribe = (!strcasecmp(command, "unsubscribe") ||
!strcasecmp(command, "punsubscribe") ||
!strcasecmp(command, "sunsubscribe"));
if (!strcasecmp(command,"sync") ||
!strcasecmp(command,"psync")) config.slave_mode = 1;
......@@ -1807,6 +1936,7 @@ static int cliSendCommand(int argc, char **argv, long repeat) {
works well with the interval option. */
while(repeat < 0 || repeat-- > 0) {
redisAppendCommandArgv(context,argc,(const char**)argv,argvlen);
if (config.monitor_mode) {
do {
if (cliReadReply(output_raw) != REDIS_OK) {
......@@ -1823,27 +1953,15 @@ static int cliSendCommand(int argc, char **argv, long repeat) {
return REDIS_OK;
}
if (config.pubsub_mode) {
if (config.output != OUTPUT_RAW)
printf("Reading messages... (press Ctrl-C to quit)\n");
int num_expected_pubsub_push = 0;
if (is_subscribe || is_unsubscribe) {
/* When a push callback is set, redisGetReply (hiredis) loops until
* an in-band message is received, but these commands are confirmed
* using push replies only. There is one push reply per channel if
* channels are specified, otherwise at least one. */
num_expected_pubsub_push = argc > 1 ? argc - 1 : 1;
/* Unset our default PUSH handler so this works in RESP2/RESP3 */
redisSetPushCallback(context, NULL);
while (config.pubsub_mode) {
if (cliReadReply(output_raw) != REDIS_OK) {
cliPrintContextError();
exit(1);
}
fflush(stdout); /* Make it grep friendly */
if (!config.pubsub_mode || config.last_cmd_type == REDIS_REPLY_ERROR) {
if (config.push_output) {
redisSetPushCallback(context, cliPushHandler);
}
config.pubsub_mode = 0;
}
}
continue;
}
if (config.slave_mode) {
......@@ -1854,10 +1972,35 @@ static int cliSendCommand(int argc, char **argv, long repeat) {
return REDIS_ERR; /* Error = slaveMode lost connection to master */
}
/* Read response, possibly skipping pubsub/push messages. */
while (1) {
if (cliReadReply(output_raw) != REDIS_OK) {
zfree(argvlen);
return REDIS_ERR;
}
fflush(stdout);
if (config.pubsub_mode || num_expected_pubsub_push > 0) {
if (isPubsubPush(config.last_reply)) {
if (num_expected_pubsub_push > 0 &&
!strcasecmp(config.last_reply->element[0]->str, command))
{
/* This pushed message confirms the
* [p|s][un]subscribe command. */
if (is_subscribe && !config.pubsub_mode) {
config.pubsub_mode = 1;
cliRefreshPrompt();
}
if (--num_expected_pubsub_push > 0) {
continue; /* We need more of these. */
}
} else {
continue; /* Skip this pubsub message. */
}
} else if (config.last_reply->type == REDIS_REPLY_PUSH) {
continue; /* Skip other push message. */
}
}
/* Store database number when SELECT was successfully executed. */
if (!strcasecmp(command,"select") && argc == 2 &&
config.last_cmd_type != REDIS_REPLY_ERROR)
......@@ -1891,9 +2034,25 @@ static int cliSendCommand(int argc, char **argv, long repeat) {
config.in_multi = 0;
config.dbnum = 0;
config.conn_info.input_dbnum = 0;
config.resp3 = 0;
config.current_resp3 = 0;
if (config.pubsub_mode && config.push_output) {
redisSetPushCallback(context, cliPushHandler);
}
config.pubsub_mode = 0;
cliRefreshPrompt();
} else if (!strcasecmp(command,"hello")) {
if (config.last_cmd_type == REDIS_REPLY_MAP) {
config.current_resp3 = 1;
} else if (config.last_cmd_type == REDIS_REPLY_ARRAY) {
config.current_resp3 = 0;
}
} else if ((is_subscribe || is_unsubscribe) && !config.pubsub_mode) {
/* We didn't enter pubsub mode. Restore push callback. */
if (config.push_output)
redisSetPushCallback(context, cliPushHandler);
}
break;
}
if (config.cluster_reissue_command){
/* If we need to reissue the command, break to prevent a
......@@ -2644,8 +2803,17 @@ static void repl(void) {
}
cliRefreshPrompt();
while((line = linenoise(context ? config.prompt : "not connected> ")) != NULL) {
if (line[0] != '\0') {
while(1) {
line = linenoise(context ? config.prompt : "not connected> ");
if (line == NULL) {
/* ^C, ^D or similar. */
if (config.pubsub_mode) {
config.pubsub_mode = 0;
if (cliConnect(CC_FORCE) == REDIS_OK)
continue;
}
break;
} else if (line[0] != '\0') {
long repeat = 1;
int skipargs = 0;
char *endptr = NULL;
......@@ -2739,6 +2907,11 @@ static void repl(void) {
/* Free the argument vector */
sdsfreesplitres(argv,argc);
}
if (config.pubsub_mode) {
cliWaitForMessagesOrStdin();
}
/* linenoise() returns malloc-ed lines like readline() */
linenoiseFree(line);
}
......@@ -2779,6 +2952,13 @@ static int noninteractive(int argc, char **argv) {
retval = issueCommand(argc, sds_args);
sdsfreesplitres(sds_args, argc);
while (config.pubsub_mode) {
if (cliReadReply(0) != REDIS_OK) {
cliPrintContextError();
exit(1);
}
fflush(stdout);
}
return retval == REDIS_OK ? 0 : 1;
}
......@@ -8991,6 +9171,7 @@ int main(int argc, char **argv) {
config.eval_ldb_sync = 0;
config.enable_ldb_on_eval = 0;
config.last_cmd_type = -1;
config.last_reply = NULL;
config.verbose = 0;
config.set_errcode = 0;
config.no_auth_warning = 0;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment