Commit d7971f96 authored by Oran Agra's avatar Oran Agra
Browse files

Merge remote-tracking branch 'origin/unstable' into 7.0

parents d2b5a579 acfb4f7a
...@@ -482,10 +482,12 @@ void debugCommand(client *c) { ...@@ -482,10 +482,12 @@ void debugCommand(client *c) {
" Show low level client eviction pools info (maxmemory-clients).", " Show low level client eviction pools info (maxmemory-clients).",
"PAUSE-CRON <0|1>", "PAUSE-CRON <0|1>",
" Stop periodic cron job processing.", " Stop periodic cron job processing.",
"REPLYBUFFER-PEAK-RESET-TIME <NEVER||RESET|time>", "REPLYBUFFER PEAK-RESET-TIME <NEVER||RESET|time>",
" Sets the time (in milliseconds) to wait between client reply buffer peak resets.", " Sets the time (in milliseconds) to wait between client reply buffer peak resets.",
" In case NEVER is provided the last observed peak will never be reset", " In case NEVER is provided the last observed peak will never be reset",
" In case RESET is provided the peak reset time will be restored to the default value", " In case RESET is provided the peak reset time will be restored to the default value",
"REPLYBUFFER RESIZING <0|1>",
" Enable or disable the replay buffer resize cron job",
NULL NULL
}; };
addReplyHelp(c, help); addReplyHelp(c, help);
...@@ -793,6 +795,10 @@ NULL ...@@ -793,6 +795,10 @@ NULL
* also have a normal reply type after the attribute. */ * also have a normal reply type after the attribute. */
addReplyBulkCString(c,"Some real reply following the attribute"); addReplyBulkCString(c,"Some real reply following the attribute");
} else if (!strcasecmp(name,"push")) { } else if (!strcasecmp(name,"push")) {
if (c->resp < 3) {
addReplyError(c,"RESP2 is not supported by this command");
return;
}
addReplyPushLen(c,2); addReplyPushLen(c,2);
addReplyBulkCString(c,"server-cpu-usage"); addReplyBulkCString(c,"server-cpu-usage");
addReplyLongLong(c,42); addReplyLongLong(c,42);
...@@ -962,13 +968,20 @@ NULL ...@@ -962,13 +968,20 @@ NULL
{ {
server.pause_cron = atoi(c->argv[2]->ptr); server.pause_cron = atoi(c->argv[2]->ptr);
addReply(c,shared.ok); addReply(c,shared.ok);
} else if (!strcasecmp(c->argv[1]->ptr,"replybuffer-peak-reset-time") && c->argc == 3 ) { } else if (!strcasecmp(c->argv[1]->ptr,"replybuffer") && c->argc == 4 ) {
if (!strcasecmp(c->argv[2]->ptr, "never")) { if(!strcasecmp(c->argv[2]->ptr, "peak-reset-time")) {
if (!strcasecmp(c->argv[3]->ptr, "never")) {
server.reply_buffer_peak_reset_time = -1; server.reply_buffer_peak_reset_time = -1;
} else if(!strcasecmp(c->argv[2]->ptr, "reset")) { } else if(!strcasecmp(c->argv[3]->ptr, "reset")) {
server.reply_buffer_peak_reset_time = REPLY_BUFFER_DEFAULT_PEAK_RESET_TIME; server.reply_buffer_peak_reset_time = REPLY_BUFFER_DEFAULT_PEAK_RESET_TIME;
} else { } else {
if (getLongFromObjectOrReply(c, c->argv[2], &server.reply_buffer_peak_reset_time, NULL) != C_OK) if (getLongFromObjectOrReply(c, c->argv[3], &server.reply_buffer_peak_reset_time, NULL) != C_OK)
return;
}
} else if(!strcasecmp(c->argv[2]->ptr,"resizing")) {
server.reply_buffer_resizing_enabled = atoi(c->argv[3]->ptr);
} else {
addReplySubcommandSyntaxError(c);
return; return;
} }
addReply(c, shared.ok); addReply(c, shared.ok);
...@@ -1943,7 +1956,7 @@ void sigsegvHandler(int sig, siginfo_t *info, void *secret) { ...@@ -1943,7 +1956,7 @@ void sigsegvHandler(int sig, siginfo_t *info, void *secret) {
serverLog(LL_WARNING, serverLog(LL_WARNING,
"Accessing address: %p", (void*)info->si_addr); "Accessing address: %p", (void*)info->si_addr);
} }
if (info->si_code <= SI_USER && info->si_pid != -1) { if (info->si_code == SI_USER && info->si_pid != -1) {
serverLog(LL_WARNING, "Killed by PID: %ld, UID: %d", (long) info->si_pid, info->si_uid); serverLog(LL_WARNING, "Killed by PID: %ld, UID: %d", (long) info->si_pid, info->si_uid);
} }
......
...@@ -407,7 +407,7 @@ long activeDefragSdsListAndDict(list *l, dict *d, int dict_val_type) { ...@@ -407,7 +407,7 @@ long activeDefragSdsListAndDict(list *l, dict *d, int dict_val_type) {
* new pointer. Additionally, we try to defrag the dictEntry in that dict. * new pointer. Additionally, we try to defrag the dictEntry in that dict.
* Oldkey mey be a dead pointer and should not be accessed (we get a * Oldkey mey be a dead pointer and should not be accessed (we get a
* pre-calculated hash value). Newkey may be null if the key pointer wasn't * pre-calculated hash value). Newkey may be null if the key pointer wasn't
* moved. Return value is the the dictEntry if found, or NULL if not found. * moved. Return value is the dictEntry if found, or NULL if not found.
* NOTE: this is very ugly code, but it let's us avoid the complication of * NOTE: this is very ugly code, but it let's us avoid the complication of
* doing a scan on another dict. */ * doing a scan on another dict. */
dictEntry* replaceSatelliteDictKeyPtrAndOrDefragDictEntry(dict *d, sds oldkey, sds newkey, uint64_t hash, long *defragged) { dictEntry* replaceSatelliteDictKeyPtrAndOrDefragDictEntry(dict *d, sds oldkey, sds newkey, uint64_t hash, long *defragged) {
...@@ -1196,7 +1196,7 @@ void activeDefragCycle(void) { ...@@ -1196,7 +1196,7 @@ void activeDefragCycle(void) {
cursor = dictScan(db->dict, cursor, defragScanCallback, defragDictBucketCallback, db); cursor = dictScan(db->dict, cursor, defragScanCallback, defragDictBucketCallback, db);
/* Once in 16 scan iterations, 512 pointer reallocations. or 64 keys /* Once in 16 scan iterations, 512 pointer reallocations. or 64 keys
* (if we have a lot of pointers in one hash bucket or rehasing), * (if we have a lot of pointers in one hash bucket or rehashing),
* check if we reached the time limit. * check if we reached the time limit.
* But regardless, don't start a new db in this loop, this is because after * But regardless, don't start a new db in this loop, this is because after
* the last db we call defragOtherGlobals, which must be done in one cycle */ * the last db we call defragOtherGlobals, which must be done in one cycle */
......
...@@ -508,6 +508,7 @@ void evalGenericCommand(client *c, int evalsha) { ...@@ -508,6 +508,7 @@ void evalGenericCommand(client *c, int evalsha) {
scriptRunCtx rctx; scriptRunCtx rctx;
if (scriptPrepareForRun(&rctx, lctx.lua_client, c, lua_cur_script, l->flags, ro) != C_OK) { if (scriptPrepareForRun(&rctx, lctx.lua_client, c, lua_cur_script, l->flags, ro) != C_OK) {
lua_pop(lua,2); /* Remove the function and error handler. */
return; return;
} }
rctx.flags |= SCRIPT_EVAL_MODE; /* mark the current run as EVAL (as opposed to FCALL) so we'll rctx.flags |= SCRIPT_EVAL_MODE; /* mark the current run as EVAL (as opposed to FCALL) so we'll
......
...@@ -57,6 +57,12 @@ struct functionsLibCtx { ...@@ -57,6 +57,12 @@ struct functionsLibCtx {
dict *engines_stats; /* Per engine statistics */ dict *engines_stats; /* Per engine statistics */
}; };
typedef struct functionsLibMataData {
sds engine;
sds name;
sds code;
} functionsLibMataData;
dictType engineDictType = { dictType engineDictType = {
dictSdsCaseHash, /* hash function */ dictSdsCaseHash, /* hash function */
dictSdsDup, /* key dup */ dictSdsDup, /* key dup */
...@@ -124,7 +130,6 @@ static size_t functionMallocSize(functionInfo *fi) { ...@@ -124,7 +130,6 @@ static size_t functionMallocSize(functionInfo *fi) {
static size_t libraryMallocSize(functionLibInfo *li) { static size_t libraryMallocSize(functionLibInfo *li) {
return zmalloc_size(li) + sdsZmallocSize(li->name) return zmalloc_size(li) + sdsZmallocSize(li->name)
+ (li->desc ? sdsZmallocSize(li->desc) : 0)
+ sdsZmallocSize(li->code); + sdsZmallocSize(li->code);
} }
...@@ -157,7 +162,6 @@ static void engineLibraryFree(functionLibInfo* li) { ...@@ -157,7 +162,6 @@ static void engineLibraryFree(functionLibInfo* li) {
dictRelease(li->functions); dictRelease(li->functions);
sdsfree(li->name); sdsfree(li->name);
sdsfree(li->code); sdsfree(li->code);
if (li->desc) sdsfree(li->desc);
zfree(li); zfree(li);
} }
...@@ -265,14 +269,13 @@ int functionLibCreateFunction(sds name, void *function, functionLibInfo *li, sds ...@@ -265,14 +269,13 @@ int functionLibCreateFunction(sds name, void *function, functionLibInfo *li, sds
return C_OK; return C_OK;
} }
static functionLibInfo* engineLibraryCreate(sds name, engineInfo *ei, sds desc, sds code) { static functionLibInfo* engineLibraryCreate(sds name, engineInfo *ei, sds code) {
functionLibInfo *li = zmalloc(sizeof(*li)); functionLibInfo *li = zmalloc(sizeof(*li));
*li = (functionLibInfo) { *li = (functionLibInfo) {
.name = sdsdup(name), .name = sdsdup(name),
.functions = dictCreate(&libraryFunctionDictType), .functions = dictCreate(&libraryFunctionDictType),
.ei = ei, .ei = ei,
.code = sdsdup(code), .code = sdsdup(code),
.desc = desc ? sdsdup(desc) : NULL,
}; };
return li; return li;
} }
...@@ -540,17 +543,11 @@ void functionListCommand(client *c) { ...@@ -540,17 +543,11 @@ void functionListCommand(client *c) {
} }
} }
++reply_len; ++reply_len;
addReplyMapLen(c, with_code? 5 : 4); addReplyMapLen(c, with_code? 4 : 3);
addReplyBulkCString(c, "library_name"); addReplyBulkCString(c, "library_name");
addReplyBulkCBuffer(c, li->name, sdslen(li->name)); addReplyBulkCBuffer(c, li->name, sdslen(li->name));
addReplyBulkCString(c, "engine"); addReplyBulkCString(c, "engine");
addReplyBulkCBuffer(c, li->ei->name, sdslen(li->ei->name)); addReplyBulkCBuffer(c, li->ei->name, sdslen(li->ei->name));
addReplyBulkCString(c, "description");
if (li->desc) {
addReplyBulkCBuffer(c, li->desc, sdslen(li->desc));
} else {
addReplyNull(c);
}
addReplyBulkCString(c, "functions"); addReplyBulkCString(c, "functions");
addReplyArrayLen(c, dictSize(li->functions)); addReplyArrayLen(c, dictSize(li->functions));
...@@ -745,11 +742,11 @@ void functionRestoreCommand(client *c) { ...@@ -745,11 +742,11 @@ void functionRestoreCommand(client *c) {
err = sdsnew("can not read data type"); err = sdsnew("can not read data type");
goto load_error; goto load_error;
} }
if (type != RDB_OPCODE_FUNCTION) { if (type != RDB_OPCODE_FUNCTION && type != RDB_OPCODE_FUNCTION2) {
err = sdsnew("given type is not a function"); err = sdsnew("given type is not a function");
goto load_error; goto load_error;
} }
if (rdbFunctionLoad(&payload, rdbver, functions_lib_ctx, RDBFLAGS_NONE, &err) != C_OK) { if (rdbFunctionLoad(&payload, rdbver, functions_lib_ctx, type, RDBFLAGS_NONE, &err) != C_OK) {
if (!err) { if (!err) {
err = sdsnew("failed loading the given functions payload"); err = sdsnew("failed loading the given functions payload");
} }
...@@ -868,36 +865,111 @@ static int functionsVerifyName(sds name) { ...@@ -868,36 +865,111 @@ static int functionsVerifyName(sds name) {
return C_OK; return C_OK;
} }
/* Compile and save the given library, return C_OK on success and C_ERR on failure. int functionExtractLibMetaData(sds payload, functionsLibMataData *md, sds *err) {
* In case on failure the err out param is set with relevant error message */ sds name = NULL;
int functionsCreateWithLibraryCtx(sds lib_name,sds engine_name, sds desc, sds code, sds desc = NULL;
int replace, sds* err, functionsLibCtx *lib_ctx) { sds engine = NULL;
sds code = NULL;
if (strncmp(payload, "#!", 2) != 0) {
*err = sdsnew("Missing library metadata");
return C_ERR;
}
char *shebang_end = strchr(payload, '\n');
if (shebang_end == NULL) {
*err = sdsnew("Invalid library metadata");
return C_ERR;
}
size_t shebang_len = shebang_end - payload;
sds shebang = sdsnewlen(payload, shebang_len);
int numparts;
sds *parts = sdssplitargs(shebang, &numparts);
sdsfree(shebang);
if (!parts || numparts == 0) {
*err = sdsnew("Invalid library metadata");
sdsfreesplitres(parts, numparts);
return C_ERR;
}
engine = sdsdup(parts[0]);
sdsrange(engine, 2, -1);
for (int i = 1 ; i < numparts ; ++i) {
sds part = parts[i];
if (strncasecmp(part, "name=", 5) == 0) {
if (name) {
*err = sdscatfmt(sdsempty(), "Invalid metadata value, name argument was given multiple times");
goto error;
}
name = sdsdup(part);
sdsrange(name, 5, -1);
continue;
}
*err = sdscatfmt(sdsempty(), "Invalid metadata value given: %s", part);
goto error;
}
if (!name) {
*err = sdsnew("Library name was not given");
goto error;
}
sdsfreesplitres(parts, numparts);
md->name = name;
md->code = sdsnewlen(shebang_end, sdslen(payload) - shebang_len);
md->engine = engine;
return C_OK;
error:
if (name) sdsfree(name);
if (desc) sdsfree(desc);
if (engine) sdsfree(engine);
if (code) sdsfree(code);
sdsfreesplitres(parts, numparts);
return C_ERR;
}
void functionFreeLibMetaData(functionsLibMataData *md) {
if (md->code) sdsfree(md->code);
if (md->name) sdsfree(md->name);
if (md->engine) sdsfree(md->engine);
}
/* Compile and save the given library, return the loaded library name on success
* and NULL on failure. In case on failure the err out param is set with relevant error message */
sds functionsCreateWithLibraryCtx(sds code, int replace, sds* err, functionsLibCtx *lib_ctx) {
dictIterator *iter = NULL; dictIterator *iter = NULL;
dictEntry *entry = NULL; dictEntry *entry = NULL;
if (functionsVerifyName(lib_name)) { functionLibInfo *new_li = NULL;
functionLibInfo *old_li = NULL;
functionsLibMataData md = {0};
if (functionExtractLibMetaData(code, &md, err) != C_OK) {
return NULL;
}
if (functionsVerifyName(md.name)) {
*err = sdsnew("Library names can only contain letters and numbers and must be at least one character long"); *err = sdsnew("Library names can only contain letters and numbers and must be at least one character long");
return C_ERR; goto error;
} }
engineInfo *ei = dictFetchValue(engines, engine_name); engineInfo *ei = dictFetchValue(engines, md.engine);
if (!ei) { if (!ei) {
*err = sdsnew("Engine not found"); *err = sdscatfmt(sdsempty(), "Engine '%S' not found", md.engine);
return C_ERR; goto error;
} }
engine *engine = ei->engine; engine *engine = ei->engine;
functionLibInfo *old_li = dictFetchValue(lib_ctx->libraries, lib_name); old_li = dictFetchValue(lib_ctx->libraries, md.name);
if (old_li && !replace) { if (old_li && !replace) {
*err = sdsnew("Library already exists"); *err = sdscatfmt(sdsempty(), "Library '%S' already exists", md.name);
return C_ERR; goto error;
} }
if (old_li) { if (old_li) {
libraryUnlink(lib_ctx, old_li); libraryUnlink(lib_ctx, old_li);
} }
functionLibInfo *new_li = engineLibraryCreate(lib_name, ei, desc, code); new_li = engineLibraryCreate(md.name, ei, code);
if (engine->create(engine->engine_ctx, new_li, code, err) != C_OK) { if (engine->create(engine->engine_ctx, new_li, md.code, err) != C_OK) {
goto error; goto error;
} }
...@@ -925,48 +997,34 @@ int functionsCreateWithLibraryCtx(sds lib_name,sds engine_name, sds desc, sds co ...@@ -925,48 +997,34 @@ int functionsCreateWithLibraryCtx(sds lib_name,sds engine_name, sds desc, sds co
engineLibraryFree(old_li); engineLibraryFree(old_li);
} }
return C_OK; sds loaded_lib_name = md.name;
md.name = NULL;
functionFreeLibMetaData(&md);
return loaded_lib_name;
error: error:
if (iter) dictReleaseIterator(iter); if (iter) dictReleaseIterator(iter);
engineLibraryFree(new_li); if (new_li) engineLibraryFree(new_li);
if (old_li) { if (old_li) libraryLink(lib_ctx, old_li);
libraryLink(lib_ctx, old_li); functionFreeLibMetaData(&md);
} return NULL;
return C_ERR;
} }
/* /*
* FUNCTION LOAD <ENGINE NAME> <LIBRARY NAME> * FUNCTION LOAD [REPLACE] <LIBRARY CODE>
* [REPLACE] [DESC <LIBRARY DESCRIPTION>] <LIBRARY CODE>
*
* ENGINE NAME - name of the engine to use the run the library
* LIBRARY NAME - name of the library
* REPLACE - optional, replace existing library * REPLACE - optional, replace existing library
* DESCRIPTION - optional, library description
* LIBRARY CODE - library code to pass to the engine * LIBRARY CODE - library code to pass to the engine
*/ */
void functionLoadCommand(client *c) { void functionLoadCommand(client *c) {
robj *engine_name = c->argv[2];
robj *library_name = c->argv[3];
int replace = 0; int replace = 0;
int argc_pos = 4; int argc_pos = 2;
sds desc = NULL;
while (argc_pos < c->argc - 1) { while (argc_pos < c->argc - 1) {
robj *next_arg = c->argv[argc_pos++]; robj *next_arg = c->argv[argc_pos++];
if (!strcasecmp(next_arg->ptr, "replace")) { if (!strcasecmp(next_arg->ptr, "replace")) {
replace = 1; replace = 1;
continue; continue;
} }
if (!strcasecmp(next_arg->ptr, "description")) {
if (argc_pos >= c->argc) {
addReplyError(c, "Bad function description");
return;
}
desc = c->argv[argc_pos++]->ptr;
continue;
}
addReplyErrorFormat(c, "Unknown option given: %s", (char*)next_arg->ptr); addReplyErrorFormat(c, "Unknown option given: %s", (char*)next_arg->ptr);
return; return;
} }
...@@ -978,8 +1036,8 @@ void functionLoadCommand(client *c) { ...@@ -978,8 +1036,8 @@ void functionLoadCommand(client *c) {
robj *code = c->argv[argc_pos]; robj *code = c->argv[argc_pos];
sds err = NULL; sds err = NULL;
if (functionsCreateWithLibraryCtx(library_name->ptr, engine_name->ptr, sds library_name = NULL;
desc, code->ptr, replace, &err, curr_functions_lib_ctx) != C_OK) if (!(library_name = functionsCreateWithLibraryCtx(code->ptr, replace, &err, curr_functions_lib_ctx)))
{ {
addReplyErrorSds(c, err); addReplyErrorSds(c, err);
return; return;
...@@ -987,7 +1045,7 @@ void functionLoadCommand(client *c) { ...@@ -987,7 +1045,7 @@ void functionLoadCommand(client *c) {
/* Indicate that the command changed the data so it will be replicated and /* Indicate that the command changed the data so it will be replicated and
* counted as a data change (for persistence configuration) */ * counted as a data change (for persistence configuration) */
server.dirty++; server.dirty++;
addReply(c, shared.ok); addReplyBulkSds(c, library_name);
} }
/* Return memory usage of all the engines combine */ /* Return memory usage of all the engines combine */
......
...@@ -106,12 +106,10 @@ struct functionLibInfo { ...@@ -106,12 +106,10 @@ struct functionLibInfo {
dict *functions; /* Functions dictionary */ dict *functions; /* Functions dictionary */
engineInfo *ei; /* Pointer to the function engine */ engineInfo *ei; /* Pointer to the function engine */
sds code; /* Library code */ sds code; /* Library code */
sds desc; /* Library description */
}; };
int functionsRegisterEngine(const char *engine_name, engine *engine_ctx); int functionsRegisterEngine(const char *engine_name, engine *engine_ctx);
int functionsCreateWithLibraryCtx(sds lib_name, sds engine_name, sds desc, sds code, sds functionsCreateWithLibraryCtx(sds code, int replace, sds* err, functionsLibCtx *lib_ctx);
int replace, sds* err, functionsLibCtx *lib_ctx);
unsigned long functionsMemory(); unsigned long functionsMemory();
unsigned long functionsMemoryOverhead(); unsigned long functionsMemoryOverhead();
unsigned long functionsNum(); unsigned long functionsNum();
......
...@@ -46,7 +46,7 @@ ...@@ -46,7 +46,7 @@
/* Interleave lower bits of x and y, so the bits of x /* Interleave lower bits of x and y, so the bits of x
* are in the even positions and bits from y in the odd; * are in the even positions and bits from y in the odd;
* x and y must initially be less than 2**32 (65536). * x and y must initially be less than 2**32 (4294967296).
* From: https://graphics.stanford.edu/~seander/bithacks.html#InterleaveBMN * From: https://graphics.stanford.edu/~seander/bithacks.html#InterleaveBMN
*/ */
static inline uint64_t interleave64(uint32_t xlo, uint32_t ylo) { static inline uint64_t interleave64(uint32_t xlo, uint32_t ylo) {
......
...@@ -399,6 +399,11 @@ struct commandHelp { ...@@ -399,6 +399,11 @@ struct commandHelp {
"Bind a hash slot to a specific node", "Bind a hash slot to a specific node",
12, 12,
"3.0.0" }, "3.0.0" },
{ "CLUSTER SHARDS",
"",
"Get array of cluster slots to node mappings",
12,
"7.0.0" },
{ "CLUSTER SLAVES", { "CLUSTER SLAVES",
"node-id", "node-id",
"List replica nodes of the specified master node", "List replica nodes of the specified master node",
...@@ -431,7 +436,7 @@ struct commandHelp { ...@@ -431,7 +436,7 @@ struct commandHelp {
"2.8.13" }, "2.8.13" },
{ "COMMAND GETKEYSANDFLAGS", { "COMMAND GETKEYSANDFLAGS",
"", "",
"Extract keys given a full Redis command", "Extract keys and access flags given a full Redis command",
9, 9,
"7.0.0" }, "7.0.0" },
{ "COMMAND HELP", { "COMMAND HELP",
...@@ -630,7 +635,7 @@ struct commandHelp { ...@@ -630,7 +635,7 @@ struct commandHelp {
10, 10,
"7.0.0" }, "7.0.0" },
{ "FUNCTION LOAD", { "FUNCTION LOAD",
"engine-name library-name [REPLACE] [DESCRIPTION library-description] function-code", "[REPLACE] function-code",
"Create a function with the given arguments (name, code, description)", "Create a function with the given arguments (name, code, description)",
10, 10,
"7.0.0" }, "7.0.0" },
...@@ -1019,6 +1024,11 @@ struct commandHelp { ...@@ -1019,6 +1024,11 @@ struct commandHelp {
"Load a module", "Load a module",
9, 9,
"4.0.0" }, "4.0.0" },
{ "MODULE LOADEX",
"path [CONFIG name value [name value ...]] [ARGS arg [arg ...]]",
"Load a module with extended parameters",
9,
"7.0.0" },
{ "MODULE UNLOAD", { "MODULE UNLOAD",
"name", "name",
"Unload a module", "Unload a module",
......
...@@ -958,7 +958,7 @@ unsigned char *lpPrependInteger(unsigned char *lp, long long lval) { ...@@ -958,7 +958,7 @@ unsigned char *lpPrependInteger(unsigned char *lp, long long lval) {
return lpInsertInteger(lp, lval, p, LP_BEFORE, NULL); return lpInsertInteger(lp, lval, p, LP_BEFORE, NULL);
} }
/* Append the specified element 'ele' of length 'len' at the end of the /* Append the specified element 'ele' of length 'size' at the end of the
* listpack. It is implemented in terms of lpInsert(), so the return value is * listpack. It is implemented in terms of lpInsert(), so the return value is
* the same as lpInsert(). */ * the same as lpInsert(). */
unsigned char *lpAppend(unsigned char *lp, unsigned char *ele, uint32_t size) { unsigned char *lpAppend(unsigned char *lp, unsigned char *ele, uint32_t size) {
......
...@@ -108,7 +108,7 @@ void nolocks_localtime(struct tm *tmp, time_t t, time_t tz, int dst) { ...@@ -108,7 +108,7 @@ void nolocks_localtime(struct tm *tmp, time_t t, time_t tz, int dst) {
int main(void) { int main(void) {
/* Obtain timezone and daylight info. */ /* Obtain timezone and daylight info. */
tzset(); /* Now 'timezome' global is populated. */ tzset(); /* Now 'timezone' global is populated. */
time_t t = time(NULL); time_t t = time(NULL);
struct tm *aux = localtime(&t); struct tm *aux = localtime(&t);
int daylight_active = aux->tm_isdst; int daylight_active = aux->tm_isdst;
......
...@@ -352,6 +352,9 @@ typedef struct RedisModuleServerInfoData { ...@@ -352,6 +352,9 @@ typedef struct RedisModuleServerInfoData {
#define REDISMODULE_ARGV_RESP_3 (1<<3) #define REDISMODULE_ARGV_RESP_3 (1<<3)
#define REDISMODULE_ARGV_RESP_AUTO (1<<4) #define REDISMODULE_ARGV_RESP_AUTO (1<<4)
#define REDISMODULE_ARGV_CHECK_ACL (1<<5) #define REDISMODULE_ARGV_CHECK_ACL (1<<5)
#define REDISMODULE_ARGV_SCRIPT_MODE (1<<6)
#define REDISMODULE_ARGV_NO_WRITES (1<<7)
#define REDISMODULE_ARGV_CALL_REPLIES_AS_ERRORS (1<<8)
/* Determine whether Redis should signalModifiedKey implicitly. /* Determine whether Redis should signalModifiedKey implicitly.
* In case 'ctx' has no 'module' member (and therefore no module->options), * In case 'ctx' has no 'module' member (and therefore no module->options),
...@@ -393,6 +396,40 @@ typedef struct RedisModuleKeyOptCtx { ...@@ -393,6 +396,40 @@ typedef struct RedisModuleKeyOptCtx {
as `copy2`, 'from_dbid' and 'to_dbid' are both valid. */ as `copy2`, 'from_dbid' and 'to_dbid' are both valid. */
} RedisModuleKeyOptCtx; } RedisModuleKeyOptCtx;
/* Data structures related to redis module configurations */
/* The function signatures for module config get callbacks. These are identical to the ones exposed in redismodule.h. */
typedef RedisModuleString * (*RedisModuleConfigGetStringFunc)(const char *name, void *privdata);
typedef long long (*RedisModuleConfigGetNumericFunc)(const char *name, void *privdata);
typedef int (*RedisModuleConfigGetBoolFunc)(const char *name, void *privdata);
typedef int (*RedisModuleConfigGetEnumFunc)(const char *name, void *privdata);
/* The function signatures for module config set callbacks. These are identical to the ones exposed in redismodule.h. */
typedef int (*RedisModuleConfigSetStringFunc)(const char *name, RedisModuleString *val, void *privdata, RedisModuleString **err);
typedef int (*RedisModuleConfigSetNumericFunc)(const char *name, long long val, void *privdata, RedisModuleString **err);
typedef int (*RedisModuleConfigSetBoolFunc)(const char *name, int val, void *privdata, RedisModuleString **err);
typedef int (*RedisModuleConfigSetEnumFunc)(const char *name, int val, void *privdata, RedisModuleString **err);
/* Apply signature, identical to redismodule.h */
typedef int (*RedisModuleConfigApplyFunc)(RedisModuleCtx *ctx, void *privdata, RedisModuleString **err);
/* Struct representing a module config. These are stored in a list in the module struct */
struct ModuleConfig {
sds name; /* Name of config without the module name appended to the front */
void *privdata; /* Optional data passed into the module config callbacks */
union get_fn { /* The get callback specified by the module */
RedisModuleConfigGetStringFunc get_string;
RedisModuleConfigGetNumericFunc get_numeric;
RedisModuleConfigGetBoolFunc get_bool;
RedisModuleConfigGetEnumFunc get_enum;
} get_fn;
union set_fn { /* The set callback specified by the module */
RedisModuleConfigSetStringFunc set_string;
RedisModuleConfigSetNumericFunc set_numeric;
RedisModuleConfigSetBoolFunc set_bool;
RedisModuleConfigSetEnumFunc set_enum;
} set_fn;
RedisModuleConfigApplyFunc apply_fn;
RedisModule *module;
};
/* -------------------------------------------------------------------------- /* --------------------------------------------------------------------------
* Prototypes * Prototypes
* -------------------------------------------------------------------------- */ * -------------------------------------------------------------------------- */
...@@ -596,7 +633,10 @@ static void moduleFreeKeyIterator(RedisModuleKey *key) { ...@@ -596,7 +633,10 @@ static void moduleFreeKeyIterator(RedisModuleKey *key) {
serverAssert(key->iter != NULL); serverAssert(key->iter != NULL);
switch (key->value->type) { switch (key->value->type) {
case OBJ_LIST: listTypeReleaseIterator(key->iter); break; case OBJ_LIST: listTypeReleaseIterator(key->iter); break;
case OBJ_STREAM: zfree(key->iter); break; case OBJ_STREAM:
streamIteratorStop(key->iter);
zfree(key->iter);
break;
default: serverAssert(0); /* No key->iter for other types. */ default: serverAssert(0); /* No key->iter for other types. */
} }
key->iter = NULL; key->iter = NULL;
...@@ -1935,6 +1975,16 @@ int moduleIsModuleCommand(void *module_handle, struct redisCommand *cmd) { ...@@ -1935,6 +1975,16 @@ int moduleIsModuleCommand(void *module_handle, struct redisCommand *cmd) {
* ## Module information and time measurement * ## Module information and time measurement
* -------------------------------------------------------------------------- */ * -------------------------------------------------------------------------- */
int moduleListConfigMatch(void *config, void *name) {
return strcasecmp(((ModuleConfig *) config)->name, (char *) name) == 0;
}
void moduleListFree(void *config) {
ModuleConfig *module_config = (ModuleConfig *) config;
sdsfree(module_config->name);
zfree(config);
}
void RM_SetModuleAttribs(RedisModuleCtx *ctx, const char *name, int ver, int apiver) { void RM_SetModuleAttribs(RedisModuleCtx *ctx, const char *name, int ver, int apiver) {
/* Called by RM_Init() to setup the `ctx->module` structure. /* Called by RM_Init() to setup the `ctx->module` structure.
* *
...@@ -1951,7 +2001,11 @@ void RM_SetModuleAttribs(RedisModuleCtx *ctx, const char *name, int ver, int api ...@@ -1951,7 +2001,11 @@ void RM_SetModuleAttribs(RedisModuleCtx *ctx, const char *name, int ver, int api
module->usedby = listCreate(); module->usedby = listCreate();
module->using = listCreate(); module->using = listCreate();
module->filters = listCreate(); module->filters = listCreate();
module->module_configs = listCreate();
listSetMatchMethod(module->module_configs, moduleListConfigMatch);
listSetFreeMethod(module->module_configs, moduleListFree);
module->in_call = 0; module->in_call = 0;
module->configs_initialized = 0;
module->in_hook = 0; module->in_hook = 0;
module->options = 0; module->options = 0;
module->info_cb = 0; module->info_cb = 0;
...@@ -2250,7 +2304,7 @@ RedisModuleString *RM_CreateStringFromLongLong(RedisModuleCtx *ctx, long long ll ...@@ -2250,7 +2304,7 @@ RedisModuleString *RM_CreateStringFromLongLong(RedisModuleCtx *ctx, long long ll
* The returned string must be released with RedisModule_FreeString() or by * The returned string must be released with RedisModule_FreeString() or by
* enabling automatic memory management. */ * enabling automatic memory management. */
RedisModuleString *RM_CreateStringFromDouble(RedisModuleCtx *ctx, double d) { RedisModuleString *RM_CreateStringFromDouble(RedisModuleCtx *ctx, double d) {
char buf[128]; char buf[MAX_D2STRING_CHARS];
size_t len = d2string(buf,sizeof(buf),d); size_t len = d2string(buf,sizeof(buf),d);
return RM_CreateString(ctx,buf,len); return RM_CreateString(ctx,buf,len);
} }
...@@ -2884,8 +2938,11 @@ void RM_ReplySetSetLength(RedisModuleCtx *ctx, long len) { ...@@ -2884,8 +2938,11 @@ void RM_ReplySetSetLength(RedisModuleCtx *ctx, long len) {
} }
/* Very similar to RedisModule_ReplySetMapLength /* Very similar to RedisModule_ReplySetMapLength
* Visit https://github.com/antirez/RESP3/blob/master/spec.md for more info about RESP3. */ * Visit https://github.com/antirez/RESP3/blob/master/spec.md for more info about RESP3.
*
* Must not be called if RM_ReplyWithAttribute returned an error. */
void RM_ReplySetAttributeLength(RedisModuleCtx *ctx, long len) { void RM_ReplySetAttributeLength(RedisModuleCtx *ctx, long len) {
if (ctx->client->resp == 2) return;
moduleReplySetCollectionLength(ctx, len, COLLECTION_REPLY_ATTRIBUTE); moduleReplySetCollectionLength(ctx, len, COLLECTION_REPLY_ATTRIBUTE);
} }
...@@ -5103,6 +5160,7 @@ int RM_StreamIteratorStop(RedisModuleKey *key) { ...@@ -5103,6 +5160,7 @@ int RM_StreamIteratorStop(RedisModuleKey *key) {
errno = EBADF; errno = EBADF;
return REDISMODULE_ERR; return REDISMODULE_ERR;
} }
streamIteratorStop(key->iter);
zfree(key->iter); zfree(key->iter);
key->iter = NULL; key->iter = NULL;
return REDISMODULE_OK; return REDISMODULE_OK;
...@@ -5544,6 +5602,12 @@ robj **moduleCreateArgvFromUserFormat(const char *cmdname, const char *fmt, int ...@@ -5544,6 +5602,12 @@ robj **moduleCreateArgvFromUserFormat(const char *cmdname, const char *fmt, int
if (flags) (*flags) |= REDISMODULE_ARGV_RESP_AUTO; if (flags) (*flags) |= REDISMODULE_ARGV_RESP_AUTO;
} else if (*p == 'C') { } else if (*p == 'C') {
if (flags) (*flags) |= REDISMODULE_ARGV_CHECK_ACL; if (flags) (*flags) |= REDISMODULE_ARGV_CHECK_ACL;
} else if (*p == 'S') {
if (flags) (*flags) |= REDISMODULE_ARGV_SCRIPT_MODE;
} else if (*p == 'W') {
if (flags) (*flags) |= REDISMODULE_ARGV_NO_WRITES;
} else if (*p == 'E') {
if (flags) (*flags) |= REDISMODULE_ARGV_CALL_REPLIES_AS_ERRORS;
} else { } else {
goto fmterr; goto fmterr;
} }
...@@ -5583,6 +5647,17 @@ fmterr: ...@@ -5583,6 +5647,17 @@ fmterr:
* same as the client attached to the given RedisModuleCtx. This will * same as the client attached to the given RedisModuleCtx. This will
* probably used when you want to pass the reply directly to the client. * probably used when you want to pass the reply directly to the client.
* * `C` -- Check if command can be executed according to ACL rules. * * `C` -- Check if command can be executed according to ACL rules.
* * 'S' -- Run the command in a script mode, this means that it will raise
* an error if a command which are not allowed inside a script
* (flagged with the `deny-script` flag) is invoked (like SHUTDOWN).
* In addition, on script mode, write commands are not allowed if there are
* not enough good replicas (as configured with `min-replicas-to-write`)
* or when the server is unable to persist to the disk.
* * 'W' -- Do not allow to run any write command (flagged with the `write` flag).
* * 'E' -- Return error as RedisModuleCallReply. If there is an error before
* invoking the command, the error is returned using errno mechanism.
* This flag allows to get the error also as an error CallReply with
* relevant error message.
* * **...**: The actual arguments to the Redis command. * * **...**: The actual arguments to the Redis command.
* *
* On success a RedisModuleCallReply object is returned, otherwise * On success a RedisModuleCallReply object is returned, otherwise
...@@ -5597,6 +5672,8 @@ fmterr: ...@@ -5597,6 +5672,8 @@ fmterr:
* * ENETDOWN: operation in Cluster instance when cluster is down. * * ENETDOWN: operation in Cluster instance when cluster is down.
* * ENOTSUP: No ACL user for the specified module context * * ENOTSUP: No ACL user for the specified module context
* * EACCES: Command cannot be executed, according to ACL rules * * EACCES: Command cannot be executed, according to ACL rules
* * ENOSPC: Write command is not allowed
* * ESPIPE: Command not allowed on script mode
* *
* Example code fragment: * Example code fragment:
* *
...@@ -5616,11 +5693,13 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch ...@@ -5616,11 +5693,13 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch
va_list ap; va_list ap;
RedisModuleCallReply *reply = NULL; RedisModuleCallReply *reply = NULL;
int replicate = 0; /* Replicate this command? */ int replicate = 0; /* Replicate this command? */
int error_as_call_replies = 0; /* return errors as RedisModuleCallReply object */
/* Handle arguments. */ /* Handle arguments. */
va_start(ap, fmt); va_start(ap, fmt);
argv = moduleCreateArgvFromUserFormat(cmdname,fmt,&argc,&argv_len,&flags,ap); argv = moduleCreateArgvFromUserFormat(cmdname,fmt,&argc,&argv_len,&flags,ap);
replicate = flags & REDISMODULE_ARGV_REPLICATE; replicate = flags & REDISMODULE_ARGV_REPLICATE;
error_as_call_replies = flags & REDISMODULE_ARGV_CALL_REPLIES_AS_ERRORS;
va_end(ap); va_end(ap);
c = moduleAllocTempClient(); c = moduleAllocTempClient();
...@@ -5643,6 +5722,10 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch ...@@ -5643,6 +5722,10 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch
/* We handle the above format error only when the client is setup so that /* We handle the above format error only when the client is setup so that
* we can free it normally. */ * we can free it normally. */
if (argv == NULL) { if (argv == NULL) {
/* We do not return a call reply here this is an error that should only
* be catch by the module indicating wrong fmt was given, the module should
* handle this error and decide how to continue. It is not an error that
* should be propagated to the user. */
errno = EBADF; errno = EBADF;
goto cleanup; goto cleanup;
} }
...@@ -5656,6 +5739,11 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch ...@@ -5656,6 +5739,11 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch
cmd = lookupCommand(c->argv,c->argc); cmd = lookupCommand(c->argv,c->argc);
if (!cmd) { if (!cmd) {
errno = ENOENT; errno = ENOENT;
if (error_as_call_replies) {
sds msg = sdscatfmt(sdsempty(),"Unknown Redis "
"command '%S'.",c->argv[0]->ptr);
reply = callReplyCreateError(msg, ctx);
}
goto cleanup; goto cleanup;
} }
c->cmd = c->lastcmd = c->realcmd = cmd; c->cmd = c->lastcmd = c->realcmd = cmd;
...@@ -5663,9 +5751,66 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch ...@@ -5663,9 +5751,66 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch
/* Basic arity checks. */ /* Basic arity checks. */
if ((cmd->arity > 0 && cmd->arity != argc) || (argc < -cmd->arity)) { if ((cmd->arity > 0 && cmd->arity != argc) || (argc < -cmd->arity)) {
errno = EINVAL; errno = EINVAL;
if (error_as_call_replies) {
sds msg = sdscatfmt(sdsempty(), "Wrong number of "
"args calling Redis command '%S'.", c->cmd->fullname);
reply = callReplyCreateError(msg, ctx);
}
goto cleanup;
}
if (flags & REDISMODULE_ARGV_SCRIPT_MODE) {
/* Basically on script mode we want to only allow commands that can
* be executed on scripts (CMD_NOSCRIPT is not set on the command flags) */
if (cmd->flags & CMD_NOSCRIPT) {
errno = ESPIPE;
if (error_as_call_replies) {
sds msg = sdscatfmt(sdsempty(), "command '%S' is not allowed on script mode", c->cmd->fullname);
reply = callReplyCreateError(msg, ctx);
}
goto cleanup;
}
}
if (cmd->flags & CMD_WRITE) {
if (flags & REDISMODULE_ARGV_NO_WRITES) {
errno = ENOSPC;
if (error_as_call_replies) {
sds msg = sdscatfmt(sdsempty(), "Write command '%S' was "
"called while write is not allowed.", c->cmd->fullname);
reply = callReplyCreateError(msg, ctx);
}
goto cleanup;
}
if (flags & REDISMODULE_ARGV_SCRIPT_MODE) {
/* on script mode, if a command is a write command,
* We will not run it if we encounter disk error
* or we do not have enough replicas */
if (!checkGoodReplicasStatus()) {
errno = ENOSPC;
if (error_as_call_replies) {
sds msg = sdsdup(shared.noreplicaserr->ptr);
reply = callReplyCreateError(msg, ctx);
}
goto cleanup; goto cleanup;
} }
int deny_write_type = writeCommandsDeniedByDiskError();
if (deny_write_type != DISK_ERROR_TYPE_NONE) {
errno = ENOSPC;
if (error_as_call_replies) {
sds msg = writeCommandsGetDiskErrorMessage(deny_write_type);
reply = callReplyCreateError(msg, ctx);
}
goto cleanup;
}
}
}
/* Check if the user can run this command according to the current /* Check if the user can run this command according to the current
* ACLs. */ * ACLs. */
if (flags & REDISMODULE_ARGV_CHECK_ACL) { if (flags & REDISMODULE_ARGV_CHECK_ACL) {
...@@ -5674,12 +5819,20 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch ...@@ -5674,12 +5819,20 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch
if (ctx->client->user == NULL) { if (ctx->client->user == NULL) {
errno = ENOTSUP; errno = ENOTSUP;
if (error_as_call_replies) {
sds msg = sdsnew("acl verification failed, context is not attached to a client.");
reply = callReplyCreateError(msg, ctx);
}
goto cleanup; goto cleanup;
} }
acl_retval = ACLCheckAllUserCommandPerm(ctx->client->user,c->cmd,c->argv,c->argc,&acl_errpos); acl_retval = ACLCheckAllUserCommandPerm(ctx->client->user,c->cmd,c->argv,c->argc,&acl_errpos);
if (acl_retval != ACL_OK) { if (acl_retval != ACL_OK) {
sds object = (acl_retval == ACL_DENIED_CMD) ? sdsdup(c->cmd->fullname) : sdsdup(c->argv[acl_errpos]->ptr); sds object = (acl_retval == ACL_DENIED_CMD) ? sdsdup(c->cmd->fullname) : sdsdup(c->argv[acl_errpos]->ptr);
addACLLogEntry(ctx->client, acl_retval, ACL_LOG_CTX_MODULE, -1, ctx->client->user->name, object); addACLLogEntry(ctx->client, acl_retval, ACL_LOG_CTX_MODULE, -1, ctx->client->user->name, object);
if (error_as_call_replies) {
sds msg = sdscatfmt(sdsempty(), "acl verification failed, %s.", getAclErrorMessage(acl_retval));
reply = callReplyCreateError(msg, ctx);
}
errno = EACCES; errno = EACCES;
goto cleanup; goto cleanup;
} }
...@@ -5696,13 +5849,26 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch ...@@ -5696,13 +5849,26 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch
if (getNodeByQuery(c,c->cmd,c->argv,c->argc,NULL,&error_code) != if (getNodeByQuery(c,c->cmd,c->argv,c->argc,NULL,&error_code) !=
server.cluster->myself) server.cluster->myself)
{ {
sds msg = NULL;
if (error_code == CLUSTER_REDIR_DOWN_RO_STATE) { if (error_code == CLUSTER_REDIR_DOWN_RO_STATE) {
if (error_as_call_replies) {
msg = sdscatfmt(sdsempty(), "Can not execute a write command '%S' while the cluster is down and readonly", c->cmd->fullname);
}
errno = EROFS; errno = EROFS;
} else if (error_code == CLUSTER_REDIR_DOWN_STATE) { } else if (error_code == CLUSTER_REDIR_DOWN_STATE) {
if (error_as_call_replies) {
msg = sdscatfmt(sdsempty(), "Can not execute a command '%S' while the cluster is down", c->cmd->fullname);
}
errno = ENETDOWN; errno = ENETDOWN;
} else { } else {
if (error_as_call_replies) {
msg = sdsnew("Attempted to access a non local key in a cluster node");
}
errno = EPERM; errno = EPERM;
} }
if (msg) {
reply = callReplyCreateError(msg, ctx);
}
goto cleanup; goto cleanup;
} }
} }
...@@ -5744,9 +5910,9 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch ...@@ -5744,9 +5910,9 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch
} }
reply = callReplyCreate(proto, c->deferred_reply_errors, ctx); reply = callReplyCreate(proto, c->deferred_reply_errors, ctx);
c->deferred_reply_errors = NULL; /* now the responsibility of the reply object. */ c->deferred_reply_errors = NULL; /* now the responsibility of the reply object. */
autoMemoryAdd(ctx,REDISMODULE_AM_REPLY,reply);
cleanup: cleanup:
if (reply) autoMemoryAdd(ctx,REDISMODULE_AM_REPLY,reply);
if (ctx->module) ctx->module->in_call--; if (ctx->module) ctx->module->in_call--;
moduleReleaseTempClient(c); moduleReleaseTempClient(c);
return reply; return reply;
...@@ -7785,8 +7951,9 @@ size_t RM_GetClusterSize(void) { ...@@ -7785,8 +7951,9 @@ size_t RM_GetClusterSize(void) {
} }
/* Populate the specified info for the node having as ID the specified 'id', /* Populate the specified info for the node having as ID the specified 'id',
* then returns REDISMODULE_OK. Otherwise if the node ID does not exist from * then returns REDISMODULE_OK. Otherwise if the format of node ID is invalid
* the POV of this local node, REDISMODULE_ERR is returned. * or the node ID does not exist from the POV of this local node, REDISMODULE_ERR
* is returned.
* *
* The arguments `ip`, `master_id`, `port` and `flags` can be NULL in case we don't * The arguments `ip`, `master_id`, `port` and `flags` can be NULL in case we don't
* need to populate back certain info. If an `ip` and `master_id` (only populated * need to populate back certain info. If an `ip` and `master_id` (only populated
...@@ -7806,7 +7973,7 @@ size_t RM_GetClusterSize(void) { ...@@ -7806,7 +7973,7 @@ size_t RM_GetClusterSize(void) {
int RM_GetClusterNodeInfo(RedisModuleCtx *ctx, const char *id, char *ip, char *master_id, int *port, int *flags) { int RM_GetClusterNodeInfo(RedisModuleCtx *ctx, const char *id, char *ip, char *master_id, int *port, int *flags) {
UNUSED(ctx); UNUSED(ctx);
clusterNode *node = clusterLookupNode(id); clusterNode *node = clusterLookupNode(id, strlen(id));
if (node == NULL || if (node == NULL ||
node->flags & (CLUSTER_NODE_NOADDR|CLUSTER_NODE_HANDSHAKE)) node->flags & (CLUSTER_NODE_NOADDR|CLUSTER_NODE_HANDSHAKE))
{ {
...@@ -8607,6 +8774,24 @@ int RM_DeauthenticateAndCloseClient(RedisModuleCtx *ctx, uint64_t client_id) { ...@@ -8607,6 +8774,24 @@ int RM_DeauthenticateAndCloseClient(RedisModuleCtx *ctx, uint64_t client_id) {
return REDISMODULE_OK; return REDISMODULE_OK;
} }
/* Redact the client command argument specified at the given position. Redacted arguments
* are obfuscated in user facing commands such as SLOWLOG or MONITOR, as well as
* never being written to server logs. This command may be called multiple times on the
* same position.
*
* Note that the command name, position 0, can not be redacted.
*
* Returns REDISMODULE_OK if the argument was redacted and REDISMODULE_ERR if there
* was an invalid parameter passed in or the position is outside the client
* argument range. */
int RM_RedactClientCommandArgument(RedisModuleCtx *ctx, int pos) {
if (!ctx || !ctx->client || pos <= 0 || ctx->client->argc <= pos) {
return REDISMODULE_ERR;
}
redactClientCommandArgument(ctx->client, pos);
return REDISMODULE_OK;
}
/* Return the X.509 client-side certificate used by the client to authenticate /* Return the X.509 client-side certificate used by the client to authenticate
* this connection. * this connection.
* *
...@@ -9972,6 +10157,7 @@ static uint64_t moduleEventVersions[] = { ...@@ -9972,6 +10157,7 @@ static uint64_t moduleEventVersions[] = {
-1, /* REDISMODULE_EVENT_FORK_CHILD */ -1, /* REDISMODULE_EVENT_FORK_CHILD */
-1, /* REDISMODULE_EVENT_REPL_ASYNC_LOAD */ -1, /* REDISMODULE_EVENT_REPL_ASYNC_LOAD */
-1, /* REDISMODULE_EVENT_EVENTLOOP */ -1, /* REDISMODULE_EVENT_EVENTLOOP */
-1, /* REDISMODULE_EVENT_CONFIG */
}; };
/* Register to be notified, via a callback, when the specified server event /* Register to be notified, via a callback, when the specified server event
...@@ -10192,7 +10378,7 @@ static uint64_t moduleEventVersions[] = { ...@@ -10192,7 +10378,7 @@ static uint64_t moduleEventVersions[] = {
* are now triggered when repl-diskless-load is set to swapdb. * are now triggered when repl-diskless-load is set to swapdb.
* *
* Called when repl-diskless-load config is set to swapdb, * Called when repl-diskless-load config is set to swapdb,
* And redis needs to backup the the current database for the * And redis needs to backup the current database for the
* possibility to be restored later. A module with global data and * possibility to be restored later. A module with global data and
* maybe with aux_load and aux_save callbacks may need to use this * maybe with aux_load and aux_save callbacks may need to use this
* notification to backup / restore / discard its globals. * notification to backup / restore / discard its globals.
...@@ -10232,6 +10418,20 @@ static uint64_t moduleEventVersions[] = { ...@@ -10232,6 +10418,20 @@ static uint64_t moduleEventVersions[] = {
* * `REDISMODULE_SUBEVENT_EVENTLOOP_BEFORE_SLEEP` * * `REDISMODULE_SUBEVENT_EVENTLOOP_BEFORE_SLEEP`
* * `REDISMODULE_SUBEVENT_EVENTLOOP_AFTER_SLEEP` * * `REDISMODULE_SUBEVENT_EVENTLOOP_AFTER_SLEEP`
* *
* * RedisModule_Event_Config
*
* Called when a configuration event happens
* The following sub events are available:
*
* * `REDISMODULE_SUBEVENT_CONFIG_CHANGE`
*
* The data pointer can be casted to a RedisModuleConfigChange
* structure with the following fields:
*
* const char **config_names; // An array of C string pointers containing the
* // name of each modified configuration item
* uint32_t num_changes; // The number of elements in the config_names array
*
* The function returns REDISMODULE_OK if the module was successfully subscribed * The function returns REDISMODULE_OK if the module was successfully subscribed
* for the specified event. If the API is called from a wrong context or unsupported event * for the specified event. If the API is called from a wrong context or unsupported event
* is given then REDISMODULE_ERR is returned. */ * is given then REDISMODULE_ERR is returned. */
...@@ -10309,6 +10509,8 @@ int RM_IsSubEventSupported(RedisModuleEvent event, int64_t subevent) { ...@@ -10309,6 +10509,8 @@ int RM_IsSubEventSupported(RedisModuleEvent event, int64_t subevent) {
return subevent < _REDISMODULE_SUBEVENT_FORK_CHILD_NEXT; return subevent < _REDISMODULE_SUBEVENT_FORK_CHILD_NEXT;
case REDISMODULE_EVENT_EVENTLOOP: case REDISMODULE_EVENT_EVENTLOOP:
return subevent < _REDISMODULE_SUBEVENT_EVENTLOOP_NEXT; return subevent < _REDISMODULE_SUBEVENT_EVENTLOOP_NEXT;
case REDISMODULE_EVENT_CONFIG:
return subevent < _REDISMODULE_SUBEVENT_CONFIG_NEXT;
default: default:
break; break;
} }
...@@ -10385,6 +10587,8 @@ void moduleFireServerEvent(uint64_t eid, int subid, void *data) { ...@@ -10385,6 +10587,8 @@ void moduleFireServerEvent(uint64_t eid, int subid, void *data) {
moduledata = data; moduledata = data;
} else if (eid == REDISMODULE_EVENT_SWAPDB) { } else if (eid == REDISMODULE_EVENT_SWAPDB) {
moduledata = data; moduledata = data;
} else if (eid == REDISMODULE_EVENT_CONFIG) {
moduledata = data;
} }
el->module->in_hook++; el->module->in_hook++;
...@@ -10528,9 +10732,21 @@ void moduleRegisterCoreAPI(void); ...@@ -10528,9 +10732,21 @@ void moduleRegisterCoreAPI(void);
void moduleInitModulesSystemLast(void) { void moduleInitModulesSystemLast(void) {
} }
dictType sdsKeyValueHashDictType = {
dictSdsCaseHash, /* hash function */
NULL, /* key dup */
NULL, /* val dup */
dictSdsKeyCaseCompare, /* key compare */
dictSdsDestructor, /* key destructor */
dictSdsDestructor, /* val destructor */
NULL /* allow to expand */
};
void moduleInitModulesSystem(void) { void moduleInitModulesSystem(void) {
moduleUnblockedClients = listCreate(); moduleUnblockedClients = listCreate();
server.loadmodule_queue = listCreate(); server.loadmodule_queue = listCreate();
server.module_configs_queue = dictCreate(&sdsKeyValueHashDictType);
modules = dictCreate(&modulesDictType); modules = dictCreate(&modulesDictType);
/* Set up the keyspace notification subscriber list and static client */ /* Set up the keyspace notification subscriber list and static client */
...@@ -10603,6 +10819,20 @@ void moduleLoadQueueEntryFree(struct moduleLoadQueueEntry *loadmod) { ...@@ -10603,6 +10819,20 @@ void moduleLoadQueueEntryFree(struct moduleLoadQueueEntry *loadmod) {
zfree(loadmod); zfree(loadmod);
} }
/* Remove Module Configs from standardConfig array in config.c */
void moduleRemoveConfigs(RedisModule *module) {
listIter li;
listNode *ln;
listRewind(module->module_configs, &li);
while ((ln = listNext(&li))) {
ModuleConfig *config = listNodeValue(ln);
sds module_name = sdsnew(module->name);
sds full_name = sdscat(sdscat(module_name, "."), config->name); /* ModuleName.ModuleConfig */
removeConfig(full_name);
sdsfree(full_name);
}
}
/* Load all the modules in the server.loadmodule_queue list, which is /* Load all the modules in the server.loadmodule_queue list, which is
* populated by `loadmodule` directives in the configuration file. * populated by `loadmodule` directives in the configuration file.
* We can't load modules directly when processing the configuration file * We can't load modules directly when processing the configuration file
...@@ -10619,7 +10849,7 @@ void moduleLoadFromQueue(void) { ...@@ -10619,7 +10849,7 @@ void moduleLoadFromQueue(void) {
listRewind(server.loadmodule_queue,&li); listRewind(server.loadmodule_queue,&li);
while((ln = listNext(&li))) { while((ln = listNext(&li))) {
struct moduleLoadQueueEntry *loadmod = ln->value; struct moduleLoadQueueEntry *loadmod = ln->value;
if (moduleLoad(loadmod->path,(void **)loadmod->argv,loadmod->argc) if (moduleLoad(loadmod->path,(void **)loadmod->argv,loadmod->argc, 0)
== C_ERR) == C_ERR)
{ {
serverLog(LL_WARNING, serverLog(LL_WARNING,
...@@ -10630,6 +10860,10 @@ void moduleLoadFromQueue(void) { ...@@ -10630,6 +10860,10 @@ void moduleLoadFromQueue(void) {
moduleLoadQueueEntryFree(loadmod); moduleLoadQueueEntryFree(loadmod);
listDelNode(server.loadmodule_queue, ln); listDelNode(server.loadmodule_queue, ln);
} }
if (dictSize(server.module_configs_queue)) {
serverLog(LL_WARNING, "Module Configuration detected without loadmodule directive or no ApplyConfig call: aborting");
exit(1);
}
} }
void moduleFreeModuleStructure(struct RedisModule *module) { void moduleFreeModuleStructure(struct RedisModule *module) {
...@@ -10637,6 +10871,7 @@ void moduleFreeModuleStructure(struct RedisModule *module) { ...@@ -10637,6 +10871,7 @@ void moduleFreeModuleStructure(struct RedisModule *module) {
listRelease(module->filters); listRelease(module->filters);
listRelease(module->usedby); listRelease(module->usedby);
listRelease(module->using); listRelease(module->using);
listRelease(module->module_configs);
sdsfree(module->name); sdsfree(module->name);
moduleLoadQueueEntryFree(module->loadmod); moduleLoadQueueEntryFree(module->loadmod);
zfree(module); zfree(module);
...@@ -10717,15 +10952,56 @@ void moduleUnregisterCommands(struct RedisModule *module) { ...@@ -10717,15 +10952,56 @@ void moduleUnregisterCommands(struct RedisModule *module) {
dictReleaseIterator(di); dictReleaseIterator(di);
} }
/* We parse argv to add sds "NAME VALUE" pairs to the server.module_configs_queue list of configs.
* We also increment the module_argv pointer to just after ARGS if there are args, otherwise
* we set it to NULL */
int parseLoadexArguments(RedisModuleString ***module_argv, int *module_argc) {
int args_specified = 0;
RedisModuleString **argv = *module_argv;
int argc = *module_argc;
for (int i = 0; i < argc; i++) {
char *arg_val = argv[i]->ptr;
if (!strcasecmp(arg_val, "CONFIG")) {
if (i + 2 >= argc) {
serverLog(LL_NOTICE, "CONFIG specified without name value pair");
return REDISMODULE_ERR;
}
sds name = sdsdup(argv[i + 1]->ptr);
sds value = sdsdup(argv[i + 2]->ptr);
if (!dictReplace(server.module_configs_queue, name, value)) sdsfree(name);
i += 2;
} else if (!strcasecmp(arg_val, "ARGS")) {
args_specified = 1;
i++;
if (i >= argc) {
*module_argv = NULL;
*module_argc = 0;
} else {
*module_argv = argv + i;
*module_argc = argc - i;
}
break;
} else {
serverLog(LL_NOTICE, "Syntax Error from arguments to loadex around %s.", arg_val);
return REDISMODULE_ERR;
}
}
if (!args_specified) {
*module_argv = NULL;
*module_argc = 0;
}
return REDISMODULE_OK;
}
/* Load a module and initialize it. On success C_OK is returned, otherwise /* Load a module and initialize it. On success C_OK is returned, otherwise
* C_ERR is returned. */ * C_ERR is returned. */
int moduleLoad(const char *path, void **module_argv, int module_argc) { int moduleLoad(const char *path, void **module_argv, int module_argc, int is_loadex) {
int (*onload)(void *, void **, int); int (*onload)(void *, void **, int);
void *handle; void *handle;
struct stat st; struct stat st;
if (stat(path, &st) == 0) if (stat(path, &st) == 0) {
{ // this check is best effort /* This check is best effort */
if (!(st.st_mode & (S_IXUSR | S_IXGRP | S_IXOTH))) { if (!(st.st_mode & (S_IXUSR | S_IXGRP | S_IXOTH))) {
serverLog(LL_WARNING, "Module %s failed to load: It does not have execute permissions.", path); serverLog(LL_WARNING, "Module %s failed to load: It does not have execute permissions.", path);
return C_ERR; return C_ERR;
...@@ -10749,16 +11025,17 @@ int moduleLoad(const char *path, void **module_argv, int module_argc) { ...@@ -10749,16 +11025,17 @@ int moduleLoad(const char *path, void **module_argv, int module_argc) {
moduleCreateContext(&ctx, NULL, REDISMODULE_CTX_TEMP_CLIENT); /* We pass NULL since we don't have a module yet. */ moduleCreateContext(&ctx, NULL, REDISMODULE_CTX_TEMP_CLIENT); /* We pass NULL since we don't have a module yet. */
selectDb(ctx.client, 0); selectDb(ctx.client, 0);
if (onload((void*)&ctx,module_argv,module_argc) == REDISMODULE_ERR) { if (onload((void*)&ctx,module_argv,module_argc) == REDISMODULE_ERR) {
serverLog(LL_WARNING,
"Module %s initialization failed. Module not loaded",path);
if (ctx.module) { if (ctx.module) {
moduleUnregisterCommands(ctx.module); moduleUnregisterCommands(ctx.module);
moduleUnregisterSharedAPI(ctx.module); moduleUnregisterSharedAPI(ctx.module);
moduleUnregisterUsedAPI(ctx.module); moduleUnregisterUsedAPI(ctx.module);
moduleRemoveConfigs(ctx.module);
moduleFreeModuleStructure(ctx.module); moduleFreeModuleStructure(ctx.module);
} }
moduleFreeContext(&ctx); moduleFreeContext(&ctx);
dlclose(handle); dlclose(handle);
serverLog(LL_WARNING,
"Module %s initialization failed. Module not loaded",path);
return C_ERR; return C_ERR;
} }
...@@ -10776,15 +11053,30 @@ int moduleLoad(const char *path, void **module_argv, int module_argc) { ...@@ -10776,15 +11053,30 @@ int moduleLoad(const char *path, void **module_argv, int module_argc) {
} }
serverLog(LL_NOTICE,"Module '%s' loaded from %s",ctx.module->name,path); serverLog(LL_NOTICE,"Module '%s' loaded from %s",ctx.module->name,path);
if (listLength(ctx.module->module_configs) && !ctx.module->configs_initialized) {
serverLogRaw(LL_WARNING, "Module Configurations were not set, likely a missing LoadConfigs call. Unloading the module.");
moduleUnload(ctx.module->name);
moduleFreeContext(&ctx);
return C_ERR;
}
if (is_loadex && dictSize(server.module_configs_queue)) {
serverLogRaw(LL_WARNING, "Loadex configurations were not applied, likely due to invalid arguments. Unloading the module.");
moduleUnload(ctx.module->name);
moduleFreeContext(&ctx);
return C_ERR;
}
/* Fire the loaded modules event. */ /* Fire the loaded modules event. */
moduleFireServerEvent(REDISMODULE_EVENT_MODULE_CHANGE, moduleFireServerEvent(REDISMODULE_EVENT_MODULE_CHANGE,
REDISMODULE_SUBEVENT_MODULE_LOADED, REDISMODULE_SUBEVENT_MODULE_LOADED,
ctx.module); ctx.module);
moduleFreeContext(&ctx); moduleFreeContext(&ctx);
return C_OK; return C_OK;
} }
/* Unload the module registered with the specified name. On success /* Unload the module registered with the specified name. On success
* C_OK is returned, otherwise C_ERR is returned and errno is set * C_OK is returned, otherwise C_ERR is returned and errno is set
* to the following values depending on the type of error: * to the following values depending on the type of error:
...@@ -10836,6 +11128,7 @@ int moduleUnload(sds name) { ...@@ -10836,6 +11128,7 @@ int moduleUnload(sds name) {
moduleUnregisterSharedAPI(module); moduleUnregisterSharedAPI(module);
moduleUnregisterUsedAPI(module); moduleUnregisterUsedAPI(module);
moduleUnregisterFilters(module); moduleUnregisterFilters(module);
moduleRemoveConfigs(module);
/* Remove any notification subscribers this module might have */ /* Remove any notification subscribers this module might have */
moduleUnsubscribeNotifications(module); moduleUnsubscribeNotifications(module);
...@@ -10964,10 +11257,433 @@ sds genModulesInfoString(sds info) { ...@@ -10964,10 +11257,433 @@ sds genModulesInfoString(sds info) {
return info; return info;
} }
/* --------------------------------------------------------------------------
* Module Configurations API internals
* -------------------------------------------------------------------------- */
/* Check if the configuration name is already registered */
int isModuleConfigNameRegistered(RedisModule *module, sds name) {
listNode *match = listSearchKey(module->module_configs, (void *) name);
return match != NULL;
}
/* Assert that the flags passed into the RM_RegisterConfig Suite are valid */
int moduleVerifyConfigFlags(unsigned int flags, configType type) {
if ((flags & ~(REDISMODULE_CONFIG_DEFAULT
| REDISMODULE_CONFIG_IMMUTABLE
| REDISMODULE_CONFIG_SENSITIVE
| REDISMODULE_CONFIG_HIDDEN
| REDISMODULE_CONFIG_PROTECTED
| REDISMODULE_CONFIG_DENY_LOADING
| REDISMODULE_CONFIG_MEMORY))) {
serverLogRaw(LL_WARNING, "Invalid flag(s) for configuration");
return REDISMODULE_ERR;
}
if (type != NUMERIC_CONFIG && flags & REDISMODULE_CONFIG_MEMORY) {
serverLogRaw(LL_WARNING, "Numeric flag provided for non-numeric configuration.");
return REDISMODULE_ERR;
}
return REDISMODULE_OK;
}
int moduleVerifyConfigName(sds name) {
if (sdslen(name) == 0) {
serverLogRaw(LL_WARNING, "Module config names cannot be an empty string.");
return REDISMODULE_ERR;
}
for (size_t i = 0 ; i < sdslen(name) ; ++i) {
char curr_char = name[i];
if ((curr_char >= 'a' && curr_char <= 'z') ||
(curr_char >= 'A' && curr_char <= 'Z') ||
(curr_char >= '0' && curr_char <= '9') ||
(curr_char == '_') || (curr_char == '-'))
{
continue;
}
serverLog(LL_WARNING, "Invalid character %c in Module Config name %s.", curr_char, name);
return REDISMODULE_ERR;
}
return REDISMODULE_OK;
}
/* This is a series of set functions for each type that act as dispatchers for
* config.c to call module set callbacks. */
#define CONFIG_ERR_SIZE 256
static char configerr[CONFIG_ERR_SIZE];
static void propagateErrorString(RedisModuleString *err_in, const char **err) {
if (err_in) {
strncpy(configerr, err_in->ptr, CONFIG_ERR_SIZE);
configerr[CONFIG_ERR_SIZE - 1] = '\0';
decrRefCount(err_in);
*err = configerr;
}
}
int setModuleBoolConfig(ModuleConfig *config, int val, const char **err) {
RedisModuleString *error = NULL;
int return_code = config->set_fn.set_bool(config->name, val, config->privdata, &error);
propagateErrorString(error, err);
return return_code == REDISMODULE_OK ? 1 : 0;
}
int setModuleStringConfig(ModuleConfig *config, sds strval, const char **err) {
RedisModuleString *error = NULL;
RedisModuleString *new = createStringObject(strval, sdslen(strval));
int return_code = config->set_fn.set_string(config->name, new, config->privdata, &error);
propagateErrorString(error, err);
decrRefCount(new);
return return_code == REDISMODULE_OK ? 1 : 0;
}
int setModuleEnumConfig(ModuleConfig *config, int val, const char **err) {
RedisModuleString *error = NULL;
int return_code = config->set_fn.set_enum(config->name, val, config->privdata, &error);
propagateErrorString(error, err);
return return_code == REDISMODULE_OK ? 1 : 0;
}
int setModuleNumericConfig(ModuleConfig *config, long long val, const char **err) {
RedisModuleString *error = NULL;
int return_code = config->set_fn.set_numeric(config->name, val, config->privdata, &error);
propagateErrorString(error, err);
return return_code == REDISMODULE_OK ? 1 : 0;
}
/* This is a series of get functions for each type that act as dispatchers for
* config.c to call module set callbacks. */
int getModuleBoolConfig(ModuleConfig *module_config) {
return module_config->get_fn.get_bool(module_config->name, module_config->privdata);
}
sds getModuleStringConfig(ModuleConfig *module_config) {
RedisModuleString *val = module_config->get_fn.get_string(module_config->name, module_config->privdata);
return val ? sdsdup(val->ptr) : NULL;
}
int getModuleEnumConfig(ModuleConfig *module_config) {
return module_config->get_fn.get_enum(module_config->name, module_config->privdata);
}
long long getModuleNumericConfig(ModuleConfig *module_config) {
return module_config->get_fn.get_numeric(module_config->name, module_config->privdata);
}
/* This function takes a module and a list of configs stored as sds NAME VALUE pairs.
* It attempts to call set on each of these configs. */
int loadModuleConfigs(RedisModule *module) {
listIter li;
listNode *ln;
const char *err = NULL;
listRewind(module->module_configs, &li);
while ((ln = listNext(&li))) {
ModuleConfig *module_config = listNodeValue(ln);
sds config_name = sdscatfmt(sdsempty(), "%s.%s", module->name, module_config->name);
dictEntry *config_argument = dictFind(server.module_configs_queue, config_name);
if (config_argument) {
if (!performModuleConfigSetFromName(dictGetKey(config_argument), dictGetVal(config_argument), &err)) {
serverLog(LL_WARNING, "Issue during loading of configuration %s : %s", (sds) dictGetKey(config_argument), err);
sdsfree(config_name);
dictEmpty(server.module_configs_queue, NULL);
return REDISMODULE_ERR;
}
} else {
if (!performModuleConfigSetDefaultFromName(config_name, &err)) {
serverLog(LL_WARNING, "Issue attempting to set default value of configuration %s : %s", module_config->name, err);
sdsfree(config_name);
dictEmpty(server.module_configs_queue, NULL);
return REDISMODULE_ERR;
}
}
dictDelete(server.module_configs_queue, config_name);
sdsfree(config_name);
}
module->configs_initialized = 1;
return REDISMODULE_OK;
}
/* Add module_config to the list if the apply and privdata do not match one already in it. */
void addModuleConfigApply(list *module_configs, ModuleConfig *module_config) {
if (!module_config->apply_fn) return;
listIter li;
listNode *ln;
ModuleConfig *pending_apply;
listRewind(module_configs, &li);
while ((ln = listNext(&li))) {
pending_apply = listNodeValue(ln);
if (pending_apply->apply_fn == module_config->apply_fn && pending_apply->privdata == module_config->privdata) {
return;
}
}
listAddNodeTail(module_configs, module_config);
}
/* Call apply on all module configs specified in set, if an apply function was specified at registration time. */
int moduleConfigApplyConfig(list *module_configs, const char **err, const char **err_arg_name) {
if (!listLength(module_configs)) return 1;
listIter li;
listNode *ln;
ModuleConfig *module_config;
RedisModuleString *error = NULL;
RedisModuleCtx ctx;
listRewind(module_configs, &li);
while ((ln = listNext(&li))) {
module_config = listNodeValue(ln);
moduleCreateContext(&ctx, module_config->module, REDISMODULE_CTX_NONE);
if (module_config->apply_fn(&ctx, module_config->privdata, &error)) {
if (err_arg_name) *err_arg_name = module_config->name;
propagateErrorString(error, err);
moduleFreeContext(&ctx);
return 0;
}
moduleFreeContext(&ctx);
}
return 1;
}
/* --------------------------------------------------------------------------
* ## Module Configurations API
* -------------------------------------------------------------------------- */
/* Create a module config object. */
ModuleConfig *createModuleConfig(sds name, RedisModuleConfigApplyFunc apply_fn, void *privdata, RedisModule *module) {
ModuleConfig *new_config = zmalloc(sizeof(ModuleConfig));
new_config->name = sdsdup(name);
new_config->apply_fn = apply_fn;
new_config->privdata = privdata;
new_config->module = module;
return new_config;
}
int moduleConfigValidityCheck(RedisModule *module, sds name, unsigned int flags, configType type) {
if (moduleVerifyConfigFlags(flags, type) || moduleVerifyConfigName(name)) {
errno = EINVAL;
return REDISMODULE_ERR;
}
if (isModuleConfigNameRegistered(module, name)) {
serverLog(LL_WARNING, "Configuration by the name: %s already registered", name);
errno = EALREADY;
return REDISMODULE_ERR;
}
return REDISMODULE_OK;
}
unsigned int maskModuleConfigFlags(unsigned int flags) {
unsigned int new_flags = 0;
if (flags & REDISMODULE_CONFIG_DEFAULT) new_flags |= MODIFIABLE_CONFIG;
if (flags & REDISMODULE_CONFIG_IMMUTABLE) new_flags |= IMMUTABLE_CONFIG;
if (flags & REDISMODULE_CONFIG_HIDDEN) new_flags |= HIDDEN_CONFIG;
if (flags & REDISMODULE_CONFIG_PROTECTED) new_flags |= PROTECTED_CONFIG;
if (flags & REDISMODULE_CONFIG_DENY_LOADING) new_flags |= DENY_LOADING_CONFIG;
return new_flags;
}
unsigned int maskModuleNumericConfigFlags(unsigned int flags) {
unsigned int new_flags = 0;
if (flags & REDISMODULE_CONFIG_MEMORY) new_flags |= MEMORY_CONFIG;
return new_flags;
}
/* Create a string config that Redis users can interact with via the Redis config file,
* `CONFIG SET`, `CONFIG GET`, and `CONFIG REWRITE` commands.
*
* The actual config value is owned by the module, and the `getfn`, `setfn` and optional
* `applyfn` callbacks that are provided to Redis in order to access or manipulate the
* value. The `getfn` callback retrieves the value from the module, while the `setfn`
* callback provides a value to be stored into the module config.
* The optional `applyfn` callback is called after a `CONFIG SET` command modified one or
* more configs using the `setfn` callback and can be used to atomically apply a config
* after several configs were changed together.
* If there are multiple configs with `applyfn` callbacks set by a single `CONFIG SET`
* command, they will be deduplicated if their `applyfn` function and `privdata` pointers
* are identical, and the callback will only be run once.
* Both the `setfn` and `applyfn` can return an error if the provided value is invalid or
* cannot be used.
* The config also declares a type for the value that is validated by Redis and
* provided to the module. The config system provides the following types:
*
* * Redis String: Binary safe string data.
* * Enum: One of a finite number of string tokens, provided during registration.
* * Numeric: 64 bit signed integer, which also supports min and max values.
* * Bool: Yes or no value.
*
* The `setfn` callback is expected to return REDISMODULE_OK when the value is successfully
* applied. It can also return REDISMODULE_ERR if the value can't be applied, and the
* *err pointer can be set with a RedisModuleString error message to provide to the client.
* This RedisModuleString will be freed by redis after returning from the set callback.
*
* All configs are registered with a name, a type, a default value, private data that is made
* available in the callbacks, as well as several flags that modify the behavior of the config.
* The name must only contain alphanumeric characters or dashes. The supported flags are:
*
* * REDISMODULE_CONFIG_DEFAULT: The default flags for a config. This creates a config that can be modified after startup.
* * REDISMODULE_CONFIG_IMMUTABLE: This config can only be provided loading time.
* * REDISMODULE_CONFIG_SENSITIVE: The value stored in this config is redacted from all logging.
* * REDISMODULE_CONFIG_HIDDEN: The name is hidden from `CONFIG GET` with pattern matching.
* * REDISMODULE_CONFIG_PROTECTED: This config will be only be modifiable based off the value of enable-protected-configs.
* * REDISMODULE_CONFIG_DENY_LOADING: This config is not modifiable while the server is loading data.
* * REDISMODULE_CONFIG_MEMORY: For numeric configs, this config will convert data unit notations into their byte equivalent.
*
* Default values are used on startup to set the value if it is not provided via the config file
* or command line. Default values are also used to compare to on a config rewrite.
*
* Notes:
*
* 1. On string config sets that the string passed to the set callback will be freed after execution and the module must retain it.
* 2. On string config gets the string will not be consumed and will be valid after execution.
*
* Example implementation:
*
* RedisModuleString *strval;
* int adjustable = 1;
* RedisModuleString *getStringConfigCommand(const char *name, void *privdata) {
* return strval;
* }
*
* int setStringConfigCommand(const char *name, RedisModuleString *new, void *privdata, RedisModuleString **err) {
* if (adjustable) {
* RedisModule_Free(strval);
* RedisModule_RetainString(NULL, new);
* strval = new;
* return REDISMODULE_OK;
* }
* *err = RedisModule_CreateString(NULL, "Not adjustable.", 15);
* return REDISMODULE_ERR;
* }
* ...
* RedisModule_RegisterStringConfig(ctx, "string", NULL, REDISMODULE_CONFIG_DEFAULT, getStringConfigCommand, setStringConfigCommand, NULL, NULL);
*
* If the registration fails, REDISMODULE_ERR is returned and one of the following
* errno is set:
* * EINVAL: The provided flags are invalid for the registration or the name of the config contains invalid characters.
* * EALREADY: The provided configuration name is already used. */
int RM_RegisterStringConfig(RedisModuleCtx *ctx, const char *name, const char *default_val, unsigned int flags, RedisModuleConfigGetStringFunc getfn, RedisModuleConfigSetStringFunc setfn, RedisModuleConfigApplyFunc applyfn, void *privdata) {
RedisModule *module = ctx->module;
sds config_name = sdsnew(name);
if (moduleConfigValidityCheck(module, config_name, flags, NUMERIC_CONFIG)) {
sdsfree(config_name);
return REDISMODULE_ERR;
}
ModuleConfig *new_config = createModuleConfig(config_name, applyfn, privdata, module);
sdsfree(config_name);
new_config->get_fn.get_string = getfn;
new_config->set_fn.set_string = setfn;
listAddNodeTail(module->module_configs, new_config);
flags = maskModuleConfigFlags(flags);
addModuleStringConfig(module->name, name, flags, new_config, default_val ? sdsnew(default_val) : NULL);
return REDISMODULE_OK;
}
/* Create a bool config that server clients can interact with via the
* `CONFIG SET`, `CONFIG GET`, and `CONFIG REWRITE` commands. See
* RedisModule_RegisterStringConfig for detailed information about configs. */
int RM_RegisterBoolConfig(RedisModuleCtx *ctx, const char *name, int default_val, unsigned int flags, RedisModuleConfigGetBoolFunc getfn, RedisModuleConfigSetBoolFunc setfn, RedisModuleConfigApplyFunc applyfn, void *privdata) {
RedisModule *module = ctx->module;
sds config_name = sdsnew(name);
if (moduleConfigValidityCheck(module, config_name, flags, BOOL_CONFIG)) {
sdsfree(config_name);
return REDISMODULE_ERR;
}
ModuleConfig *new_config = createModuleConfig(config_name, applyfn, privdata, module);
sdsfree(config_name);
new_config->get_fn.get_bool = getfn;
new_config->set_fn.set_bool = setfn;
listAddNodeTail(module->module_configs, new_config);
flags = maskModuleConfigFlags(flags);
addModuleBoolConfig(module->name, name, flags, new_config, default_val);
return REDISMODULE_OK;
}
/*
* Create an enum config that server clients can interact with via the
* `CONFIG SET`, `CONFIG GET`, and `CONFIG REWRITE` commands.
* Enum configs are a set of string tokens to corresponding integer values, where
* the string value is exposed to Redis clients but the value passed Redis and the
* module is the integer value. These values are defined in enum_values, an array
* of null-terminated c strings, and int_vals, an array of enum values who has an
* index partner in enum_values.
* Example Implementation:
* const char *enum_vals[3] = {"first", "second", "third"};
* const int int_vals[3] = {0, 2, 4};
* int enum_val = 0;
*
* int getEnumConfigCommand(const char *name, void *privdata) {
* return enum_val;
* }
*
* int setEnumConfigCommand(const char *name, int val, void *privdata, const char **err) {
* enum_val = val;
* return REDISMODULE_OK;
* }
* ...
* RedisModule_RegisterEnumConfig(ctx, "enum", 0, REDISMODULE_CONFIG_DEFAULT, enum_vals, int_vals, 3, getEnumConfigCommand, setEnumConfigCommand, NULL, NULL);
*
* See RedisModule_RegisterStringConfig for detailed general information about configs. */
int RM_RegisterEnumConfig(RedisModuleCtx *ctx, const char *name, int default_val, unsigned int flags, const char **enum_values, const int *int_values, int num_enum_vals, RedisModuleConfigGetEnumFunc getfn, RedisModuleConfigSetEnumFunc setfn, RedisModuleConfigApplyFunc applyfn, void *privdata) {
RedisModule *module = ctx->module;
sds config_name = sdsnew(name);
if (moduleConfigValidityCheck(module, config_name, flags, ENUM_CONFIG)) {
sdsfree(config_name);
return REDISMODULE_ERR;
}
ModuleConfig *new_config = createModuleConfig(config_name, applyfn, privdata, module);
sdsfree(config_name);
new_config->get_fn.get_enum = getfn;
new_config->set_fn.set_enum = setfn;
configEnum *enum_vals = zmalloc((num_enum_vals + 1) * sizeof(configEnum));
for (int i = 0; i < num_enum_vals; i++) {
enum_vals[i].name = zstrdup(enum_values[i]);
enum_vals[i].val = int_values[i];
}
enum_vals[num_enum_vals].name = NULL;
enum_vals[num_enum_vals].val = 0;
listAddNodeTail(module->module_configs, new_config);
flags = maskModuleConfigFlags(flags);
addModuleEnumConfig(module->name, name, flags, new_config, default_val, enum_vals);
return REDISMODULE_OK;
}
/*
* Create an integer config that server clients can interact with via the
* `CONFIG SET`, `CONFIG GET`, and `CONFIG REWRITE` commands. See
* RedisModule_RegisterStringConfig for detailed information about configs. */
int RM_RegisterNumericConfig(RedisModuleCtx *ctx, const char *name, long long default_val, unsigned int flags, long long min, long long max, RedisModuleConfigGetNumericFunc getfn, RedisModuleConfigSetNumericFunc setfn, RedisModuleConfigApplyFunc applyfn, void *privdata) {
RedisModule *module = ctx->module;
sds config_name = sdsnew(name);
if (moduleConfigValidityCheck(module, config_name, flags, NUMERIC_CONFIG)) {
sdsfree(config_name);
return REDISMODULE_ERR;
}
ModuleConfig *new_config = createModuleConfig(config_name, applyfn, privdata, module);
sdsfree(config_name);
new_config->get_fn.get_numeric = getfn;
new_config->set_fn.set_numeric = setfn;
listAddNodeTail(module->module_configs, new_config);
unsigned int numeric_flags = maskModuleNumericConfigFlags(flags);
flags = maskModuleConfigFlags(flags);
addModuleNumericConfig(module->name, name, flags, new_config, default_val, numeric_flags, min, max);
return REDISMODULE_OK;
}
/* Applies all pending configurations on the module load. This should be called
* after all of the configurations have been registered for the module inside of RedisModule_OnLoad.
* This API needs to be called when configurations are provided in either `MODULE LOADEX`
* or provided as startup arguments. */
int RM_LoadConfigs(RedisModuleCtx *ctx) {
if (!ctx || !ctx->module) {
return REDISMODULE_ERR;
}
RedisModule *module = ctx->module;
/* Load configs from conf file or arguments from loadex */
if (loadModuleConfigs(module)) return REDISMODULE_ERR;
return REDISMODULE_OK;
}
/* Redis MODULE command. /* Redis MODULE command.
* *
* MODULE LIST * MODULE LIST
* MODULE LOAD <path> [args...] * MODULE LOAD <path> [args...]
* MODULE LOADEX <path> [[CONFIG NAME VALUE] [CONFIG NAME VALUE]] [ARGS ...]
* MODULE UNLOAD <name> * MODULE UNLOAD <name>
*/ */
void moduleCommand(client *c) { void moduleCommand(client *c) {
...@@ -10979,6 +11695,8 @@ void moduleCommand(client *c) { ...@@ -10979,6 +11695,8 @@ void moduleCommand(client *c) {
" Return a list of loaded modules.", " Return a list of loaded modules.",
"LOAD <path> [<arg> ...]", "LOAD <path> [<arg> ...]",
" Load a module library from <path>, passing to it any optional arguments.", " Load a module library from <path>, passing to it any optional arguments.",
"LOADEX <path> [[CONFIG NAME VALUE] [CONFIG NAME VALUE]] [ARGS ...]",
" Load a module library from <path>, while passing it module configurations and optional arguments.",
"UNLOAD <name>", "UNLOAD <name>",
" Unload a module.", " Unload a module.",
NULL NULL
...@@ -10993,11 +11711,30 @@ NULL ...@@ -10993,11 +11711,30 @@ NULL
argv = &c->argv[3]; argv = &c->argv[3];
} }
if (moduleLoad(c->argv[2]->ptr,(void **)argv,argc) == C_OK) if (moduleLoad(c->argv[2]->ptr,(void **)argv,argc, 0) == C_OK)
addReply(c,shared.ok); addReply(c,shared.ok);
else else
addReplyError(c, addReplyError(c,
"Error loading the extension. Please check the server logs."); "Error loading the extension. Please check the server logs.");
} else if (!strcasecmp(subcmd,"loadex") && c->argc >= 3) {
robj **argv = NULL;
int argc = 0;
if (c->argc > 3) {
argc = c->argc - 3;
argv = &c->argv[3];
}
/* If this is a loadex command we want to populate server.module_configs_queue with
* sds NAME VALUE pairs. We also want to increment argv to just after ARGS, if supplied. */
if (parseLoadexArguments((RedisModuleString ***) &argv, &argc) == REDISMODULE_OK &&
moduleLoad(c->argv[2]->ptr, (void **)argv, argc, 1) == C_OK)
addReply(c,shared.ok);
else {
dictEmpty(server.module_configs_queue, NULL);
addReplyError(c,
"Error loading the extension. Please check the server logs.");
}
} else if (!strcasecmp(subcmd,"unload") && c->argc == 3) { } else if (!strcasecmp(subcmd,"unload") && c->argc == 3) {
if (moduleUnload(c->argv[2]->ptr) == C_OK) if (moduleUnload(c->argv[2]->ptr) == C_OK)
addReply(c,shared.ok); addReply(c,shared.ok);
...@@ -11785,6 +12522,7 @@ void moduleRegisterCoreAPI(void) { ...@@ -11785,6 +12522,7 @@ void moduleRegisterCoreAPI(void) {
REGISTER_API(IsSubEventSupported); REGISTER_API(IsSubEventSupported);
REGISTER_API(GetServerVersion); REGISTER_API(GetServerVersion);
REGISTER_API(GetClientCertificate); REGISTER_API(GetClientCertificate);
REGISTER_API(RedactClientCommandArgument);
REGISTER_API(GetCommandKeys); REGISTER_API(GetCommandKeys);
REGISTER_API(GetCommandKeysWithFlags); REGISTER_API(GetCommandKeysWithFlags);
REGISTER_API(GetCurrentCommandName); REGISTER_API(GetCurrentCommandName);
...@@ -11799,4 +12537,9 @@ void moduleRegisterCoreAPI(void) { ...@@ -11799,4 +12537,9 @@ void moduleRegisterCoreAPI(void) {
REGISTER_API(EventLoopDel); REGISTER_API(EventLoopDel);
REGISTER_API(EventLoopAddOneShot); REGISTER_API(EventLoopAddOneShot);
REGISTER_API(Yield); REGISTER_API(Yield);
REGISTER_API(RegisterBoolConfig);
REGISTER_API(RegisterNumericConfig);
REGISTER_API(RegisterStringConfig);
REGISTER_API(RegisterEnumConfig);
REGISTER_API(LoadConfigs);
} }
...@@ -31,7 +31,6 @@ ...@@ -31,7 +31,6 @@
* POSSIBILITY OF SUCH DAMAGE. * POSSIBILITY OF SUCH DAMAGE.
*/ */
#define REDISMODULE_EXPERIMENTAL_API
#include "../redismodule.h" #include "../redismodule.h"
#include <pthread.h> #include <pthread.h>
#include <unistd.h> #include <unistd.h>
......
...@@ -31,7 +31,6 @@ ...@@ -31,7 +31,6 @@
* POSSIBILITY OF SUCH DAMAGE. * POSSIBILITY OF SUCH DAMAGE.
*/ */
#define REDISMODULE_EXPERIMENTAL_API
#include "../redismodule.h" #include "../redismodule.h"
#include <stdio.h> #include <stdio.h>
#include <stdlib.h> #include <stdlib.h>
......
...@@ -30,7 +30,6 @@ ...@@ -30,7 +30,6 @@
* POSSIBILITY OF SUCH DAMAGE. * POSSIBILITY OF SUCH DAMAGE.
*/ */
#define REDISMODULE_EXPERIMENTAL_API
#include "../redismodule.h" #include "../redismodule.h"
#include <stdio.h> #include <stdio.h>
#include <stdlib.h> #include <stdlib.h>
......
...@@ -33,7 +33,6 @@ ...@@ -33,7 +33,6 @@
* POSSIBILITY OF SUCH DAMAGE. * POSSIBILITY OF SUCH DAMAGE.
*/ */
#define REDISMODULE_EXPERIMENTAL_API
#include "../redismodule.h" #include "../redismodule.h"
#include <stdio.h> #include <stdio.h>
#include <stdlib.h> #include <stdlib.h>
......
...@@ -30,7 +30,6 @@ ...@@ -30,7 +30,6 @@
* POSSIBILITY OF SUCH DAMAGE. * POSSIBILITY OF SUCH DAMAGE.
*/ */
#define REDISMODULE_EXPERIMENTAL_API
#include "../redismodule.h" #include "../redismodule.h"
#include <stdio.h> #include <stdio.h>
#include <stdlib.h> #include <stdlib.h>
......
...@@ -30,7 +30,6 @@ ...@@ -30,7 +30,6 @@
* POSSIBILITY OF SUCH DAMAGE. * POSSIBILITY OF SUCH DAMAGE.
*/ */
#define REDISMODULE_EXPERIMENTAL_API
#include "../redismodule.h" #include "../redismodule.h"
#include <stdio.h> #include <stdio.h>
#include <stdlib.h> #include <stdlib.h>
......
...@@ -147,7 +147,6 @@ client *createClient(connection *conn) { ...@@ -147,7 +147,6 @@ client *createClient(connection *conn) {
c->ref_block_pos = 0; c->ref_block_pos = 0;
c->qb_pos = 0; c->qb_pos = 0;
c->querybuf = sdsempty(); c->querybuf = sdsempty();
c->pending_querybuf = sdsempty();
c->querybuf_peak = 0; c->querybuf_peak = 0;
c->reqtype = 0; c->reqtype = 0;
c->argc = 0; c->argc = 0;
...@@ -167,6 +166,7 @@ client *createClient(connection *conn) { ...@@ -167,6 +166,7 @@ client *createClient(connection *conn) {
c->repl_start_cmd_stream_on_ack = 0; c->repl_start_cmd_stream_on_ack = 0;
c->reploff = 0; c->reploff = 0;
c->read_reploff = 0; c->read_reploff = 0;
c->repl_applied = 0;
c->repl_ack_off = 0; c->repl_ack_off = 0;
c->repl_ack_time = 0; c->repl_ack_time = 0;
c->repl_last_partial_write = 0; c->repl_last_partial_write = 0;
...@@ -201,7 +201,7 @@ client *createClient(connection *conn) { ...@@ -201,7 +201,7 @@ client *createClient(connection *conn) {
c->pending_read_list_node = NULL; c->pending_read_list_node = NULL;
c->client_tracking_redirection = 0; c->client_tracking_redirection = 0;
c->client_tracking_prefixes = NULL; c->client_tracking_prefixes = NULL;
c->last_memory_usage = c->last_memory_usage_on_bucket_update = 0; c->last_memory_usage = 0;
c->last_memory_type = CLIENT_TYPE_NORMAL; c->last_memory_type = CLIENT_TYPE_NORMAL;
c->auth_callback = NULL; c->auth_callback = NULL;
c->auth_callback_privdata = NULL; c->auth_callback_privdata = NULL;
...@@ -1568,7 +1568,6 @@ void freeClient(client *c) { ...@@ -1568,7 +1568,6 @@ void freeClient(client *c) {
/* Free the query buffer */ /* Free the query buffer */
sdsfree(c->querybuf); sdsfree(c->querybuf);
sdsfree(c->pending_querybuf);
c->querybuf = NULL; c->querybuf = NULL;
/* Deallocate structures used to block on blocking ops. */ /* Deallocate structures used to block on blocking ops. */
...@@ -1940,7 +1939,7 @@ int writeToClient(client *c, int handler_installed) { ...@@ -1940,7 +1939,7 @@ int writeToClient(client *c, int handler_installed) {
if (!clientHasPendingReplies(c)) { if (!clientHasPendingReplies(c)) {
c->sentlen = 0; c->sentlen = 0;
/* Note that writeToClient() is called in a threaded way, but /* Note that writeToClient() is called in a threaded way, but
* adDeleteFileEvent() is not thread safe: however writeToClient() * aeDeleteFileEvent() is not thread safe: however writeToClient()
* is always called with handler_installed set to 0 from threads * is always called with handler_installed set to 0 from threads
* so we are fine. */ * so we are fine. */
if (handler_installed) { if (handler_installed) {
...@@ -1954,6 +1953,10 @@ int writeToClient(client *c, int handler_installed) { ...@@ -1954,6 +1953,10 @@ int writeToClient(client *c, int handler_installed) {
return C_ERR; return C_ERR;
} }
} }
/* Update client's memory usage after writing.
* Since this isn't thread safe we do this conditionally. In case of threaded writes this is done in
* handleClientsWithPendingWritesUsingThreads(). */
if (io_threads_op == IO_THREADS_OP_IDLE)
updateClientMemUsage(c); updateClientMemUsage(c);
return C_OK; return C_OK;
} }
...@@ -2128,7 +2131,7 @@ int processInlineBuffer(client *c) { ...@@ -2128,7 +2131,7 @@ int processInlineBuffer(client *c) {
* we got some desynchronization in the protocol, for example * we got some desynchronization in the protocol, for example
* because of a PSYNC gone bad. * because of a PSYNC gone bad.
* *
* However the is an exception: masters may send us just a newline * However there is an exception: masters may send us just a newline
* to keep the connection active. */ * to keep the connection active. */
if (querylen != 0 && c->flags & CLIENT_MASTER) { if (querylen != 0 && c->flags & CLIENT_MASTER) {
sdsfreesplitres(argv,argc); sdsfreesplitres(argv,argc);
...@@ -2292,8 +2295,12 @@ int processMultibulkBuffer(client *c) { ...@@ -2292,8 +2295,12 @@ int processMultibulkBuffer(client *c) {
} }
c->qb_pos = newline-c->querybuf+2; c->qb_pos = newline-c->querybuf+2;
if (ll >= PROTO_MBULK_BIG_ARG) { if (!(c->flags & CLIENT_MASTER) && ll >= PROTO_MBULK_BIG_ARG) {
/* If we are going to read a large object from network /* When the client is not a master client (because master
* client's querybuf can only be trimmed after data applied
* and sent to replicas).
*
* If we are going to read a large object from network
* try to make it likely that it will start at c->querybuf * try to make it likely that it will start at c->querybuf
* boundary so that we can optimize object creation * boundary so that we can optimize object creation
* avoiding a large copy of data. * avoiding a large copy of data.
...@@ -2324,10 +2331,11 @@ int processMultibulkBuffer(client *c) { ...@@ -2324,10 +2331,11 @@ int processMultibulkBuffer(client *c) {
c->argv = zrealloc(c->argv, sizeof(robj*)*c->argv_len); c->argv = zrealloc(c->argv, sizeof(robj*)*c->argv_len);
} }
/* Optimization: if the buffer contains JUST our bulk element /* Optimization: if a non-master client's buffer contains JUST our bulk element
* instead of creating a new object by *copying* the sds we * instead of creating a new object by *copying* the sds we
* just use the current sds string. */ * just use the current sds string. */
if (c->qb_pos == 0 && if (!(c->flags & CLIENT_MASTER) &&
c->qb_pos == 0 &&
c->bulklen >= PROTO_MBULK_BIG_ARG && c->bulklen >= PROTO_MBULK_BIG_ARG &&
sdslen(c->querybuf) == (size_t)(c->bulklen+2)) sdslen(c->querybuf) == (size_t)(c->bulklen+2))
{ {
...@@ -2388,8 +2396,8 @@ void commandProcessed(client *c) { ...@@ -2388,8 +2396,8 @@ void commandProcessed(client *c) {
if (c->flags & CLIENT_MASTER) { if (c->flags & CLIENT_MASTER) {
long long applied = c->reploff - prev_offset; long long applied = c->reploff - prev_offset;
if (applied) { if (applied) {
replicationFeedStreamFromMasterStream(c->pending_querybuf,applied); replicationFeedStreamFromMasterStream(c->querybuf+c->repl_applied,applied);
sdsrange(c->pending_querybuf,applied,-1); c->repl_applied += applied;
} }
} }
} }
...@@ -2432,13 +2440,22 @@ int processCommandAndResetClient(client *c) { ...@@ -2432,13 +2440,22 @@ int processCommandAndResetClient(client *c) {
/* This function will execute any fully parsed commands pending on /* This function will execute any fully parsed commands pending on
* the client. Returns C_ERR if the client is no longer valid after executing * the client. Returns C_ERR if the client is no longer valid after executing
* the command, and C_OK for all other cases. */ * the command, and C_OK for all other cases. */
int processPendingCommandsAndResetClient(client *c) { int processPendingCommandAndInputBuffer(client *c) {
if (c->flags & CLIENT_PENDING_COMMAND) { if (c->flags & CLIENT_PENDING_COMMAND) {
c->flags &= ~CLIENT_PENDING_COMMAND; c->flags &= ~CLIENT_PENDING_COMMAND;
if (processCommandAndResetClient(c) == C_ERR) { if (processCommandAndResetClient(c) == C_ERR) {
return C_ERR; return C_ERR;
} }
} }
/* Now process client if it has more data in it's buffer.
*
* Note: when a master client steps into this function,
* it can always satisfy this condition, because its querbuf
* contains data not applied. */
if (c->querybuf && sdslen(c->querybuf) > 0) {
return processInputBuffer(c);
}
return C_OK; return C_OK;
} }
...@@ -2510,8 +2527,26 @@ int processInputBuffer(client *c) { ...@@ -2510,8 +2527,26 @@ int processInputBuffer(client *c) {
} }
} }
if (c->flags & CLIENT_MASTER) {
/* If the client is a master, trim the querybuf to repl_applied,
* since master client is very special, its querybuf not only
* used to parse command, but also proxy to sub-replicas.
*
* Here are some scenarios we cannot trim to qb_pos:
* 1. we don't receive complete command from master
* 2. master client blocked cause of client pause
* 3. io threads operate read, master client flagged with CLIENT_PENDING_COMMAND
*
* In these scenarios, qb_pos points to the part of the current command
* or the beginning of next command, and the current command is not applied yet,
* so the repl_applied is not equal to qb_pos. */
if (c->repl_applied) {
sdsrange(c->querybuf,c->repl_applied,-1);
c->qb_pos -= c->repl_applied;
c->repl_applied = 0;
}
} else if (c->qb_pos) {
/* Trim to pos */ /* Trim to pos */
if (c->qb_pos) {
sdsrange(c->querybuf,c->qb_pos,-1); sdsrange(c->querybuf,c->qb_pos,-1);
c->qb_pos = 0; c->qb_pos = 0;
} }
...@@ -2519,6 +2554,7 @@ int processInputBuffer(client *c) { ...@@ -2519,6 +2554,7 @@ int processInputBuffer(client *c) {
/* Update client memory usage after processing the query buffer, this is /* Update client memory usage after processing the query buffer, this is
* important in case the query buffer is big and wasn't drained during * important in case the query buffer is big and wasn't drained during
* the above loop (because of partially sent big commands). */ * the above loop (because of partially sent big commands). */
if (io_threads_op == IO_THREADS_OP_IDLE)
updateClientMemUsage(c); updateClientMemUsage(c);
return C_OK; return C_OK;
...@@ -2546,16 +2582,22 @@ void readQueryFromClient(connection *conn) { ...@@ -2546,16 +2582,22 @@ void readQueryFromClient(connection *conn) {
if (c->reqtype == PROTO_REQ_MULTIBULK && c->multibulklen && c->bulklen != -1 if (c->reqtype == PROTO_REQ_MULTIBULK && c->multibulklen && c->bulklen != -1
&& c->bulklen >= PROTO_MBULK_BIG_ARG) && c->bulklen >= PROTO_MBULK_BIG_ARG)
{ {
ssize_t remaining = (size_t)(c->bulklen+2)-sdslen(c->querybuf); ssize_t remaining = (size_t)(c->bulklen+2)-(sdslen(c->querybuf)-c->qb_pos);
big_arg = 1; big_arg = 1;
/* Note that the 'remaining' variable may be zero in some edge case, /* Note that the 'remaining' variable may be zero in some edge case,
* for example once we resume a blocked client after CLIENT PAUSE. */ * for example once we resume a blocked client after CLIENT PAUSE. */
if (remaining > 0) readlen = remaining; if (remaining > 0) readlen = remaining;
/* Master client needs expand the readlen when meet BIG_ARG(see #9100),
* but doesn't need align to the next arg, we can read more data. */
if (c->flags & CLIENT_MASTER && readlen < PROTO_IOBUF_LEN)
readlen = PROTO_IOBUF_LEN;
} }
qblen = sdslen(c->querybuf); qblen = sdslen(c->querybuf);
if (big_arg || sdsalloc(c->querybuf) < PROTO_IOBUF_LEN) { if (!(c->flags & CLIENT_MASTER) && // master client's querybuf can grow greedy.
(big_arg || sdsalloc(c->querybuf) < PROTO_IOBUF_LEN)) {
/* When reading a BIG_ARG we won't be reading more than that one arg /* When reading a BIG_ARG we won't be reading more than that one arg
* into the query buffer, so we don't need to pre-allocate more than we * into the query buffer, so we don't need to pre-allocate more than we
* need, so using the non-greedy growing. For an initial allocation of * need, so using the non-greedy growing. For an initial allocation of
...@@ -2585,12 +2627,6 @@ void readQueryFromClient(connection *conn) { ...@@ -2585,12 +2627,6 @@ void readQueryFromClient(connection *conn) {
} }
freeClientAsync(c); freeClientAsync(c);
goto done; goto done;
} else if (c->flags & CLIENT_MASTER) {
/* Append the query buffer to the pending (not applied) buffer
* of the master. We'll use this buffer later in order to have a
* copy of the string applied by the last command executed. */
c->pending_querybuf = sdscatlen(c->pending_querybuf,
c->querybuf+qblen,nread);
} }
sdsIncrLen(c->querybuf,nread); sdsIncrLen(c->querybuf,nread);
...@@ -2868,8 +2904,6 @@ void clientCommand(client *c) { ...@@ -2868,8 +2904,6 @@ void clientCommand(client *c) {
" Control the replies sent to the current connection.", " Control the replies sent to the current connection.",
"SETNAME <name>", "SETNAME <name>",
" Assign the name <name> to the current connection.", " Assign the name <name> to the current connection.",
"GETNAME",
" Get the name of the current connection.",
"UNBLOCK <clientid> [TIMEOUT|ERROR]", "UNBLOCK <clientid> [TIMEOUT|ERROR]",
" Unblock the specified blocked client.", " Unblock the specified blocked client.",
"TRACKING (ON|OFF) [REDIRECT <id>] [BCAST] [PREFIX <prefix> [...]]", "TRACKING (ON|OFF) [REDIRECT <id>] [BCAST] [PREFIX <prefix> [...]]",
...@@ -3075,6 +3109,10 @@ NULL ...@@ -3075,6 +3109,10 @@ NULL
if (getLongLongFromObjectOrReply(c,c->argv[2],&id,NULL) if (getLongLongFromObjectOrReply(c,c->argv[2],&id,NULL)
!= C_OK) return; != C_OK) return;
struct client *target = lookupClientByID(id); struct client *target = lookupClientByID(id);
/* Note that we never try to unblock a client blocked on a module command, which
* doesn't have a timeout callback (even in the case of UNBLOCK ERROR).
* The reason is that we assume that if a command doesn't expect to be timedout,
* it also doesn't expect to be unblocked by CLIENT UNBLOCK */
if (target && target->flags & CLIENT_BLOCKED && moduleBlockedClientMayTimeout(target)) { if (target && target->flags & CLIENT_BLOCKED && moduleBlockedClientMayTimeout(target)) {
if (unblock_error) if (unblock_error)
addReplyError(target, addReplyError(target,
...@@ -3464,7 +3502,11 @@ static void retainOriginalCommandVector(client *c) { ...@@ -3464,7 +3502,11 @@ static void retainOriginalCommandVector(client *c) {
* original_argv array. */ * original_argv array. */
void redactClientCommandArgument(client *c, int argc) { void redactClientCommandArgument(client *c, int argc) {
retainOriginalCommandVector(c); retainOriginalCommandVector(c);
decrRefCount(c->argv[argc]); if (c->original_argv[argc] == shared.redacted) {
/* This argument has already been redacted */
return;
}
decrRefCount(c->original_argv[argc]);
c->original_argv[argc] = shared.redacted; c->original_argv[argc] = shared.redacted;
} }
...@@ -4165,8 +4207,8 @@ int handleClientsWithPendingWritesUsingThreads(void) { ...@@ -4165,8 +4207,8 @@ int handleClientsWithPendingWritesUsingThreads(void) {
while((ln = listNext(&li))) { while((ln = listNext(&li))) {
client *c = listNodeValue(ln); client *c = listNodeValue(ln);
/* Update the client in the mem usage buckets after we're done processing it in the io-threads */ /* Update the client in the mem usage after we're done processing it in the io-threads */
updateClientMemUsageBucket(c); updateClientMemUsage(c);
/* Install the write handler if there are pending writes in some /* Install the write handler if there are pending writes in some
* of the clients. */ * of the clients. */
...@@ -4274,17 +4316,10 @@ int handleClientsWithPendingReadsUsingThreads(void) { ...@@ -4274,17 +4316,10 @@ int handleClientsWithPendingReadsUsingThreads(void) {
continue; continue;
} }
/* Once io-threads are idle we can update the client in the mem usage buckets */ /* Once io-threads are idle we can update the client in the mem usage */
updateClientMemUsageBucket(c); updateClientMemUsage(c);
if (processPendingCommandsAndResetClient(c) == C_ERR) {
/* If the client is no longer valid, we avoid
* processing the client later. So we just go
* to the next. */
continue;
}
if (processInputBuffer(c) == C_ERR) { if (processPendingCommandAndInputBuffer(c) == C_ERR) {
/* If the client is no longer valid, we avoid /* If the client is no longer valid, we avoid
* processing the client later. So we just go * processing the client later. So we just go
* to the next. */ * to the next. */
......
...@@ -1532,7 +1532,7 @@ NULL ...@@ -1532,7 +1532,7 @@ NULL
} else if (!strcasecmp(c->argv[1]->ptr,"stats") && c->argc == 2) { } else if (!strcasecmp(c->argv[1]->ptr,"stats") && c->argc == 2) {
struct redisMemOverhead *mh = getMemoryOverheadData(); struct redisMemOverhead *mh = getMemoryOverheadData();
addReplyMapLen(c,26+mh->num_dbs); addReplyMapLen(c,27+mh->num_dbs);
addReplyBulkCString(c,"peak.allocated"); addReplyBulkCString(c,"peak.allocated");
addReplyLongLong(c,mh->peak_allocated); addReplyLongLong(c,mh->peak_allocated);
...@@ -1552,6 +1552,9 @@ NULL ...@@ -1552,6 +1552,9 @@ NULL
addReplyBulkCString(c,"clients.normal"); addReplyBulkCString(c,"clients.normal");
addReplyLongLong(c,mh->clients_normal); addReplyLongLong(c,mh->clients_normal);
addReplyBulkCString(c,"cluster.links");
addReplyLongLong(c,mh->cluster_links);
addReplyBulkCString(c,"aof.buffer"); addReplyBulkCString(c,"aof.buffer");
addReplyLongLong(c,mh->aof_buffer); addReplyLongLong(c,mh->aof_buffer);
......
...@@ -38,7 +38,7 @@ ...@@ -38,7 +38,7 @@
*/ */
#include <sys/types.h> #include <sys/types.h>
#include <stdint.h>
#include <errno.h> #include <errno.h>
#include <stdlib.h> #include <stdlib.h>
...@@ -62,7 +62,7 @@ static inline void swapfunc (char *, char *, size_t, int); ...@@ -62,7 +62,7 @@ static inline void swapfunc (char *, char *, size_t, int);
} while (--i > 0); \ } while (--i > 0); \
} }
#define SWAPINIT(a, es) swaptype = ((char *)a - (char *)0) % sizeof(long) || \ #define SWAPINIT(a, es) swaptype = (uintptr_t)a % sizeof(long) || \
es % sizeof(long) ? 2 : es == sizeof(long)? 0 : 1; es % sizeof(long) ? 2 : es == sizeof(long)? 0 : 1;
static inline void static inline void
......
...@@ -905,9 +905,9 @@ int raxInsert(rax *rax, unsigned char *s, size_t len, void *data, void **old) { ...@@ -905,9 +905,9 @@ int raxInsert(rax *rax, unsigned char *s, size_t len, void *data, void **old) {
return raxGenericInsert(rax,s,len,data,old,1); return raxGenericInsert(rax,s,len,data,old,1);
} }
/* Non overwriting insert function: this if an element with the same key /* Non overwriting insert function: if an element with the same key
* exists, the value is not updated and the function returns 0. * exists, the value is not updated and the function returns 0.
* This is a just a wrapper for raxGenericInsert(). */ * This is just a wrapper for raxGenericInsert(). */
int raxTryInsert(rax *rax, unsigned char *s, size_t len, void *data, void **old) { int raxTryInsert(rax *rax, unsigned char *s, size_t len, void *data, void **old) {
return raxGenericInsert(rax,s,len,data,old,0); return raxGenericInsert(rax,s,len,data,old,0);
} }
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment