Commit 20618c71 authored by Oran Agra's avatar Oran Agra
Browse files

Merge remote-tracking branch 'origin/unstable' into 7.0

parents fb4e0d40 89772ed8
...@@ -41,6 +41,97 @@ ...@@ -41,6 +41,97 @@
#include <ctype.h> #include <ctype.h>
#include <math.h> #include <math.h>
/* Globals that are added by the Lua libraries */
static char *libraries_allow_list[] = {
"string",
"cjson",
"bit",
"cmsgpack",
"math",
"table",
"struct",
NULL,
};
/* Redis Lua API globals */
static char *redis_api_allow_list[] = {
"redis",
"__redis__err__handler", /* error handler for eval, currently located on globals.
Should move to registry. */
NULL,
};
/* Lua builtins */
static char *lua_builtins_allow_list[] = {
"xpcall",
"tostring",
"getfenv",
"setmetatable",
"next",
"assert",
"tonumber",
"rawequal",
"collectgarbage",
"getmetatable",
"rawset",
"pcall",
"coroutine",
"type",
"_G",
"select",
"unpack",
"gcinfo",
"pairs",
"rawget",
"loadstring",
"ipairs",
"_VERSION",
"setfenv",
"load",
"error",
NULL,
};
/* Lua builtins which are not documented on the Lua documentation */
static char *lua_builtins_not_documented_allow_list[] = {
"newproxy",
NULL,
};
/* Lua builtins which are allowed on initialization but will be removed right after */
static char *lua_builtins_removed_after_initialization_allow_list[] = {
"debug", /* debug will be set to nil after the error handler will be created */
NULL,
};
/* Those allow lists was created from the globals that was
* available to the user when the allow lists was first introduce.
* Because we do not want to break backward compatibility we keep
* all the globals. The allow lists will prevent us from accidentally
* creating unwanted globals in the future.
*
* Also notice that the allow list is only checked on start time,
* after that the global table is locked so not need to check anything.*/
static char **allow_lists[] = {
libraries_allow_list,
redis_api_allow_list,
lua_builtins_allow_list,
lua_builtins_not_documented_allow_list,
lua_builtins_removed_after_initialization_allow_list,
NULL,
};
/* Deny list contains elements which we know we do not want to add to globals
* and there is no need to print a warning message form them. We will print a
* log message only if an element was added to the globals and the element is
* not on the allow list nor on the back list. */
static char *deny_list[] = {
"dofile",
"loadfile",
"print",
NULL,
};
static int redis_math_random (lua_State *L); static int redis_math_random (lua_State *L);
static int redis_math_randomseed (lua_State *L); static int redis_math_randomseed (lua_State *L);
static void redisProtocolToLuaType_Int(void *ctx, long long val, const char *proto, size_t proto_len); static void redisProtocolToLuaType_Int(void *ctx, long long val, const char *proto, size_t proto_len);
...@@ -1113,15 +1204,6 @@ static void luaLoadLibraries(lua_State *lua) { ...@@ -1113,15 +1204,6 @@ static void luaLoadLibraries(lua_State *lua) {
#endif #endif
} }
/* Remove a functions that we don't want to expose to the Redis scripting
* environment. */
static void luaRemoveUnsupportedFunctions(lua_State *lua) {
lua_pushnil(lua);
lua_setglobal(lua,"loadfile");
lua_pushnil(lua);
lua_setglobal(lua,"dofile");
}
/* Return sds of the string value located on stack at the given index. /* Return sds of the string value located on stack at the given index.
* Return NULL if the value is not a string. */ * Return NULL if the value is not a string. */
sds luaGetStringSds(lua_State *lua, int index) { sds luaGetStringSds(lua_State *lua, int index) {
...@@ -1135,107 +1217,120 @@ sds luaGetStringSds(lua_State *lua, int index) { ...@@ -1135,107 +1217,120 @@ sds luaGetStringSds(lua_State *lua, int index) {
return str_sds; return str_sds;
} }
/* This function installs metamethods in the global table _G that prevent static int luaProtectedTableError(lua_State *lua) {
* the creation of globals accidentally. int argc = lua_gettop(lua);
* if (argc != 2) {
* It should be the last to be called in the scripting engine initialization serverLog(LL_WARNING, "malicious code trying to call luaProtectedTableError with wrong arguments");
* sequence, because it may interact with creation of globals. luaL_error(lua, "Wrong number of arguments to luaProtectedTableError");
* }
* On Legacy Lua (eval) we need to check 'w ~= \"main\"' otherwise we will not be able if (!lua_isstring(lua, -1) && !lua_isnumber(lua, -1)) {
* to create the global 'function <sha> ()' variable. On Functions Lua engine we do not use luaL_error(lua, "Second argument to luaProtectedTableError must be a string or number");
* this trick so it's not needed. */ }
void luaEnableGlobalsProtection(lua_State *lua, int is_eval) { const char *variable_name = lua_tostring(lua, -1);
char *s[32]; luaL_error(lua, "Script attempted to access nonexistent global variable '%s'", variable_name);
sds code = sdsempty(); return 0;
int j = 0;
/* strict.lua from: http://metalua.luaforge.net/src/lib/strict.lua.html.
* Modified to be adapted to Redis. */
s[j++]="local dbg=debug\n";
s[j++]="local mt = {}\n";
s[j++]="setmetatable(_G, mt)\n";
s[j++]="mt.__newindex = function (t, n, v)\n";
s[j++]=" if dbg.getinfo(2) then\n";
s[j++]=" local w = dbg.getinfo(2, \"S\").what\n";
s[j++]= is_eval ? " if w ~= \"main\" and w ~= \"C\" then\n" : " if w ~= \"C\" then\n";
s[j++]=" error(\"Script attempted to create global variable '\"..tostring(n)..\"'\", 2)\n";
s[j++]=" end\n";
s[j++]=" end\n";
s[j++]=" rawset(t, n, v)\n";
s[j++]="end\n";
s[j++]="mt.__index = function (t, n)\n";
s[j++]=" if dbg.getinfo(2) and dbg.getinfo(2, \"S\").what ~= \"C\" then\n";
s[j++]=" error(\"Script attempted to access nonexistent global variable '\"..tostring(n)..\"'\", 2)\n";
s[j++]=" end\n";
s[j++]=" return rawget(t, n)\n";
s[j++]="end\n";
s[j++]="debug = nil\n";
s[j++]=NULL;
for (j = 0; s[j] != NULL; j++) code = sdscatlen(code,s[j],strlen(s[j]));
luaL_loadbuffer(lua,code,sdslen(code),"@enable_strict_lua");
lua_pcall(lua,0,0,0);
sdsfree(code);
} }
/* Create a global protection function and put it to registry. /* Set a special metatable on the table on the top of the stack.
* This need to be called once in the lua_State lifetime. * The metatable will raise an error if the user tries to fetch
* After called it is possible to use luaSetGlobalProtection * an un-existing value.
* to set global protection on a give table.
* *
* The function assumes the Lua stack have a least enough * The function assumes the Lua stack have a least enough
* space to push 2 element, its up to the caller to verify * space to push 2 element, its up to the caller to verify
* this before calling this function. * this before calling this function. */
* void luaSetErrorMetatable(lua_State *lua) {
* Notice, the difference between this and luaEnableGlobalsProtection lua_newtable(lua); /* push metatable */
* is that luaEnableGlobalsProtection is enabling global protection lua_pushcfunction(lua, luaProtectedTableError); /* push get error handler */
* on the current Lua globals. This registering a global protection lua_setfield(lua, -2, "__index");
* function that later can be applied on any table. */ lua_setmetatable(lua, -2);
void luaRegisterGlobalProtectionFunction(lua_State *lua) { }
lua_pushstring(lua, REGISTRY_SET_GLOBALS_PROTECTION_NAME);
char *global_protection_func = "local dbg = debug\n" static int luaNewIndexAllowList(lua_State *lua) {
"local globals_protection = function (t)\n" int argc = lua_gettop(lua);
" local mt = {}\n" if (argc != 3) {
" setmetatable(t, mt)\n" serverLog(LL_WARNING, "malicious code trying to call luaProtectedTableError with wrong arguments");
" mt.__newindex = function (t, n, v)\n" luaL_error(lua, "Wrong number of arguments to luaNewIndexAllowList");
" if dbg.getinfo(2) then\n" }
" local w = dbg.getinfo(2, \"S\").what\n" if (!lua_istable(lua, -3)) {
" if w ~= \"C\" then\n" luaL_error(lua, "first argument to luaNewIndexAllowList must be a table");
" error(\"Script attempted to create global variable '\"..tostring(n)..\"'\", 2)\n" }
" end" if (!lua_isstring(lua, -2) && !lua_isnumber(lua, -2)) {
" end" luaL_error(lua, "Second argument to luaNewIndexAllowList must be a string or number");
" rawset(t, n, v)\n" }
" end\n" const char *variable_name = lua_tostring(lua, -2);
" mt.__index = function (t, n)\n" /* check if the key is in our allow list */
" if dbg.getinfo(2) and dbg.getinfo(2, \"S\").what ~= \"C\" then\n"
" error(\"Script attempted to access nonexistent global variable '\"..tostring(n)..\"'\", 2)\n" char ***allow_l = allow_lists;
" end\n" for (; *allow_l ; ++allow_l){
" return rawget(t, n)\n" char **c = *allow_l;
" end\n" for (; *c ; ++c) {
"end\n" if (strcmp(*c, variable_name) == 0) {
"return globals_protection"; break;
int res = luaL_loadbuffer(lua, global_protection_func, strlen(global_protection_func), "@global_protection_def"); }
serverAssert(res == 0); }
res = lua_pcall(lua,0,1,0); if (*c) {
serverAssert(res == 0); break;
lua_settable(lua, LUA_REGISTRYINDEX); }
}
if (!*allow_l) {
/* Search the value on the back list, if its there we know that it was removed
* on purpose and there is no need to print a warning. */
char **c = deny_list;
for ( ; *c ; ++c) {
if (strcmp(*c, variable_name) == 0) {
break;
}
}
if (!*c) {
serverLog(LL_WARNING, "A key '%s' was added to Lua globals which is not on the globals allow list nor listed on the deny list.", variable_name);
}
} else {
lua_rawset(lua, -3);
}
return 0;
} }
/* Set global protection on a given table. /* Set a metatable with '__newindex' function that verify that
* The table need to be located on the top of the lua stack. * the new index appears on our globals while list.
* After called, it will no longer be possible to set
* new items on the table. The function is not removing
* the table from the top of the stack!
* *
* The function assumes the Lua stack have a least enough * The metatable is set on the table which located on the top
* space to push 2 element, its up to the caller to verify * of the stack.
* this before calling this function. */ */
void luaSetGlobalProtection(lua_State *lua) { void luaSetAllowListProtection(lua_State *lua) {
lua_pushstring(lua, REGISTRY_SET_GLOBALS_PROTECTION_NAME); lua_newtable(lua); /* push metatable */
lua_gettable(lua, LUA_REGISTRYINDEX); lua_pushcfunction(lua, luaNewIndexAllowList); /* push get error handler */
lua_pushvalue(lua, -2); lua_setfield(lua, -2, "__newindex");
int res = lua_pcall(lua, 1, 0, 0); lua_setmetatable(lua, -2);
serverAssert(res == 0); }
/* Set the readonly flag on the table located on the top of the stack
* and recursively call this function on each table located on the original
* table. Also, recursively call this function on the metatables.*/
void luaSetTableProtectionRecursively(lua_State *lua) {
/* This protect us from a loop in case we already visited the table
* For example, globals has '_G' key which is pointing back to globals. */
if (lua_isreadonlytable(lua, -1)) {
return;
}
/* protect the current table */
lua_enablereadonlytable(lua, -1, 1);
lua_checkstack(lua, 2);
lua_pushnil(lua); /* Use nil to start iteration. */
while (lua_next(lua,-2)) {
/* Stack now: table, key, value */
if (lua_istable(lua, -1)) {
luaSetTableProtectionRecursively(lua);
}
lua_pop(lua, 1);
}
/* protect the metatable if exists */
if (lua_getmetatable(lua, -1)) {
luaSetTableProtectionRecursively(lua);
lua_pop(lua, 1); /* pop the metatable */
}
} }
void luaRegisterVersion(lua_State* lua) { void luaRegisterVersion(lua_State* lua) {
...@@ -1272,8 +1367,11 @@ void luaRegisterLogFunction(lua_State* lua) { ...@@ -1272,8 +1367,11 @@ void luaRegisterLogFunction(lua_State* lua) {
} }
void luaRegisterRedisAPI(lua_State* lua) { void luaRegisterRedisAPI(lua_State* lua) {
lua_pushvalue(lua, LUA_GLOBALSINDEX);
luaSetAllowListProtection(lua);
lua_pop(lua, 1);
luaLoadLibraries(lua); luaLoadLibraries(lua);
luaRemoveUnsupportedFunctions(lua);
lua_pushcfunction(lua,luaRedisPcall); lua_pushcfunction(lua,luaRedisPcall);
lua_setglobal(lua, "pcall"); lua_setglobal(lua, "pcall");
...@@ -1504,9 +1602,19 @@ void luaCallFunction(scriptRunCtx* run_ctx, lua_State *lua, robj** keys, size_t ...@@ -1504,9 +1602,19 @@ void luaCallFunction(scriptRunCtx* run_ctx, lua_State *lua, robj** keys, size_t
* EVAL received. */ * EVAL received. */
luaCreateArray(lua,keys,nkeys); luaCreateArray(lua,keys,nkeys);
/* On eval, keys and arguments are globals. */ /* On eval, keys and arguments are globals. */
if (run_ctx->flags & SCRIPT_EVAL_MODE) lua_setglobal(lua,"KEYS"); if (run_ctx->flags & SCRIPT_EVAL_MODE){
/* open global protection to set KEYS */
lua_enablereadonlytable(lua, LUA_GLOBALSINDEX, 0);
lua_setglobal(lua,"KEYS");
lua_enablereadonlytable(lua, LUA_GLOBALSINDEX, 1);
}
luaCreateArray(lua,args,nargs); luaCreateArray(lua,args,nargs);
if (run_ctx->flags & SCRIPT_EVAL_MODE) lua_setglobal(lua,"ARGV"); if (run_ctx->flags & SCRIPT_EVAL_MODE){
/* open global protection to set ARGV */
lua_enablereadonlytable(lua, LUA_GLOBALSINDEX, 0);
lua_setglobal(lua,"ARGV");
lua_enablereadonlytable(lua, LUA_GLOBALSINDEX, 1);
}
/* At this point whether this script was never seen before or if it was /* At this point whether this script was never seen before or if it was
* already defined, we can call it. * already defined, we can call it.
......
...@@ -67,9 +67,10 @@ typedef struct errorInfo { ...@@ -67,9 +67,10 @@ typedef struct errorInfo {
void luaRegisterRedisAPI(lua_State* lua); void luaRegisterRedisAPI(lua_State* lua);
sds luaGetStringSds(lua_State *lua, int index); sds luaGetStringSds(lua_State *lua, int index);
void luaEnableGlobalsProtection(lua_State *lua, int is_eval);
void luaRegisterGlobalProtectionFunction(lua_State *lua); void luaRegisterGlobalProtectionFunction(lua_State *lua);
void luaSetGlobalProtection(lua_State *lua); void luaSetErrorMetatable(lua_State *lua);
void luaSetAllowListProtection(lua_State *lua);
void luaSetTableProtectionRecursively(lua_State *lua);
void luaRegisterLogFunction(lua_State* lua); void luaRegisterLogFunction(lua_State* lua);
void luaRegisterVersion(lua_State* lua); void luaRegisterVersion(lua_State* lua);
void luaPushErrorBuff(lua_State *lua, sds err_buff); void luaPushErrorBuff(lua_State *lua, sds err_buff);
......
...@@ -705,7 +705,7 @@ void sentinelEvent(int level, char *type, sentinelRedisInstance *ri, ...@@ -705,7 +705,7 @@ void sentinelEvent(int level, char *type, sentinelRedisInstance *ri,
if (level != LL_DEBUG) { if (level != LL_DEBUG) {
channel = createStringObject(type,strlen(type)); channel = createStringObject(type,strlen(type));
payload = createStringObject(msg,strlen(msg)); payload = createStringObject(msg,strlen(msg));
pubsubPublishMessage(channel,payload); pubsubPublishMessage(channel,payload,0);
decrRefCount(channel); decrRefCount(channel);
decrRefCount(payload); decrRefCount(payload);
} }
......
This diff is collapsed.
...@@ -603,6 +603,7 @@ typedef enum { ...@@ -603,6 +603,7 @@ typedef enum {
#define NOTIFY_KEY_MISS (1<<11) /* m (Note: This one is excluded from NOTIFY_ALL on purpose) */ #define NOTIFY_KEY_MISS (1<<11) /* m (Note: This one is excluded from NOTIFY_ALL on purpose) */
#define NOTIFY_LOADED (1<<12) /* module only key space notification, indicate a key loaded from rdb */ #define NOTIFY_LOADED (1<<12) /* module only key space notification, indicate a key loaded from rdb */
#define NOTIFY_MODULE (1<<13) /* d, module key space notification */ #define NOTIFY_MODULE (1<<13) /* d, module key space notification */
#define NOTIFY_NEW (1<<14) /* n, new key notification */
#define NOTIFY_ALL (NOTIFY_GENERIC | NOTIFY_STRING | NOTIFY_LIST | NOTIFY_SET | NOTIFY_HASH | NOTIFY_ZSET | NOTIFY_EXPIRED | NOTIFY_EVICTED | NOTIFY_STREAM | NOTIFY_MODULE) /* A flag */ #define NOTIFY_ALL (NOTIFY_GENERIC | NOTIFY_STRING | NOTIFY_LIST | NOTIFY_SET | NOTIFY_HASH | NOTIFY_ZSET | NOTIFY_EXPIRED | NOTIFY_EVICTED | NOTIFY_STREAM | NOTIFY_MODULE) /* A flag */
/* Using the following macro you can run code inside serverCron() with the /* Using the following macro you can run code inside serverCron() with the
...@@ -1061,7 +1062,7 @@ typedef struct replBacklog { ...@@ -1061,7 +1062,7 @@ typedef struct replBacklog {
listNode *ref_repl_buf_node; /* Referenced node of replication buffer blocks, listNode *ref_repl_buf_node; /* Referenced node of replication buffer blocks,
* see the definition of replBufBlock. */ * see the definition of replBufBlock. */
size_t unindexed_count; /* The count from last creating index block. */ size_t unindexed_count; /* The count from last creating index block. */
rax *blocks_index; /* The index of reocrded blocks of replication rax *blocks_index; /* The index of recorded blocks of replication
* buffer for quickly searching replication * buffer for quickly searching replication
* offset on partial resynchronization. */ * offset on partial resynchronization. */
long long histlen; /* Backlog actual data length */ long long histlen; /* Backlog actual data length */
...@@ -1106,6 +1107,7 @@ typedef struct client { ...@@ -1106,6 +1107,7 @@ typedef struct client {
buffer or object being sent. */ buffer or object being sent. */
time_t ctime; /* Client creation time. */ time_t ctime; /* Client creation time. */
long duration; /* Current command duration. Used for measuring latency of blocking/non-blocking cmds */ long duration; /* Current command duration. Used for measuring latency of blocking/non-blocking cmds */
int slot; /* The slot the client is executing against. Set to -1 if no slot is being used */
time_t lastinteraction; /* Time of the last interaction, used for timeout */ time_t lastinteraction; /* Time of the last interaction, used for timeout */
time_t obuf_soft_limit_reached_time; time_t obuf_soft_limit_reached_time;
uint64_t flags; /* Client flags: CLIENT_* macros. */ uint64_t flags; /* Client flags: CLIENT_* macros. */
...@@ -1323,6 +1325,15 @@ struct redisMemOverhead { ...@@ -1323,6 +1325,15 @@ struct redisMemOverhead {
} *db; } *db;
}; };
/* Replication error behavior determines the replica behavior
* when it receives an error over the replication stream. In
* either case the error is logged. */
typedef enum {
PROPAGATION_ERR_BEHAVIOR_IGNORE = 0,
PROPAGATION_ERR_BEHAVIOR_PANIC,
PROPAGATION_ERR_BEHAVIOR_PANIC_ON_REPLICAS
} replicationErrorBehavior;
/* This structure can be optionally passed to RDB save/load functions in /* This structure can be optionally passed to RDB save/load functions in
* order to implement additional functionalities, by storing and loading * order to implement additional functionalities, by storing and loading
* metadata to the RDB file. * metadata to the RDB file.
...@@ -1451,6 +1462,7 @@ struct redisServer { ...@@ -1451,6 +1462,7 @@ struct redisServer {
redisAtomic unsigned int lruclock; /* Clock for LRU eviction */ redisAtomic unsigned int lruclock; /* Clock for LRU eviction */
volatile sig_atomic_t shutdown_asap; /* Shutdown ordered by signal handler. */ volatile sig_atomic_t shutdown_asap; /* Shutdown ordered by signal handler. */
mstime_t shutdown_mstime; /* Timestamp to limit graceful shutdown. */ mstime_t shutdown_mstime; /* Timestamp to limit graceful shutdown. */
int last_sig_received; /* Indicates the last SIGNAL received, if any (e.g., SIGINT or SIGTERM). */
int shutdown_flags; /* Flags passed to prepareForShutdown(). */ int shutdown_flags; /* Flags passed to prepareForShutdown(). */
int activerehashing; /* Incremental rehash in serverCron() */ int activerehashing; /* Incremental rehash in serverCron() */
int active_defrag_running; /* Active defragmentation running (holds current scan aggressiveness) */ int active_defrag_running; /* Active defragmentation running (holds current scan aggressiveness) */
...@@ -1493,6 +1505,7 @@ struct redisServer { ...@@ -1493,6 +1505,7 @@ struct redisServer {
socketFds ipfd; /* TCP socket file descriptors */ socketFds ipfd; /* TCP socket file descriptors */
socketFds tlsfd; /* TLS socket file descriptors */ socketFds tlsfd; /* TLS socket file descriptors */
int sofd; /* Unix socket file descriptor */ int sofd; /* Unix socket file descriptor */
uint32_t socket_mark_id; /* ID for listen socket marking */
socketFds cfd; /* Cluster bus listening socket */ socketFds cfd; /* Cluster bus listening socket */
list *clients; /* List of active clients */ list *clients; /* List of active clients */
list *clients_to_close; /* Clients to close asynchronously */ list *clients_to_close; /* Clients to close asynchronously */
...@@ -1555,6 +1568,7 @@ struct redisServer { ...@@ -1555,6 +1568,7 @@ struct redisServer {
monotime stat_last_active_defrag_time; /* Timestamp of current active defrag start */ monotime stat_last_active_defrag_time; /* Timestamp of current active defrag start */
size_t stat_peak_memory; /* Max used memory record */ size_t stat_peak_memory; /* Max used memory record */
long long stat_aof_rewrites; /* number of aof file rewrites performed */ long long stat_aof_rewrites; /* number of aof file rewrites performed */
long long stat_aofrw_consecutive_failures; /* The number of consecutive failures of aofrw */
long long stat_rdb_saves; /* number of rdb saves performed */ long long stat_rdb_saves; /* number of rdb saves performed */
long long stat_fork_time; /* Time needed to perform latest fork() */ long long stat_fork_time; /* Time needed to perform latest fork() */
double stat_fork_rate; /* Fork rate in GB/sec. */ double stat_fork_rate; /* Fork rate in GB/sec. */
...@@ -1717,6 +1731,8 @@ struct redisServer { ...@@ -1717,6 +1731,8 @@ struct redisServer {
* abort(). useful for Valgrind. */ * abort(). useful for Valgrind. */
/* Shutdown */ /* Shutdown */
int shutdown_timeout; /* Graceful shutdown time limit in seconds. */ int shutdown_timeout; /* Graceful shutdown time limit in seconds. */
int shutdown_on_sigint; /* Shutdown flags configured for SIGINT. */
int shutdown_on_sigterm; /* Shutdown flags configured for SIGTERM. */
/* Replication (master) */ /* Replication (master) */
char replid[CONFIG_RUN_ID_SIZE+1]; /* My current replication ID. */ char replid[CONFIG_RUN_ID_SIZE+1]; /* My current replication ID. */
...@@ -1769,6 +1785,10 @@ struct redisServer { ...@@ -1769,6 +1785,10 @@ struct redisServer {
int replica_announced; /* If true, replica is announced by Sentinel */ int replica_announced; /* If true, replica is announced by Sentinel */
int slave_announce_port; /* Give the master this listening port. */ int slave_announce_port; /* Give the master this listening port. */
char *slave_announce_ip; /* Give the master this ip address. */ char *slave_announce_ip; /* Give the master this ip address. */
int propagation_error_behavior; /* Configures the behavior of the replica
* when it receives an error on the replication stream */
int repl_ignore_disk_write_error; /* Configures whether replicas panic when unable to
* persist writes to AOF. */
/* The following two fields is where we store master PSYNC replid/offset /* The following two fields is where we store master PSYNC replid/offset
* while the PSYNC is in progress. At the end we'll copy the fields into * while the PSYNC is in progress. At the end we'll copy the fields into
* the server->master client structure. */ * the server->master client structure. */
...@@ -2042,6 +2062,7 @@ typedef struct redisCommandArg { ...@@ -2042,6 +2062,7 @@ typedef struct redisCommandArg {
const char *summary; const char *summary;
const char *since; const char *since;
int flags; int flags;
const char *deprecated_since;
struct redisCommandArg *subargs; struct redisCommandArg *subargs;
/* runtime populated data */ /* runtime populated data */
int num_args; int num_args;
...@@ -2349,6 +2370,7 @@ int moduleGetCommandChannelsViaAPI(struct redisCommand *cmd, robj **argv, int ar ...@@ -2349,6 +2370,7 @@ int moduleGetCommandChannelsViaAPI(struct redisCommand *cmd, robj **argv, int ar
moduleType *moduleTypeLookupModuleByID(uint64_t id); moduleType *moduleTypeLookupModuleByID(uint64_t id);
void moduleTypeNameByID(char *name, uint64_t moduleid); void moduleTypeNameByID(char *name, uint64_t moduleid);
const char *moduleTypeModuleName(moduleType *mt); const char *moduleTypeModuleName(moduleType *mt);
const char *moduleNameFromCommand(struct redisCommand *cmd);
void moduleFreeContext(struct RedisModuleCtx *ctx); void moduleFreeContext(struct RedisModuleCtx *ctx);
void unblockClientFromModule(client *c); void unblockClientFromModule(client *c);
void moduleHandleBlockedClients(void); void moduleHandleBlockedClients(void);
...@@ -2509,7 +2531,7 @@ void unprotectClient(client *c); ...@@ -2509,7 +2531,7 @@ void unprotectClient(client *c);
void initThreadedIO(void); void initThreadedIO(void);
client *lookupClientByID(uint64_t id); client *lookupClientByID(uint64_t id);
int authRequired(client *c); int authRequired(client *c);
void clientInstallWriteHandler(client *c); void putClientInPendingWriteQueue(client *c);
#ifdef __GNUC__ #ifdef __GNUC__
void addReplyErrorFormatEx(client *c, int flags, const char *fmt, ...) void addReplyErrorFormatEx(client *c, int flags, const char *fmt, ...)
...@@ -2860,6 +2882,8 @@ struct redisCommand *lookupCommandBySds(sds s); ...@@ -2860,6 +2882,8 @@ struct redisCommand *lookupCommandBySds(sds s);
struct redisCommand *lookupCommandByCStringLogic(dict *commands, const char *s); struct redisCommand *lookupCommandByCStringLogic(dict *commands, const char *s);
struct redisCommand *lookupCommandByCString(const char *s); struct redisCommand *lookupCommandByCString(const char *s);
struct redisCommand *lookupCommandOrOriginal(robj **argv, int argc); struct redisCommand *lookupCommandOrOriginal(robj **argv, int argc);
int commandCheckExistence(client *c, sds *err);
int commandCheckArity(client *c, sds *err);
void startCommandExecution(); void startCommandExecution();
int incrCommandStatsOnError(struct redisCommand *cmd, int flags); int incrCommandStatsOnError(struct redisCommand *cmd, int flags);
void call(client *c, int flags); void call(client *c, int flags);
...@@ -2877,7 +2901,7 @@ int prepareForShutdown(int flags); ...@@ -2877,7 +2901,7 @@ int prepareForShutdown(int flags);
void replyToClientsBlockedOnShutdown(void); void replyToClientsBlockedOnShutdown(void);
int abortShutdown(void); int abortShutdown(void);
void afterCommand(client *c); void afterCommand(client *c);
int inNestedCall(void); int mustObeyClient(client *c);
#ifdef __GNUC__ #ifdef __GNUC__
void _serverLog(int level, const char *fmt, ...) void _serverLog(int level, const char *fmt, ...)
__attribute__((format(printf, 2, 3))); __attribute__((format(printf, 2, 3)));
...@@ -2962,8 +2986,8 @@ int pubsubUnsubscribeAllChannels(client *c, int notify); ...@@ -2962,8 +2986,8 @@ int pubsubUnsubscribeAllChannels(client *c, int notify);
int pubsubUnsubscribeShardAllChannels(client *c, int notify); int pubsubUnsubscribeShardAllChannels(client *c, int notify);
void pubsubUnsubscribeShardChannels(robj **channels, unsigned int count); void pubsubUnsubscribeShardChannels(robj **channels, unsigned int count);
int pubsubUnsubscribeAllPatterns(client *c, int notify); int pubsubUnsubscribeAllPatterns(client *c, int notify);
int pubsubPublishMessage(robj *channel, robj *message); int pubsubPublishMessage(robj *channel, robj *message, int sharded);
int pubsubPublishMessageShard(robj *channel, robj *message); int pubsubPublishMessageAndPropagateToCluster(robj *channel, robj *message, int sharded);
void addReplyPubsubMessage(client *c, robj *channel, robj *msg); void addReplyPubsubMessage(client *c, robj *channel, robj *msg);
int serverPubsubSubscriptionCount(); int serverPubsubSubscriptionCount();
int serverPubsubShardSubscriptionCount(); int serverPubsubShardSubscriptionCount();
......
...@@ -146,39 +146,24 @@ robj *hashTypeGetValueObject(robj *o, sds field) { ...@@ -146,39 +146,24 @@ robj *hashTypeGetValueObject(robj *o, sds field) {
* exist. */ * exist. */
size_t hashTypeGetValueLength(robj *o, sds field) { size_t hashTypeGetValueLength(robj *o, sds field) {
size_t len = 0; size_t len = 0;
if (o->encoding == OBJ_ENCODING_LISTPACK) {
unsigned char *vstr = NULL; unsigned char *vstr = NULL;
unsigned int vlen = UINT_MAX; unsigned int vlen = UINT_MAX;
long long vll = LLONG_MAX; long long vll = LLONG_MAX;
if (hashTypeGetFromListpack(o, field, &vstr, &vlen, &vll) == 0) if (hashTypeGetValue(o, field, &vstr, &vlen, &vll) == C_OK)
len = vstr ? vlen : sdigits10(vll); len = vstr ? vlen : sdigits10(vll);
} else if (o->encoding == OBJ_ENCODING_HT) {
sds aux;
if ((aux = hashTypeGetFromHashTable(o, field)) != NULL)
len = sdslen(aux);
} else {
serverPanic("Unknown hash encoding");
}
return len; return len;
} }
/* Test if the specified field exists in the given hash. Returns 1 if the field /* Test if the specified field exists in the given hash. Returns 1 if the field
* exists, and 0 when it doesn't. */ * exists, and 0 when it doesn't. */
int hashTypeExists(robj *o, sds field) { int hashTypeExists(robj *o, sds field) {
if (o->encoding == OBJ_ENCODING_LISTPACK) {
unsigned char *vstr = NULL; unsigned char *vstr = NULL;
unsigned int vlen = UINT_MAX; unsigned int vlen = UINT_MAX;
long long vll = LLONG_MAX; long long vll = LLONG_MAX;
if (hashTypeGetFromListpack(o, field, &vstr, &vlen, &vll) == 0) return 1; return hashTypeGetValue(o, field, &vstr, &vlen, &vll) == C_OK;
} else if (o->encoding == OBJ_ENCODING_HT) {
if (hashTypeGetFromHashTable(o, field) != NULL) return 1;
} else {
serverPanic("Unknown hash encoding");
}
return 0;
} }
/* Add a new field, overwrite the old with the new value if it already exists. /* Add a new field, overwrite the old with the new value if it already exists.
...@@ -205,6 +190,14 @@ int hashTypeExists(robj *o, sds field) { ...@@ -205,6 +190,14 @@ int hashTypeExists(robj *o, sds field) {
int hashTypeSet(robj *o, sds field, sds value, int flags) { int hashTypeSet(robj *o, sds field, sds value, int flags) {
int update = 0; int update = 0;
/* Check if the field is too long for listpack, and convert before adding the item.
* This is needed for HINCRBY* case since in other commands this is handled early by
* hashTypeTryConversion, so this check will be a NOP. */
if (o->encoding == OBJ_ENCODING_LISTPACK) {
if (sdslen(field) > server.hash_max_listpack_value || sdslen(value) > server.hash_max_listpack_value)
hashTypeConvert(o, OBJ_ENCODING_HT);
}
if (o->encoding == OBJ_ENCODING_LISTPACK) { if (o->encoding == OBJ_ENCODING_LISTPACK) {
unsigned char *zl, *fptr, *vptr; unsigned char *zl, *fptr, *vptr;
...@@ -717,37 +710,23 @@ void hincrbyfloatCommand(client *c) { ...@@ -717,37 +710,23 @@ void hincrbyfloatCommand(client *c) {
} }
static void addHashFieldToReply(client *c, robj *o, sds field) { static void addHashFieldToReply(client *c, robj *o, sds field) {
int ret;
if (o == NULL) { if (o == NULL) {
addReplyNull(c); addReplyNull(c);
return; return;
} }
if (o->encoding == OBJ_ENCODING_LISTPACK) {
unsigned char *vstr = NULL; unsigned char *vstr = NULL;
unsigned int vlen = UINT_MAX; unsigned int vlen = UINT_MAX;
long long vll = LLONG_MAX; long long vll = LLONG_MAX;
ret = hashTypeGetFromListpack(o, field, &vstr, &vlen, &vll); if (hashTypeGetValue(o, field, &vstr, &vlen, &vll) == C_OK) {
if (ret < 0) {
addReplyNull(c);
} else {
if (vstr) { if (vstr) {
addReplyBulkCBuffer(c, vstr, vlen); addReplyBulkCBuffer(c, vstr, vlen);
} else { } else {
addReplyBulkLongLong(c, vll); addReplyBulkLongLong(c, vll);
} }
}
} else if (o->encoding == OBJ_ENCODING_HT) {
sds value = hashTypeGetFromHashTable(o, field);
if (value == NULL)
addReplyNull(c);
else
addReplyBulkCBuffer(c, value, sdslen(value));
} else { } else {
serverPanic("Unknown hash encoding"); addReplyNull(c);
} }
} }
...@@ -907,7 +886,7 @@ void hscanCommand(client *c) { ...@@ -907,7 +886,7 @@ void hscanCommand(client *c) {
scanGenericCommand(c,o,cursor); scanGenericCommand(c,o,cursor);
} }
static void harndfieldReplyWithListpack(client *c, unsigned int count, listpackEntry *keys, listpackEntry *vals) { static void hrandfieldReplyWithListpack(client *c, unsigned int count, listpackEntry *keys, listpackEntry *vals) {
for (unsigned long i = 0; i < count; i++) { for (unsigned long i = 0; i < count; i++) {
if (vals && c->resp > 2) if (vals && c->resp > 2)
addReplyArrayLen(c,2); addReplyArrayLen(c,2);
...@@ -990,7 +969,7 @@ void hrandfieldWithCountCommand(client *c, long l, int withvalues) { ...@@ -990,7 +969,7 @@ void hrandfieldWithCountCommand(client *c, long l, int withvalues) {
sample_count = count > limit ? limit : count; sample_count = count > limit ? limit : count;
count -= sample_count; count -= sample_count;
lpRandomPairs(hash->ptr, sample_count, keys, vals); lpRandomPairs(hash->ptr, sample_count, keys, vals);
harndfieldReplyWithListpack(c, sample_count, keys, vals); hrandfieldReplyWithListpack(c, sample_count, keys, vals);
} }
zfree(keys); zfree(keys);
zfree(vals); zfree(vals);
...@@ -1092,7 +1071,7 @@ void hrandfieldWithCountCommand(client *c, long l, int withvalues) { ...@@ -1092,7 +1071,7 @@ void hrandfieldWithCountCommand(client *c, long l, int withvalues) {
if (withvalues) if (withvalues)
vals = zmalloc(sizeof(listpackEntry)*count); vals = zmalloc(sizeof(listpackEntry)*count);
serverAssert(lpRandomPairsUnique(hash->ptr, count, keys, vals) == count); serverAssert(lpRandomPairsUnique(hash->ptr, count, keys, vals) == count);
harndfieldReplyWithListpack(c, count, keys, vals); hrandfieldReplyWithListpack(c, count, keys, vals);
zfree(keys); zfree(keys);
zfree(vals); zfree(vals);
return; return;
......
...@@ -1000,7 +1000,7 @@ static int streamParseAddOrTrimArgsOrReply(client *c, streamAddTrimArgs *args, i ...@@ -1000,7 +1000,7 @@ static int streamParseAddOrTrimArgsOrReply(client *c, streamAddTrimArgs *args, i
return -1; return -1;
} }
if (c == server.master || c->id == CLIENT_ID_AOF) { if (mustObeyClient(c)) {
/* If command came from master or from AOF we must not enforce maxnodes /* If command came from master or from AOF we must not enforce maxnodes
* (The maxlen/minid argument was re-written to make sure there's no * (The maxlen/minid argument was re-written to make sure there's no
* inconsistency). */ * inconsistency). */
...@@ -1370,24 +1370,35 @@ void streamLastValidID(stream *s, streamID *maxid) ...@@ -1370,24 +1370,35 @@ void streamLastValidID(stream *s, streamID *maxid)
streamIteratorStop(&si); streamIteratorStop(&si);
} }
/* Maximum size for a stream ID string. In theory 20*2+1 should be enough,
* But to avoid chance for off by one issues and null-term, in case this will
* be used as parsing buffer, we use a slightly larger buffer. On the other
* hand considering sds header is gonna add 4 bytes, we wanna keep below the
* allocator's 48 bytes bin. */
#define STREAM_ID_STR_LEN 44
sds createStreamIDString(streamID *id) {
/* Optimization: pre-allocate a big enough buffer to avoid reallocs. */
sds str = sdsnewlen(SDS_NOINIT, STREAM_ID_STR_LEN);
sdssetlen(str, 0);
return sdscatfmt(str,"%U-%U", id->ms,id->seq);
}
/* Emit a reply in the client output buffer by formatting a Stream ID /* Emit a reply in the client output buffer by formatting a Stream ID
* in the standard <ms>-<seq> format, using the simple string protocol * in the standard <ms>-<seq> format, using the simple string protocol
* of REPL. */ * of REPL. */
void addReplyStreamID(client *c, streamID *id) { void addReplyStreamID(client *c, streamID *id) {
sds replyid = sdscatfmt(sdsempty(),"%U-%U",id->ms,id->seq); addReplyBulkSds(c,createStreamIDString(id));
addReplyBulkSds(c,replyid);
} }
void setDeferredReplyStreamID(client *c, void *dr, streamID *id) { void setDeferredReplyStreamID(client *c, void *dr, streamID *id) {
sds replyid = sdscatfmt(sdsempty(),"%U-%U",id->ms,id->seq); setDeferredReplyBulkSds(c, dr, createStreamIDString(id));
setDeferredReplyBulkSds(c, dr, replyid);
} }
/* Similar to the above function, but just creates an object, usually useful /* Similar to the above function, but just creates an object, usually useful
* for replication purposes to create arguments. */ * for replication purposes to create arguments. */
robj *createObjectFromStreamID(streamID *id) { robj *createObjectFromStreamID(streamID *id) {
return createObject(OBJ_STRING, sdscatfmt(sdsempty(),"%U-%U", return createObject(OBJ_STRING, createStreamIDString(id));
id->ms,id->seq));
} }
/* Returns non-zero if the ID is 0-0. */ /* Returns non-zero if the ID is 0-0. */
...@@ -2025,7 +2036,8 @@ void xaddCommand(client *c) { ...@@ -2025,7 +2036,8 @@ void xaddCommand(client *c) {
addReplyError(c,"Elements are too large to be stored"); addReplyError(c,"Elements are too large to be stored");
return; return;
} }
addReplyStreamID(c,&id); sds replyid = createStreamIDString(&id);
addReplyBulkCBuffer(c, replyid, sdslen(replyid));
signalModifiedKey(c,c->db,c->argv[1]); signalModifiedKey(c,c->db,c->argv[1]);
notifyKeyspaceEvent(NOTIFY_STREAM,"xadd",c->argv[1],c->db->id); notifyKeyspaceEvent(NOTIFY_STREAM,"xadd",c->argv[1],c->db->id);
...@@ -2050,9 +2062,11 @@ void xaddCommand(client *c) { ...@@ -2050,9 +2062,11 @@ void xaddCommand(client *c) {
/* Let's rewrite the ID argument with the one actually generated for /* Let's rewrite the ID argument with the one actually generated for
* AOF/replication propagation. */ * AOF/replication propagation. */
if (!parsed_args.id_given || !parsed_args.seq_given) { if (!parsed_args.id_given || !parsed_args.seq_given) {
robj *idarg = createObjectFromStreamID(&id); robj *idarg = createObject(OBJ_STRING, replyid);
rewriteClientCommandArgument(c, idpos, idarg); rewriteClientCommandArgument(c, idpos, idarg);
decrRefCount(idarg); decrRefCount(idarg);
} else {
sdsfree(replyid);
} }
/* We need to signal to blocked clients that there is new data on this /* We need to signal to blocked clients that there is new data on this
......
...@@ -38,7 +38,7 @@ int getGenericCommand(client *c); ...@@ -38,7 +38,7 @@ int getGenericCommand(client *c);
*----------------------------------------------------------------------------*/ *----------------------------------------------------------------------------*/
static int checkStringLength(client *c, long long size) { static int checkStringLength(client *c, long long size) {
if (!(c->flags & CLIENT_MASTER) && size > server.proto_max_bulk_len) { if (!mustObeyClient(c) && size > server.proto_max_bulk_len) {
addReplyError(c,"string exceeds maximum allowed size (proto-max-bulk-len)"); addReplyError(c,"string exceeds maximum allowed size (proto-max-bulk-len)");
return C_ERR; return C_ERR;
} }
...@@ -792,7 +792,7 @@ void lcsCommand(client *c) { ...@@ -792,7 +792,7 @@ void lcsCommand(client *c) {
/* Setup an uint32_t array to store at LCS[i,j] the length of the /* Setup an uint32_t array to store at LCS[i,j] the length of the
* LCS A0..i-1, B0..j-1. Note that we have a linear array here, so * LCS A0..i-1, B0..j-1. Note that we have a linear array here, so
* we index it as LCS[j+(blen+1)*j] */ * we index it as LCS[j+(blen+1)*i] */
#define LCS(A,B) lcs[(B)+((A)*(blen+1))] #define LCS(A,B) lcs[(B)+((A)*(blen+1))]
/* Try to allocate the LCS table, and abort on overflow or insufficient memory. */ /* Try to allocate the LCS table, and abort on overflow or insufficient memory. */
......
...@@ -1029,16 +1029,24 @@ unsigned char *zzlInsertAt(unsigned char *zl, unsigned char *eptr, sds ele, doub ...@@ -1029,16 +1029,24 @@ unsigned char *zzlInsertAt(unsigned char *zl, unsigned char *eptr, sds ele, doub
unsigned char *sptr; unsigned char *sptr;
char scorebuf[MAX_D2STRING_CHARS]; char scorebuf[MAX_D2STRING_CHARS];
int scorelen; int scorelen;
long long lscore;
int score_is_long = double2ll(score, &lscore);
if (!score_is_long)
scorelen = d2string(scorebuf,sizeof(scorebuf),score); scorelen = d2string(scorebuf,sizeof(scorebuf),score);
if (eptr == NULL) { if (eptr == NULL) {
zl = lpAppend(zl,(unsigned char*)ele,sdslen(ele)); zl = lpAppend(zl,(unsigned char*)ele,sdslen(ele));
if (score_is_long)
zl = lpAppendInteger(zl,lscore);
else
zl = lpAppend(zl,(unsigned char*)scorebuf,scorelen); zl = lpAppend(zl,(unsigned char*)scorebuf,scorelen);
} else { } else {
/* Insert member before the element 'eptr'. */ /* Insert member before the element 'eptr'. */
zl = lpInsertString(zl,(unsigned char*)ele,sdslen(ele),eptr,LP_BEFORE,&sptr); zl = lpInsertString(zl,(unsigned char*)ele,sdslen(ele),eptr,LP_BEFORE,&sptr);
/* Insert score after the member. */ /* Insert score after the member. */
if (score_is_long)
zl = lpInsertInteger(zl,lscore,sptr,LP_AFTER,NULL);
else
zl = lpInsertString(zl,(unsigned char*)scorebuf,scorelen,sptr,LP_AFTER,NULL); zl = lpInsertString(zl,(unsigned char*)scorebuf,scorelen,sptr,LP_AFTER,NULL);
} }
return zl; return zl;
...@@ -3964,7 +3972,7 @@ void zpopminCommand(client *c) { ...@@ -3964,7 +3972,7 @@ void zpopminCommand(client *c) {
zpopMinMaxCommand(c, ZSET_MIN); zpopMinMaxCommand(c, ZSET_MIN);
} }
/* ZMAXPOP key [<count>] */ /* ZPOPMAX key [<count>] */
void zpopmaxCommand(client *c) { void zpopmaxCommand(client *c) {
zpopMinMaxCommand(c, ZSET_MAX); zpopMinMaxCommand(c, ZSET_MAX);
} }
...@@ -4351,12 +4359,12 @@ void zmpopGenericCommand(client *c, int numkeys_idx, int is_block) { ...@@ -4351,12 +4359,12 @@ void zmpopGenericCommand(client *c, int numkeys_idx, int is_block) {
} }
} }
/* ZMPOP numkeys [<key> ...] MIN|MAX [COUNT count] */ /* ZMPOP numkeys key [<key> ...] MIN|MAX [COUNT count] */
void zmpopCommand(client *c) { void zmpopCommand(client *c) {
zmpopGenericCommand(c, 1, 0); zmpopGenericCommand(c, 1, 0);
} }
/* BZMPOP timeout numkeys [<key> ...] MIN|MAX [COUNT count] */ /* BZMPOP timeout numkeys key [<key> ...] MIN|MAX [COUNT count] */
void bzmpopCommand(client *c) { void bzmpopCommand(client *c) {
zmpopGenericCommand(c, 2, 1); zmpopGenericCommand(c, 2, 1);
} }
...@@ -58,7 +58,7 @@ int clientsCronHandleTimeout(client *c, mstime_t now_ms) { ...@@ -58,7 +58,7 @@ int clientsCronHandleTimeout(client *c, mstime_t now_ms) {
if (server.maxidletime && if (server.maxidletime &&
/* This handles the idle clients connection timeout if set. */ /* This handles the idle clients connection timeout if set. */
!(c->flags & CLIENT_SLAVE) && /* No timeout for slaves and monitors */ !(c->flags & CLIENT_SLAVE) && /* No timeout for slaves and monitors */
!(c->flags & CLIENT_MASTER) && /* No timeout for masters */ !mustObeyClient(c) && /* No timeout for masters and AOF */
!(c->flags & CLIENT_BLOCKED) && /* No timeout for BLPOP */ !(c->flags & CLIENT_BLOCKED) && /* No timeout for BLPOP */
!(c->flags & CLIENT_PUBSUB) && /* No timeout for Pub/Sub clients */ !(c->flags & CLIENT_PUBSUB) && /* No timeout for Pub/Sub clients */
(now - c->lastinteraction > server.maxidletime)) (now - c->lastinteraction > server.maxidletime))
......
...@@ -552,6 +552,36 @@ int string2d(const char *s, size_t slen, double *dp) { ...@@ -552,6 +552,36 @@ int string2d(const char *s, size_t slen, double *dp) {
return 1; return 1;
} }
/* Returns 1 if the double value can safely be represented in long long without
* precision loss, in which case the corresponding long long is stored in the out variable. */
int double2ll(double d, long long *out) {
#if (DBL_MANT_DIG >= 52) && (DBL_MANT_DIG <= 63) && (LLONG_MAX == 0x7fffffffffffffffLL)
/* Check if the float is in a safe range to be casted into a
* long long. We are assuming that long long is 64 bit here.
* Also we are assuming that there are no implementations around where
* double has precision < 52 bit.
*
* Under this assumptions we test if a double is inside a range
* where casting to long long is safe. Then using two castings we
* make sure the decimal part is zero. If all this is true we can use
* integer without precision loss.
*
* Note that numbers above 2^52 and below 2^63 use all the fraction bits as real part,
* and the exponent bits are positive, which means the "decimal" part must be 0.
* i.e. all double values in that range are representable as a long without precision loss,
* but not all long values in that range can be represented as a double.
* we only care about the first part here. */
if (d < (double)(-LLONG_MAX/2) || d > (double)(LLONG_MAX/2))
return 0;
long long ll = d;
if (ll == d) {
*out = ll;
return 1;
}
#endif
return 0;
}
/* Convert a double to a string representation. Returns the number of bytes /* Convert a double to a string representation. Returns the number of bytes
* required. The representation should always be parsable by strtod(3). * required. The representation should always be parsable by strtod(3).
* This function does not support human-friendly formatting like ld2string * This function does not support human-friendly formatting like ld2string
...@@ -572,22 +602,11 @@ int d2string(char *buf, size_t len, double value) { ...@@ -572,22 +602,11 @@ int d2string(char *buf, size_t len, double value) {
else else
len = snprintf(buf,len,"0"); len = snprintf(buf,len,"0");
} else { } else {
#if (DBL_MANT_DIG >= 52) && (LLONG_MAX == 0x7fffffffffffffffLL) long long lvalue;
/* Check if the float is in a safe range to be casted into a /* Integer printing function is much faster, check if we can safely use it. */
* long long. We are assuming that long long is 64 bit here. if (double2ll(value, &lvalue))
* Also we are assuming that there are no implementations around where len = ll2string(buf,len,lvalue);
* double has precision < 52 bit.
*
* Under this assumptions we test if a double is inside an interval
* where casting to long long is safe. Then using two castings we
* make sure the decimal part is zero. If all this is true we use
* integer printing function that is much faster. */
double min = -4503599627370495; /* (2^52)-1 */
double max = 4503599627370496; /* -(2^52) */
if (value > min && value < max && value == ((double)((long long)value)))
len = ll2string(buf,len,(long long)value);
else else
#endif
len = snprintf(buf,len,"%.17g",value); len = snprintf(buf,len,"%.17g",value);
} }
......
...@@ -75,6 +75,7 @@ int string2d(const char *s, size_t slen, double *dp); ...@@ -75,6 +75,7 @@ int string2d(const char *s, size_t slen, double *dp);
int trimDoubleString(char *buf, size_t len); int trimDoubleString(char *buf, size_t len);
int d2string(char *buf, size_t len, double value); int d2string(char *buf, size_t len, double value);
int ld2string(char *buf, size_t len, long double value, ld2string_mode mode); int ld2string(char *buf, size_t len, long double value, ld2string_mode mode);
int double2ll(double d, long long *out);
int yesnotoi(char *s); int yesnotoi(char *s);
sds getAbsolutePath(char *filename); sds getAbsolutePath(char *filename);
long getTimeZone(void); long getTimeZone(void);
......
...@@ -30,3 +30,5 @@ activerehashing yes ...@@ -30,3 +30,5 @@ activerehashing yes
enable-protected-configs yes enable-protected-configs yes
enable-debug-command yes enable-debug-command yes
enable-module-command yes enable-module-command yes
propagation-error-behavior panic
\ No newline at end of file
This diff is collapsed.
...@@ -632,7 +632,6 @@ test {corrupt payload: fuzzer findings - stream PEL without consumer} { ...@@ -632,7 +632,6 @@ test {corrupt payload: fuzzer findings - stream PEL without consumer} {
r debug set-skip-checksum-validation 1 r debug set-skip-checksum-validation 1
catch {r restore _stream 0 "\x0F\x01\x10\x00\x00\x01\x7B\x08\xF0\xB2\x34\x00\x00\x00\x00\x00\x00\x00\x00\xC3\x3B\x40\x42\x19\x42\x00\x00\x00\x18\x00\x02\x01\x01\x01\x02\x01\x84\x69\x74\x65\x6D\x05\x85\x76\x61\x6C\x75\x65\x06\x00\x20\x10\x00\x00\x20\x01\x00\x01\x20\x03\x02\x05\x01\x03\x20\x05\x40\x00\x04\x82\x5F\x31\x03\x05\x60\x19\x80\x32\x02\x05\x01\xFF\x02\x81\x00\x00\x01\x7B\x08\xF0\xB2\x34\x02\x01\x07\x6D\x79\x67\x72\x6F\x75\x70\x81\x00\x00\x01\x7B\x08\xF0\xB2\x34\x01\x01\x00\x00\x01\x7B\x08\xF0\xB2\x34\x00\x00\x00\x00\x00\x00\x00\x01\x35\xB2\xF0\x08\x7B\x01\x00\x00\x01\x01\x13\x41\x6C\x69\x63\x65\x35\xB2\xF0\x08\x7B\x01\x00\x00\x01\x00\x00\x01\x7B\x08\xF0\xB2\x34\x00\x00\x00\x00\x00\x00\x00\x01\x09\x00\x28\x2F\xE0\xC5\x04\xBB\xA7\x31"} err catch {r restore _stream 0 "\x0F\x01\x10\x00\x00\x01\x7B\x08\xF0\xB2\x34\x00\x00\x00\x00\x00\x00\x00\x00\xC3\x3B\x40\x42\x19\x42\x00\x00\x00\x18\x00\x02\x01\x01\x01\x02\x01\x84\x69\x74\x65\x6D\x05\x85\x76\x61\x6C\x75\x65\x06\x00\x20\x10\x00\x00\x20\x01\x00\x01\x20\x03\x02\x05\x01\x03\x20\x05\x40\x00\x04\x82\x5F\x31\x03\x05\x60\x19\x80\x32\x02\x05\x01\xFF\x02\x81\x00\x00\x01\x7B\x08\xF0\xB2\x34\x02\x01\x07\x6D\x79\x67\x72\x6F\x75\x70\x81\x00\x00\x01\x7B\x08\xF0\xB2\x34\x01\x01\x00\x00\x01\x7B\x08\xF0\xB2\x34\x00\x00\x00\x00\x00\x00\x00\x01\x35\xB2\xF0\x08\x7B\x01\x00\x00\x01\x01\x13\x41\x6C\x69\x63\x65\x35\xB2\xF0\x08\x7B\x01\x00\x00\x01\x00\x00\x01\x7B\x08\xF0\xB2\x34\x00\x00\x00\x00\x00\x00\x00\x01\x09\x00\x28\x2F\xE0\xC5\x04\xBB\xA7\x31"} err
assert_match "*Bad data format*" $err assert_match "*Bad data format*" $err
#catch {r XINFO STREAM _stream FULL }
r ping r ping
} }
} }
...@@ -674,7 +673,6 @@ test {corrupt payload: fuzzer findings - stream with non-integer entry id} { ...@@ -674,7 +673,6 @@ test {corrupt payload: fuzzer findings - stream with non-integer entry id} {
r config set sanitize-dump-payload yes r config set sanitize-dump-payload yes
r debug set-skip-checksum-validation 1 r debug set-skip-checksum-validation 1
catch {r restore _streambig 0 "\x0F\x03\x10\x00\x00\x01\x7B\x13\x34\xC3\xB2\x00\x00\x00\x00\x00\x00\x00\x00\xC3\x40\x4F\x40\x5C\x18\x5C\x00\x00\x00\x24\x00\x05\x01\x00\x01\x02\x01\x84\x69\x74\x65\x6D\x05\x85\x76\x61\x6C\x75\x65\x06\x40\x10\x00\x80\x20\x01\x00\x01\x20\x03\x00\x05\x20\x1C\x40\x09\x05\x01\x01\x82\x5F\x31\x03\x80\x0D\x00\x02\x20\x0D\x00\x02\xA0\x19\x00\x03\x20\x0B\x02\x82\x5F\x33\xA0\x19\x00\x04\x20\x0D\x00\x04\x20\x19\x00\xFF\x10\x00\x00\x01\x7B\x13\x34\xC3\xB2\x00\x00\x00\x00\x00\x00\x00\x05\xC3\x40\x56\x40\x61\x18\x61\x00\x00\x00\x24\x00\x05\x01\x00\x01\x02\x01\x84\x69\x74\x65\x6D\x05\x85\x76\x61\x6C\x75\x65\x06\x40\x10\x00\x00\x20\x01\x06\x01\x01\x82\x5F\x35\x03\x05\x20\x1E\x40\x0B\x03\x01\x01\x06\x01\x40\x0B\x03\x01\x01\xDF\xFB\x20\x05\x02\x82\x5F\x37\x60\x1A\x20\x0E\x00\xFC\x20\x05\x00\x08\xC0\x1B\x00\xFD\x20\x0C\x02\x82\x5F\x39\x20\x1B\x00\xFF\x10\x00\x00\x01\x7B\x13\x34\xC3\xB3\x00\x00\x00\x00\x00\x00\x00\x03\xC3\x3D\x40\x4A\x18\x4A\x00\x00\x00\x15\x00\x02\x01\x00\x01\x02\x01\x84\x69\x74\x65\x6D\x05\x85\x76\x61\x6C\x75\x65\x06\x40\x10\x00\x00\x20\x01\x40\x00\x00\x05\x60\x07\x02\xDF\xFD\x02\xC0\x23\x09\x01\x01\x86\x75\x6E\x69\x71\x75\x65\x07\xA0\x2D\x02\x08\x01\xFF\x0C\x81\x00\x00\x01\x7B\x13\x34\xC3\xB4\x00\x00\x09\x00\x9D\xBD\xD5\xB9\x33\xC4\xC5\xFF"} err catch {r restore _streambig 0 "\x0F\x03\x10\x00\x00\x01\x7B\x13\x34\xC3\xB2\x00\x00\x00\x00\x00\x00\x00\x00\xC3\x40\x4F\x40\x5C\x18\x5C\x00\x00\x00\x24\x00\x05\x01\x00\x01\x02\x01\x84\x69\x74\x65\x6D\x05\x85\x76\x61\x6C\x75\x65\x06\x40\x10\x00\x80\x20\x01\x00\x01\x20\x03\x00\x05\x20\x1C\x40\x09\x05\x01\x01\x82\x5F\x31\x03\x80\x0D\x00\x02\x20\x0D\x00\x02\xA0\x19\x00\x03\x20\x0B\x02\x82\x5F\x33\xA0\x19\x00\x04\x20\x0D\x00\x04\x20\x19\x00\xFF\x10\x00\x00\x01\x7B\x13\x34\xC3\xB2\x00\x00\x00\x00\x00\x00\x00\x05\xC3\x40\x56\x40\x61\x18\x61\x00\x00\x00\x24\x00\x05\x01\x00\x01\x02\x01\x84\x69\x74\x65\x6D\x05\x85\x76\x61\x6C\x75\x65\x06\x40\x10\x00\x00\x20\x01\x06\x01\x01\x82\x5F\x35\x03\x05\x20\x1E\x40\x0B\x03\x01\x01\x06\x01\x40\x0B\x03\x01\x01\xDF\xFB\x20\x05\x02\x82\x5F\x37\x60\x1A\x20\x0E\x00\xFC\x20\x05\x00\x08\xC0\x1B\x00\xFD\x20\x0C\x02\x82\x5F\x39\x20\x1B\x00\xFF\x10\x00\x00\x01\x7B\x13\x34\xC3\xB3\x00\x00\x00\x00\x00\x00\x00\x03\xC3\x3D\x40\x4A\x18\x4A\x00\x00\x00\x15\x00\x02\x01\x00\x01\x02\x01\x84\x69\x74\x65\x6D\x05\x85\x76\x61\x6C\x75\x65\x06\x40\x10\x00\x00\x20\x01\x40\x00\x00\x05\x60\x07\x02\xDF\xFD\x02\xC0\x23\x09\x01\x01\x86\x75\x6E\x69\x71\x75\x65\x07\xA0\x2D\x02\x08\x01\xFF\x0C\x81\x00\x00\x01\x7B\x13\x34\xC3\xB4\x00\x00\x09\x00\x9D\xBD\xD5\xB9\x33\xC4\xC5\xFF"} err
#catch {r XINFO STREAM _streambig FULL }
assert_match "*Bad data format*" $err assert_match "*Bad data format*" $err
r ping r ping
} }
...@@ -782,5 +780,15 @@ test {corrupt payload: fuzzer findings - zset zslInsert with a NAN score} { ...@@ -782,5 +780,15 @@ test {corrupt payload: fuzzer findings - zset zslInsert with a NAN score} {
} }
} }
test {corrupt payload: fuzzer findings - streamLastValidID panic} {
start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
r config set sanitize-dump-payload yes
r debug set-skip-checksum-validation 1
catch {r restore _streambig 0 "\x13\xC0\x10\x00\x00\x01\x80\x20\x48\xA0\x33\x00\x00\x00\x00\x00\x00\x00\x00\xC3\x40\x4F\x40\x5C\x18\x5C\x00\x00\x00\x24\x00\x05\x01\x00\x01\x02\x01\x84\x69\x74\x65\x6D\x05\x85\x76\x61\x6C\x75\x65\x06\x40\x10\x00\x00\x20\x01\x00\x01\x20\x03\x00\x05\x20\x1C\x40\x09\x05\x01\x01\x82\x5F\x31\x03\x80\x0D\x00\x02\x20\x0D\x00\x02\xA0\x19\x00\x03\x20\x0B\x02\x82\x5F\x33\x60\x19\x40\x2F\x02\x01\x01\x04\x20\x19\x00\xFF\x10\x00\x00\x01\x80\x20\x48\xA0\x34\x00\x00\x00\x00\x00\x00\x00\x01\xC3\x40\x51\x40\x5E\x18\x5E\x00\x00\x00\x24\x00\x05\x01\x00\x01\x02\x01\x84\x69\x74\x65\x6D\x05\x85\x76\x61\x6C\x75\x65\x06\x40\x10\x00\x00\x20\x01\x06\x01\x01\x82\x5F\x35\x03\x05\x20\x1E\x40\x0B\x03\x01\x01\x06\x01\x80\x0B\x00\x02\x20\x0B\x02\x82\x5F\x37\xA0\x19\x00\x03\x20\x0D\x00\x08\xA0\x19\x00\x04\x20\x0B\x02\x82\x5F\x39\x20\x19\x00\xFF\x10\x00\x00\x01\x80\x20\x48\xA0\x34\x00\x00\x00\x00\x00\x00\x00\x06\xC3\x3D\x40\x4A\x18\x4A\x00\x00\x00\x15\x00\x02\x01\x00\x01\x02\x01\x84\x69\x74\x65\x6D\x05\x85\x76\x61\x6C\x75\x65\x06\x40\x10\x00\x00\x20\x01\x40\x00\x00\x05\x60\x07\x02\xDF\xFA\x02\xC0\x23\x09\x01\x01\x86\x75\x6E\x69\x71\x75\x65\x07\xA0\x2D\x02\x08\x01\xFF\x0C\x81\x00\x00\x01\x80\x20\x48\xA0\x35\x00\x81\x00\x00\x01\x80\x20\x48\xA0\x33\x00\x00\x00\x0C\x00\x0A\x00\x34\x8B\x0E\x5B\x42\xCD\xD6\x08"} err
assert_match "*Bad data format*" $err
r ping
}
}
} ;# tags } ;# tags
...@@ -195,3 +195,48 @@ start_server {tags {"repl external:skip"}} { ...@@ -195,3 +195,48 @@ start_server {tags {"repl external:skip"}} {
} }
} }
} }
start_server {tags {"repl external:skip"}} {
start_server {} {
set master [srv -1 client]
set master_host [srv -1 host]
set master_port [srv -1 port]
set replica [srv 0 client]
test {First server should have role slave after SLAVEOF} {
$replica slaveof $master_host $master_port
wait_for_condition 50 100 {
[s 0 role] eq {slave}
} else {
fail "Replication not started."
}
wait_for_sync $replica
}
test {Data divergence can happen under default conditions} {
$replica config set propagation-error-behavior ignore
$master debug replicate fake-command-1
# Wait for replication to normalize
$master set foo bar2
$master wait 1 2000
# Make sure we triggered the error, by finding the critical
# message and the fake command.
assert_equal [count_log_message 0 "fake-command-1"] 1
assert_equal [count_log_message 0 "== CRITICAL =="] 1
}
test {Data divergence is allowed on writable replicas} {
$replica config set replica-read-only no
$replica set number2 foo
$master incrby number2 1
$master wait 1 2000
assert_equal [$master get number2] 1
assert_equal [$replica get number2] foo
assert_equal [count_log_message 0 "incrby"] 1
}
}
}
...@@ -49,6 +49,7 @@ TEST_MODULES = \ ...@@ -49,6 +49,7 @@ TEST_MODULES = \
hash.so \ hash.so \
zset.so \ zset.so \
stream.so \ stream.so \
mallocsize.so \
aclcheck.so \ aclcheck.so \
list.so \ list.so \
subcommands.so \ subcommands.so \
...@@ -56,7 +57,8 @@ TEST_MODULES = \ ...@@ -56,7 +57,8 @@ TEST_MODULES = \
cmdintrospection.so \ cmdintrospection.so \
eventloop.so \ eventloop.so \
moduleconfigs.so \ moduleconfigs.so \
moduleconfigstwo.so moduleconfigstwo.so \
publish.so
.PHONY: all .PHONY: all
...@@ -69,7 +71,7 @@ all: $(TEST_MODULES) ...@@ -69,7 +71,7 @@ all: $(TEST_MODULES)
$(CC) -I../../src $(CFLAGS) $(SHOBJ_CFLAGS) -fPIC -c $< -o $@ $(CC) -I../../src $(CFLAGS) $(SHOBJ_CFLAGS) -fPIC -c $< -o $@
%.so: %.xo %.so: %.xo
$(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LDFLAGS) $(LIBS) $(LD) -o $@ $^ $(SHOBJ_LDFLAGS) $(LDFLAGS) $(LIBS)
.PHONY: clean .PHONY: clean
......
...@@ -92,7 +92,7 @@ int rm_call_aclcheck_cmd(RedisModuleCtx *ctx, RedisModuleUser *user, RedisModule ...@@ -92,7 +92,7 @@ int rm_call_aclcheck_cmd(RedisModuleCtx *ctx, RedisModuleUser *user, RedisModule
if (ret != 0) { if (ret != 0) {
RedisModule_ReplyWithError(ctx, "DENIED CMD"); RedisModule_ReplyWithError(ctx, "DENIED CMD");
/* Add entry to ACL log */ /* Add entry to ACL log */
RedisModule_ACLAddLogEntry(ctx, user, argv[1]); RedisModule_ACLAddLogEntry(ctx, user, argv[1], REDISMODULE_ACL_LOG_CMD);
return REDISMODULE_OK; return REDISMODULE_OK;
} }
......
This diff is collapsed.
This diff is collapsed.
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment