Unverified Commit d375595d authored by Oran Agra's avatar Oran Agra Committed by GitHub
Browse files

Merge pull request #10652 from oranagra/redis-7.0.0

Redis 7.0.0
parents fb4e0d40 c1f30206
......@@ -64,6 +64,7 @@
#define SCRIPT_READ_ONLY (1ULL<<5) /* indicate that the current script should only perform read commands */
#define SCRIPT_ALLOW_OOM (1ULL<<6) /* indicate to allow any command even if OOM reached */
#define SCRIPT_EVAL_MODE (1ULL<<7) /* Indicate that the current script called from legacy Lua */
#define SCRIPT_ALLOW_CROSS_SLOT (1ULL<<8) /* Indicate that the current script may access keys from multiple slots */
typedef struct scriptRunCtx scriptRunCtx;
struct scriptRunCtx {
......@@ -82,6 +83,7 @@ struct scriptRunCtx {
#define SCRIPT_FLAG_ALLOW_STALE (1ULL<<2)
#define SCRIPT_FLAG_NO_CLUSTER (1ULL<<3)
#define SCRIPT_FLAG_EVAL_COMPAT_MODE (1ULL<<4) /* EVAL Script backwards compatible behavior, no shebang provided */
#define SCRIPT_FLAG_ALLOW_CROSS_SLOT (1ULL<<5)
/* Defines a script flags */
typedef struct scriptFlag {
......
......@@ -41,6 +41,97 @@
#include <ctype.h>
#include <math.h>
/* Globals that are added by the Lua libraries */
static char *libraries_allow_list[] = {
"string",
"cjson",
"bit",
"cmsgpack",
"math",
"table",
"struct",
NULL,
};
/* Redis Lua API globals */
static char *redis_api_allow_list[] = {
"redis",
"__redis__err__handler", /* error handler for eval, currently located on globals.
Should move to registry. */
NULL,
};
/* Lua builtins */
static char *lua_builtins_allow_list[] = {
"xpcall",
"tostring",
"getfenv",
"setmetatable",
"next",
"assert",
"tonumber",
"rawequal",
"collectgarbage",
"getmetatable",
"rawset",
"pcall",
"coroutine",
"type",
"_G",
"select",
"unpack",
"gcinfo",
"pairs",
"rawget",
"loadstring",
"ipairs",
"_VERSION",
"setfenv",
"load",
"error",
NULL,
};
/* Lua builtins which are not documented on the Lua documentation */
static char *lua_builtins_not_documented_allow_list[] = {
"newproxy",
NULL,
};
/* Lua builtins which are allowed on initialization but will be removed right after */
static char *lua_builtins_removed_after_initialization_allow_list[] = {
"debug", /* debug will be set to nil after the error handler will be created */
NULL,
};
/* Those allow lists was created from the globals that was
* available to the user when the allow lists was first introduce.
* Because we do not want to break backward compatibility we keep
* all the globals. The allow lists will prevent us from accidentally
* creating unwanted globals in the future.
*
* Also notice that the allow list is only checked on start time,
* after that the global table is locked so not need to check anything.*/
static char **allow_lists[] = {
libraries_allow_list,
redis_api_allow_list,
lua_builtins_allow_list,
lua_builtins_not_documented_allow_list,
lua_builtins_removed_after_initialization_allow_list,
NULL,
};
/* Deny list contains elements which we know we do not want to add to globals
* and there is no need to print a warning message form them. We will print a
* log message only if an element was added to the globals and the element is
* not on the allow list nor on the back list. */
static char *deny_list[] = {
"dofile",
"loadfile",
"print",
NULL,
};
static int redis_math_random (lua_State *L);
static int redis_math_randomseed (lua_State *L);
static void redisProtocolToLuaType_Int(void *ctx, long long val, const char *proto, size_t proto_len);
......@@ -1113,15 +1204,6 @@ static void luaLoadLibraries(lua_State *lua) {
#endif
}
/* Remove a functions that we don't want to expose to the Redis scripting
* environment. */
static void luaRemoveUnsupportedFunctions(lua_State *lua) {
lua_pushnil(lua);
lua_setglobal(lua,"loadfile");
lua_pushnil(lua);
lua_setglobal(lua,"dofile");
}
/* Return sds of the string value located on stack at the given index.
* Return NULL if the value is not a string. */
sds luaGetStringSds(lua_State *lua, int index) {
......@@ -1135,107 +1217,120 @@ sds luaGetStringSds(lua_State *lua, int index) {
return str_sds;
}
/* This function installs metamethods in the global table _G that prevent
* the creation of globals accidentally.
*
* It should be the last to be called in the scripting engine initialization
* sequence, because it may interact with creation of globals.
*
* On Legacy Lua (eval) we need to check 'w ~= \"main\"' otherwise we will not be able
* to create the global 'function <sha> ()' variable. On Functions Lua engine we do not use
* this trick so it's not needed. */
void luaEnableGlobalsProtection(lua_State *lua, int is_eval) {
char *s[32];
sds code = sdsempty();
int j = 0;
/* strict.lua from: http://metalua.luaforge.net/src/lib/strict.lua.html.
* Modified to be adapted to Redis. */
s[j++]="local dbg=debug\n";
s[j++]="local mt = {}\n";
s[j++]="setmetatable(_G, mt)\n";
s[j++]="mt.__newindex = function (t, n, v)\n";
s[j++]=" if dbg.getinfo(2) then\n";
s[j++]=" local w = dbg.getinfo(2, \"S\").what\n";
s[j++]= is_eval ? " if w ~= \"main\" and w ~= \"C\" then\n" : " if w ~= \"C\" then\n";
s[j++]=" error(\"Script attempted to create global variable '\"..tostring(n)..\"'\", 2)\n";
s[j++]=" end\n";
s[j++]=" end\n";
s[j++]=" rawset(t, n, v)\n";
s[j++]="end\n";
s[j++]="mt.__index = function (t, n)\n";
s[j++]=" if dbg.getinfo(2) and dbg.getinfo(2, \"S\").what ~= \"C\" then\n";
s[j++]=" error(\"Script attempted to access nonexistent global variable '\"..tostring(n)..\"'\", 2)\n";
s[j++]=" end\n";
s[j++]=" return rawget(t, n)\n";
s[j++]="end\n";
s[j++]="debug = nil\n";
s[j++]=NULL;
for (j = 0; s[j] != NULL; j++) code = sdscatlen(code,s[j],strlen(s[j]));
luaL_loadbuffer(lua,code,sdslen(code),"@enable_strict_lua");
lua_pcall(lua,0,0,0);
sdsfree(code);
static int luaProtectedTableError(lua_State *lua) {
int argc = lua_gettop(lua);
if (argc != 2) {
serverLog(LL_WARNING, "malicious code trying to call luaProtectedTableError with wrong arguments");
luaL_error(lua, "Wrong number of arguments to luaProtectedTableError");
}
if (!lua_isstring(lua, -1) && !lua_isnumber(lua, -1)) {
luaL_error(lua, "Second argument to luaProtectedTableError must be a string or number");
}
const char *variable_name = lua_tostring(lua, -1);
luaL_error(lua, "Script attempted to access nonexistent global variable '%s'", variable_name);
return 0;
}
/* Create a global protection function and put it to registry.
* This need to be called once in the lua_State lifetime.
* After called it is possible to use luaSetGlobalProtection
* to set global protection on a give table.
/* Set a special metatable on the table on the top of the stack.
* The metatable will raise an error if the user tries to fetch
* an un-existing value.
*
* The function assumes the Lua stack have a least enough
* space to push 2 element, its up to the caller to verify
* this before calling this function.
*
* Notice, the difference between this and luaEnableGlobalsProtection
* is that luaEnableGlobalsProtection is enabling global protection
* on the current Lua globals. This registering a global protection
* function that later can be applied on any table. */
void luaRegisterGlobalProtectionFunction(lua_State *lua) {
lua_pushstring(lua, REGISTRY_SET_GLOBALS_PROTECTION_NAME);
char *global_protection_func = "local dbg = debug\n"
"local globals_protection = function (t)\n"
" local mt = {}\n"
" setmetatable(t, mt)\n"
" mt.__newindex = function (t, n, v)\n"
" if dbg.getinfo(2) then\n"
" local w = dbg.getinfo(2, \"S\").what\n"
" if w ~= \"C\" then\n"
" error(\"Script attempted to create global variable '\"..tostring(n)..\"'\", 2)\n"
" end"
" end"
" rawset(t, n, v)\n"
" end\n"
" mt.__index = function (t, n)\n"
" if dbg.getinfo(2) and dbg.getinfo(2, \"S\").what ~= \"C\" then\n"
" error(\"Script attempted to access nonexistent global variable '\"..tostring(n)..\"'\", 2)\n"
" end\n"
" return rawget(t, n)\n"
" end\n"
"end\n"
"return globals_protection";
int res = luaL_loadbuffer(lua, global_protection_func, strlen(global_protection_func), "@global_protection_def");
serverAssert(res == 0);
res = lua_pcall(lua,0,1,0);
serverAssert(res == 0);
lua_settable(lua, LUA_REGISTRYINDEX);
* this before calling this function. */
void luaSetErrorMetatable(lua_State *lua) {
lua_newtable(lua); /* push metatable */
lua_pushcfunction(lua, luaProtectedTableError); /* push get error handler */
lua_setfield(lua, -2, "__index");
lua_setmetatable(lua, -2);
}
static int luaNewIndexAllowList(lua_State *lua) {
int argc = lua_gettop(lua);
if (argc != 3) {
serverLog(LL_WARNING, "malicious code trying to call luaProtectedTableError with wrong arguments");
luaL_error(lua, "Wrong number of arguments to luaNewIndexAllowList");
}
if (!lua_istable(lua, -3)) {
luaL_error(lua, "first argument to luaNewIndexAllowList must be a table");
}
if (!lua_isstring(lua, -2) && !lua_isnumber(lua, -2)) {
luaL_error(lua, "Second argument to luaNewIndexAllowList must be a string or number");
}
const char *variable_name = lua_tostring(lua, -2);
/* check if the key is in our allow list */
char ***allow_l = allow_lists;
for (; *allow_l ; ++allow_l){
char **c = *allow_l;
for (; *c ; ++c) {
if (strcmp(*c, variable_name) == 0) {
break;
}
}
if (*c) {
break;
}
}
if (!*allow_l) {
/* Search the value on the back list, if its there we know that it was removed
* on purpose and there is no need to print a warning. */
char **c = deny_list;
for ( ; *c ; ++c) {
if (strcmp(*c, variable_name) == 0) {
break;
}
}
if (!*c) {
serverLog(LL_WARNING, "A key '%s' was added to Lua globals which is not on the globals allow list nor listed on the deny list.", variable_name);
}
} else {
lua_rawset(lua, -3);
}
return 0;
}
/* Set global protection on a given table.
* The table need to be located on the top of the lua stack.
* After called, it will no longer be possible to set
* new items on the table. The function is not removing
* the table from the top of the stack!
/* Set a metatable with '__newindex' function that verify that
* the new index appears on our globals while list.
*
* The function assumes the Lua stack have a least enough
* space to push 2 element, its up to the caller to verify
* this before calling this function. */
void luaSetGlobalProtection(lua_State *lua) {
lua_pushstring(lua, REGISTRY_SET_GLOBALS_PROTECTION_NAME);
lua_gettable(lua, LUA_REGISTRYINDEX);
lua_pushvalue(lua, -2);
int res = lua_pcall(lua, 1, 0, 0);
serverAssert(res == 0);
* The metatable is set on the table which located on the top
* of the stack.
*/
void luaSetAllowListProtection(lua_State *lua) {
lua_newtable(lua); /* push metatable */
lua_pushcfunction(lua, luaNewIndexAllowList); /* push get error handler */
lua_setfield(lua, -2, "__newindex");
lua_setmetatable(lua, -2);
}
/* Set the readonly flag on the table located on the top of the stack
* and recursively call this function on each table located on the original
* table. Also, recursively call this function on the metatables.*/
void luaSetTableProtectionRecursively(lua_State *lua) {
/* This protect us from a loop in case we already visited the table
* For example, globals has '_G' key which is pointing back to globals. */
if (lua_isreadonlytable(lua, -1)) {
return;
}
/* protect the current table */
lua_enablereadonlytable(lua, -1, 1);
lua_checkstack(lua, 2);
lua_pushnil(lua); /* Use nil to start iteration. */
while (lua_next(lua,-2)) {
/* Stack now: table, key, value */
if (lua_istable(lua, -1)) {
luaSetTableProtectionRecursively(lua);
}
lua_pop(lua, 1);
}
/* protect the metatable if exists */
if (lua_getmetatable(lua, -1)) {
luaSetTableProtectionRecursively(lua);
lua_pop(lua, 1); /* pop the metatable */
}
}
void luaRegisterVersion(lua_State* lua) {
......@@ -1272,8 +1367,11 @@ void luaRegisterLogFunction(lua_State* lua) {
}
void luaRegisterRedisAPI(lua_State* lua) {
lua_pushvalue(lua, LUA_GLOBALSINDEX);
luaSetAllowListProtection(lua);
lua_pop(lua, 1);
luaLoadLibraries(lua);
luaRemoveUnsupportedFunctions(lua);
lua_pushcfunction(lua,luaRedisPcall);
lua_setglobal(lua, "pcall");
......@@ -1504,9 +1602,19 @@ void luaCallFunction(scriptRunCtx* run_ctx, lua_State *lua, robj** keys, size_t
* EVAL received. */
luaCreateArray(lua,keys,nkeys);
/* On eval, keys and arguments are globals. */
if (run_ctx->flags & SCRIPT_EVAL_MODE) lua_setglobal(lua,"KEYS");
if (run_ctx->flags & SCRIPT_EVAL_MODE){
/* open global protection to set KEYS */
lua_enablereadonlytable(lua, LUA_GLOBALSINDEX, 0);
lua_setglobal(lua,"KEYS");
lua_enablereadonlytable(lua, LUA_GLOBALSINDEX, 1);
}
luaCreateArray(lua,args,nargs);
if (run_ctx->flags & SCRIPT_EVAL_MODE) lua_setglobal(lua,"ARGV");
if (run_ctx->flags & SCRIPT_EVAL_MODE){
/* open global protection to set ARGV */
lua_enablereadonlytable(lua, LUA_GLOBALSINDEX, 0);
lua_setglobal(lua,"ARGV");
lua_enablereadonlytable(lua, LUA_GLOBALSINDEX, 1);
}
/* At this point whether this script was never seen before or if it was
* already defined, we can call it.
......
......@@ -67,9 +67,10 @@ typedef struct errorInfo {
void luaRegisterRedisAPI(lua_State* lua);
sds luaGetStringSds(lua_State *lua, int index);
void luaEnableGlobalsProtection(lua_State *lua, int is_eval);
void luaRegisterGlobalProtectionFunction(lua_State *lua);
void luaSetGlobalProtection(lua_State *lua);
void luaSetErrorMetatable(lua_State *lua);
void luaSetAllowListProtection(lua_State *lua);
void luaSetTableProtectionRecursively(lua_State *lua);
void luaRegisterLogFunction(lua_State* lua);
void luaRegisterVersion(lua_State* lua);
void luaPushErrorBuff(lua_State *lua, sds err_buff);
......
......@@ -705,7 +705,7 @@ void sentinelEvent(int level, char *type, sentinelRedisInstance *ri,
if (level != LL_DEBUG) {
channel = createStringObject(type,strlen(type));
payload = createStringObject(msg,strlen(msg));
pubsubPublishMessage(channel,payload);
pubsubPublishMessage(channel,payload,0);
decrRefCount(channel);
decrRefCount(payload);
}
......
This diff is collapsed.
......@@ -603,6 +603,7 @@ typedef enum {
#define NOTIFY_KEY_MISS (1<<11) /* m (Note: This one is excluded from NOTIFY_ALL on purpose) */
#define NOTIFY_LOADED (1<<12) /* module only key space notification, indicate a key loaded from rdb */
#define NOTIFY_MODULE (1<<13) /* d, module key space notification */
#define NOTIFY_NEW (1<<14) /* n, new key notification */
#define NOTIFY_ALL (NOTIFY_GENERIC | NOTIFY_STRING | NOTIFY_LIST | NOTIFY_SET | NOTIFY_HASH | NOTIFY_ZSET | NOTIFY_EXPIRED | NOTIFY_EVICTED | NOTIFY_STREAM | NOTIFY_MODULE) /* A flag */
/* Using the following macro you can run code inside serverCron() with the
......@@ -1061,7 +1062,7 @@ typedef struct replBacklog {
listNode *ref_repl_buf_node; /* Referenced node of replication buffer blocks,
* see the definition of replBufBlock. */
size_t unindexed_count; /* The count from last creating index block. */
rax *blocks_index; /* The index of reocrded blocks of replication
rax *blocks_index; /* The index of recorded blocks of replication
* buffer for quickly searching replication
* offset on partial resynchronization. */
long long histlen; /* Backlog actual data length */
......@@ -1106,6 +1107,7 @@ typedef struct client {
buffer or object being sent. */
time_t ctime; /* Client creation time. */
long duration; /* Current command duration. Used for measuring latency of blocking/non-blocking cmds */
int slot; /* The slot the client is executing against. Set to -1 if no slot is being used */
time_t lastinteraction; /* Time of the last interaction, used for timeout */
time_t obuf_soft_limit_reached_time;
uint64_t flags; /* Client flags: CLIENT_* macros. */
......@@ -1323,6 +1325,15 @@ struct redisMemOverhead {
} *db;
};
/* Replication error behavior determines the replica behavior
* when it receives an error over the replication stream. In
* either case the error is logged. */
typedef enum {
PROPAGATION_ERR_BEHAVIOR_IGNORE = 0,
PROPAGATION_ERR_BEHAVIOR_PANIC,
PROPAGATION_ERR_BEHAVIOR_PANIC_ON_REPLICAS
} replicationErrorBehavior;
/* This structure can be optionally passed to RDB save/load functions in
* order to implement additional functionalities, by storing and loading
* metadata to the RDB file.
......@@ -1451,6 +1462,7 @@ struct redisServer {
redisAtomic unsigned int lruclock; /* Clock for LRU eviction */
volatile sig_atomic_t shutdown_asap; /* Shutdown ordered by signal handler. */
mstime_t shutdown_mstime; /* Timestamp to limit graceful shutdown. */
int last_sig_received; /* Indicates the last SIGNAL received, if any (e.g., SIGINT or SIGTERM). */
int shutdown_flags; /* Flags passed to prepareForShutdown(). */
int activerehashing; /* Incremental rehash in serverCron() */
int active_defrag_running; /* Active defragmentation running (holds current scan aggressiveness) */
......@@ -1493,6 +1505,7 @@ struct redisServer {
socketFds ipfd; /* TCP socket file descriptors */
socketFds tlsfd; /* TLS socket file descriptors */
int sofd; /* Unix socket file descriptor */
uint32_t socket_mark_id; /* ID for listen socket marking */
socketFds cfd; /* Cluster bus listening socket */
list *clients; /* List of active clients */
list *clients_to_close; /* Clients to close asynchronously */
......@@ -1555,6 +1568,7 @@ struct redisServer {
monotime stat_last_active_defrag_time; /* Timestamp of current active defrag start */
size_t stat_peak_memory; /* Max used memory record */
long long stat_aof_rewrites; /* number of aof file rewrites performed */
long long stat_aofrw_consecutive_failures; /* The number of consecutive failures of aofrw */
long long stat_rdb_saves; /* number of rdb saves performed */
long long stat_fork_time; /* Time needed to perform latest fork() */
double stat_fork_rate; /* Fork rate in GB/sec. */
......@@ -1717,6 +1731,8 @@ struct redisServer {
* abort(). useful for Valgrind. */
/* Shutdown */
int shutdown_timeout; /* Graceful shutdown time limit in seconds. */
int shutdown_on_sigint; /* Shutdown flags configured for SIGINT. */
int shutdown_on_sigterm; /* Shutdown flags configured for SIGTERM. */
/* Replication (master) */
char replid[CONFIG_RUN_ID_SIZE+1]; /* My current replication ID. */
......@@ -1769,6 +1785,10 @@ struct redisServer {
int replica_announced; /* If true, replica is announced by Sentinel */
int slave_announce_port; /* Give the master this listening port. */
char *slave_announce_ip; /* Give the master this ip address. */
int propagation_error_behavior; /* Configures the behavior of the replica
* when it receives an error on the replication stream */
int repl_ignore_disk_write_error; /* Configures whether replicas panic when unable to
* persist writes to AOF. */
/* The following two fields is where we store master PSYNC replid/offset
* while the PSYNC is in progress. At the end we'll copy the fields into
* the server->master client structure. */
......@@ -2042,6 +2062,7 @@ typedef struct redisCommandArg {
const char *summary;
const char *since;
int flags;
const char *deprecated_since;
struct redisCommandArg *subargs;
/* runtime populated data */
int num_args;
......@@ -2349,6 +2370,7 @@ int moduleGetCommandChannelsViaAPI(struct redisCommand *cmd, robj **argv, int ar
moduleType *moduleTypeLookupModuleByID(uint64_t id);
void moduleTypeNameByID(char *name, uint64_t moduleid);
const char *moduleTypeModuleName(moduleType *mt);
const char *moduleNameFromCommand(struct redisCommand *cmd);
void moduleFreeContext(struct RedisModuleCtx *ctx);
void unblockClientFromModule(client *c);
void moduleHandleBlockedClients(void);
......@@ -2509,7 +2531,7 @@ void unprotectClient(client *c);
void initThreadedIO(void);
client *lookupClientByID(uint64_t id);
int authRequired(client *c);
void clientInstallWriteHandler(client *c);
void putClientInPendingWriteQueue(client *c);
#ifdef __GNUC__
void addReplyErrorFormatEx(client *c, int flags, const char *fmt, ...)
......@@ -2860,6 +2882,8 @@ struct redisCommand *lookupCommandBySds(sds s);
struct redisCommand *lookupCommandByCStringLogic(dict *commands, const char *s);
struct redisCommand *lookupCommandByCString(const char *s);
struct redisCommand *lookupCommandOrOriginal(robj **argv, int argc);
int commandCheckExistence(client *c, sds *err);
int commandCheckArity(client *c, sds *err);
void startCommandExecution();
int incrCommandStatsOnError(struct redisCommand *cmd, int flags);
void call(client *c, int flags);
......@@ -2877,7 +2901,7 @@ int prepareForShutdown(int flags);
void replyToClientsBlockedOnShutdown(void);
int abortShutdown(void);
void afterCommand(client *c);
int inNestedCall(void);
int mustObeyClient(client *c);
#ifdef __GNUC__
void _serverLog(int level, const char *fmt, ...)
__attribute__((format(printf, 2, 3)));
......@@ -2962,8 +2986,8 @@ int pubsubUnsubscribeAllChannels(client *c, int notify);
int pubsubUnsubscribeShardAllChannels(client *c, int notify);
void pubsubUnsubscribeShardChannels(robj **channels, unsigned int count);
int pubsubUnsubscribeAllPatterns(client *c, int notify);
int pubsubPublishMessage(robj *channel, robj *message);
int pubsubPublishMessageShard(robj *channel, robj *message);
int pubsubPublishMessage(robj *channel, robj *message, int sharded);
int pubsubPublishMessageAndPropagateToCluster(robj *channel, robj *message, int sharded);
void addReplyPubsubMessage(client *c, robj *channel, robj *msg);
int serverPubsubSubscriptionCount();
int serverPubsubShardSubscriptionCount();
......
......@@ -146,39 +146,24 @@ robj *hashTypeGetValueObject(robj *o, sds field) {
* exist. */
size_t hashTypeGetValueLength(robj *o, sds field) {
size_t len = 0;
if (o->encoding == OBJ_ENCODING_LISTPACK) {
unsigned char *vstr = NULL;
unsigned int vlen = UINT_MAX;
long long vll = LLONG_MAX;
if (hashTypeGetFromListpack(o, field, &vstr, &vlen, &vll) == 0)
if (hashTypeGetValue(o, field, &vstr, &vlen, &vll) == C_OK)
len = vstr ? vlen : sdigits10(vll);
} else if (o->encoding == OBJ_ENCODING_HT) {
sds aux;
if ((aux = hashTypeGetFromHashTable(o, field)) != NULL)
len = sdslen(aux);
} else {
serverPanic("Unknown hash encoding");
}
return len;
}
/* Test if the specified field exists in the given hash. Returns 1 if the field
* exists, and 0 when it doesn't. */
int hashTypeExists(robj *o, sds field) {
if (o->encoding == OBJ_ENCODING_LISTPACK) {
unsigned char *vstr = NULL;
unsigned int vlen = UINT_MAX;
long long vll = LLONG_MAX;
if (hashTypeGetFromListpack(o, field, &vstr, &vlen, &vll) == 0) return 1;
} else if (o->encoding == OBJ_ENCODING_HT) {
if (hashTypeGetFromHashTable(o, field) != NULL) return 1;
} else {
serverPanic("Unknown hash encoding");
}
return 0;
return hashTypeGetValue(o, field, &vstr, &vlen, &vll) == C_OK;
}
/* Add a new field, overwrite the old with the new value if it already exists.
......@@ -205,6 +190,14 @@ int hashTypeExists(robj *o, sds field) {
int hashTypeSet(robj *o, sds field, sds value, int flags) {
int update = 0;
/* Check if the field is too long for listpack, and convert before adding the item.
* This is needed for HINCRBY* case since in other commands this is handled early by
* hashTypeTryConversion, so this check will be a NOP. */
if (o->encoding == OBJ_ENCODING_LISTPACK) {
if (sdslen(field) > server.hash_max_listpack_value || sdslen(value) > server.hash_max_listpack_value)
hashTypeConvert(o, OBJ_ENCODING_HT);
}
if (o->encoding == OBJ_ENCODING_LISTPACK) {
unsigned char *zl, *fptr, *vptr;
......@@ -717,37 +710,23 @@ void hincrbyfloatCommand(client *c) {
}
static void addHashFieldToReply(client *c, robj *o, sds field) {
int ret;
if (o == NULL) {
addReplyNull(c);
return;
}
if (o->encoding == OBJ_ENCODING_LISTPACK) {
unsigned char *vstr = NULL;
unsigned int vlen = UINT_MAX;
long long vll = LLONG_MAX;
ret = hashTypeGetFromListpack(o, field, &vstr, &vlen, &vll);
if (ret < 0) {
addReplyNull(c);
} else {
if (hashTypeGetValue(o, field, &vstr, &vlen, &vll) == C_OK) {
if (vstr) {
addReplyBulkCBuffer(c, vstr, vlen);
} else {
addReplyBulkLongLong(c, vll);
}
}
} else if (o->encoding == OBJ_ENCODING_HT) {
sds value = hashTypeGetFromHashTable(o, field);
if (value == NULL)
addReplyNull(c);
else
addReplyBulkCBuffer(c, value, sdslen(value));
} else {
serverPanic("Unknown hash encoding");
addReplyNull(c);
}
}
......@@ -907,7 +886,7 @@ void hscanCommand(client *c) {
scanGenericCommand(c,o,cursor);
}
static void harndfieldReplyWithListpack(client *c, unsigned int count, listpackEntry *keys, listpackEntry *vals) {
static void hrandfieldReplyWithListpack(client *c, unsigned int count, listpackEntry *keys, listpackEntry *vals) {
for (unsigned long i = 0; i < count; i++) {
if (vals && c->resp > 2)
addReplyArrayLen(c,2);
......@@ -990,7 +969,7 @@ void hrandfieldWithCountCommand(client *c, long l, int withvalues) {
sample_count = count > limit ? limit : count;
count -= sample_count;
lpRandomPairs(hash->ptr, sample_count, keys, vals);
harndfieldReplyWithListpack(c, sample_count, keys, vals);
hrandfieldReplyWithListpack(c, sample_count, keys, vals);
}
zfree(keys);
zfree(vals);
......@@ -1092,7 +1071,7 @@ void hrandfieldWithCountCommand(client *c, long l, int withvalues) {
if (withvalues)
vals = zmalloc(sizeof(listpackEntry)*count);
serverAssert(lpRandomPairsUnique(hash->ptr, count, keys, vals) == count);
harndfieldReplyWithListpack(c, count, keys, vals);
hrandfieldReplyWithListpack(c, count, keys, vals);
zfree(keys);
zfree(vals);
return;
......
......@@ -1000,7 +1000,7 @@ static int streamParseAddOrTrimArgsOrReply(client *c, streamAddTrimArgs *args, i
return -1;
}
if (c == server.master || c->id == CLIENT_ID_AOF) {
if (mustObeyClient(c)) {
/* If command came from master or from AOF we must not enforce maxnodes
* (The maxlen/minid argument was re-written to make sure there's no
* inconsistency). */
......@@ -1370,24 +1370,35 @@ void streamLastValidID(stream *s, streamID *maxid)
streamIteratorStop(&si);
}
/* Maximum size for a stream ID string. In theory 20*2+1 should be enough,
* But to avoid chance for off by one issues and null-term, in case this will
* be used as parsing buffer, we use a slightly larger buffer. On the other
* hand considering sds header is gonna add 4 bytes, we wanna keep below the
* allocator's 48 bytes bin. */
#define STREAM_ID_STR_LEN 44
sds createStreamIDString(streamID *id) {
/* Optimization: pre-allocate a big enough buffer to avoid reallocs. */
sds str = sdsnewlen(SDS_NOINIT, STREAM_ID_STR_LEN);
sdssetlen(str, 0);
return sdscatfmt(str,"%U-%U", id->ms,id->seq);
}
/* Emit a reply in the client output buffer by formatting a Stream ID
* in the standard <ms>-<seq> format, using the simple string protocol
* of REPL. */
void addReplyStreamID(client *c, streamID *id) {
sds replyid = sdscatfmt(sdsempty(),"%U-%U",id->ms,id->seq);
addReplyBulkSds(c,replyid);
addReplyBulkSds(c,createStreamIDString(id));
}
void setDeferredReplyStreamID(client *c, void *dr, streamID *id) {
sds replyid = sdscatfmt(sdsempty(),"%U-%U",id->ms,id->seq);
setDeferredReplyBulkSds(c, dr, replyid);
setDeferredReplyBulkSds(c, dr, createStreamIDString(id));
}
/* Similar to the above function, but just creates an object, usually useful
* for replication purposes to create arguments. */
robj *createObjectFromStreamID(streamID *id) {
return createObject(OBJ_STRING, sdscatfmt(sdsempty(),"%U-%U",
id->ms,id->seq));
return createObject(OBJ_STRING, createStreamIDString(id));
}
/* Returns non-zero if the ID is 0-0. */
......@@ -2025,7 +2036,8 @@ void xaddCommand(client *c) {
addReplyError(c,"Elements are too large to be stored");
return;
}
addReplyStreamID(c,&id);
sds replyid = createStreamIDString(&id);
addReplyBulkCBuffer(c, replyid, sdslen(replyid));
signalModifiedKey(c,c->db,c->argv[1]);
notifyKeyspaceEvent(NOTIFY_STREAM,"xadd",c->argv[1],c->db->id);
......@@ -2050,9 +2062,11 @@ void xaddCommand(client *c) {
/* Let's rewrite the ID argument with the one actually generated for
* AOF/replication propagation. */
if (!parsed_args.id_given || !parsed_args.seq_given) {
robj *idarg = createObjectFromStreamID(&id);
robj *idarg = createObject(OBJ_STRING, replyid);
rewriteClientCommandArgument(c, idpos, idarg);
decrRefCount(idarg);
} else {
sdsfree(replyid);
}
/* We need to signal to blocked clients that there is new data on this
......
......@@ -38,7 +38,7 @@ int getGenericCommand(client *c);
*----------------------------------------------------------------------------*/
static int checkStringLength(client *c, long long size) {
if (!(c->flags & CLIENT_MASTER) && size > server.proto_max_bulk_len) {
if (!mustObeyClient(c) && size > server.proto_max_bulk_len) {
addReplyError(c,"string exceeds maximum allowed size (proto-max-bulk-len)");
return C_ERR;
}
......@@ -792,7 +792,7 @@ void lcsCommand(client *c) {
/* Setup an uint32_t array to store at LCS[i,j] the length of the
* LCS A0..i-1, B0..j-1. Note that we have a linear array here, so
* we index it as LCS[j+(blen+1)*j] */
* we index it as LCS[j+(blen+1)*i] */
#define LCS(A,B) lcs[(B)+((A)*(blen+1))]
/* Try to allocate the LCS table, and abort on overflow or insufficient memory. */
......
......@@ -1029,16 +1029,24 @@ unsigned char *zzlInsertAt(unsigned char *zl, unsigned char *eptr, sds ele, doub
unsigned char *sptr;
char scorebuf[MAX_D2STRING_CHARS];
int scorelen;
long long lscore;
int score_is_long = double2ll(score, &lscore);
if (!score_is_long)
scorelen = d2string(scorebuf,sizeof(scorebuf),score);
if (eptr == NULL) {
zl = lpAppend(zl,(unsigned char*)ele,sdslen(ele));
if (score_is_long)
zl = lpAppendInteger(zl,lscore);
else
zl = lpAppend(zl,(unsigned char*)scorebuf,scorelen);
} else {
/* Insert member before the element 'eptr'. */
zl = lpInsertString(zl,(unsigned char*)ele,sdslen(ele),eptr,LP_BEFORE,&sptr);
/* Insert score after the member. */
if (score_is_long)
zl = lpInsertInteger(zl,lscore,sptr,LP_AFTER,NULL);
else
zl = lpInsertString(zl,(unsigned char*)scorebuf,scorelen,sptr,LP_AFTER,NULL);
}
return zl;
......@@ -3964,7 +3972,7 @@ void zpopminCommand(client *c) {
zpopMinMaxCommand(c, ZSET_MIN);
}
/* ZMAXPOP key [<count>] */
/* ZPOPMAX key [<count>] */
void zpopmaxCommand(client *c) {
zpopMinMaxCommand(c, ZSET_MAX);
}
......@@ -4351,12 +4359,12 @@ void zmpopGenericCommand(client *c, int numkeys_idx, int is_block) {
}
}
/* ZMPOP numkeys [<key> ...] MIN|MAX [COUNT count] */
/* ZMPOP numkeys key [<key> ...] MIN|MAX [COUNT count] */
void zmpopCommand(client *c) {
zmpopGenericCommand(c, 1, 0);
}
/* BZMPOP timeout numkeys [<key> ...] MIN|MAX [COUNT count] */
/* BZMPOP timeout numkeys key [<key> ...] MIN|MAX [COUNT count] */
void bzmpopCommand(client *c) {
zmpopGenericCommand(c, 2, 1);
}
......@@ -58,7 +58,7 @@ int clientsCronHandleTimeout(client *c, mstime_t now_ms) {
if (server.maxidletime &&
/* This handles the idle clients connection timeout if set. */
!(c->flags & CLIENT_SLAVE) && /* No timeout for slaves and monitors */
!(c->flags & CLIENT_MASTER) && /* No timeout for masters */
!mustObeyClient(c) && /* No timeout for masters and AOF */
!(c->flags & CLIENT_BLOCKED) && /* No timeout for BLPOP */
!(c->flags & CLIENT_PUBSUB) && /* No timeout for Pub/Sub clients */
(now - c->lastinteraction > server.maxidletime))
......
This diff is collapsed.
......@@ -75,6 +75,7 @@ int string2d(const char *s, size_t slen, double *dp);
int trimDoubleString(char *buf, size_t len);
int d2string(char *buf, size_t len, double value);
int ld2string(char *buf, size_t len, long double value, ld2string_mode mode);
int double2ll(double d, long long *out);
int yesnotoi(char *s);
sds getAbsolutePath(char *filename);
long getTimeZone(void);
......
#define REDIS_VERSION "6.9.242"
#define REDIS_VERSION_NUM 0x000609f2
#define REDIS_VERSION "7.0.0"
#define REDIS_VERSION_NUM 0x00070000
......@@ -30,3 +30,5 @@ activerehashing yes
enable-protected-configs yes
enable-debug-command yes
enable-module-command yes
propagation-error-behavior panic
\ No newline at end of file
This diff is collapsed.
......@@ -632,7 +632,6 @@ test {corrupt payload: fuzzer findings - stream PEL without consumer} {
r debug set-skip-checksum-validation 1
catch {r restore _stream 0 "\x0F\x01\x10\x00\x00\x01\x7B\x08\xF0\xB2\x34\x00\x00\x00\x00\x00\x00\x00\x00\xC3\x3B\x40\x42\x19\x42\x00\x00\x00\x18\x00\x02\x01\x01\x01\x02\x01\x84\x69\x74\x65\x6D\x05\x85\x76\x61\x6C\x75\x65\x06\x00\x20\x10\x00\x00\x20\x01\x00\x01\x20\x03\x02\x05\x01\x03\x20\x05\x40\x00\x04\x82\x5F\x31\x03\x05\x60\x19\x80\x32\x02\x05\x01\xFF\x02\x81\x00\x00\x01\x7B\x08\xF0\xB2\x34\x02\x01\x07\x6D\x79\x67\x72\x6F\x75\x70\x81\x00\x00\x01\x7B\x08\xF0\xB2\x34\x01\x01\x00\x00\x01\x7B\x08\xF0\xB2\x34\x00\x00\x00\x00\x00\x00\x00\x01\x35\xB2\xF0\x08\x7B\x01\x00\x00\x01\x01\x13\x41\x6C\x69\x63\x65\x35\xB2\xF0\x08\x7B\x01\x00\x00\x01\x00\x00\x01\x7B\x08\xF0\xB2\x34\x00\x00\x00\x00\x00\x00\x00\x01\x09\x00\x28\x2F\xE0\xC5\x04\xBB\xA7\x31"} err
assert_match "*Bad data format*" $err
#catch {r XINFO STREAM _stream FULL }
r ping
}
}
......@@ -674,7 +673,6 @@ test {corrupt payload: fuzzer findings - stream with non-integer entry id} {
r config set sanitize-dump-payload yes
r debug set-skip-checksum-validation 1
catch {r restore _streambig 0 "\x0F\x03\x10\x00\x00\x01\x7B\x13\x34\xC3\xB2\x00\x00\x00\x00\x00\x00\x00\x00\xC3\x40\x4F\x40\x5C\x18\x5C\x00\x00\x00\x24\x00\x05\x01\x00\x01\x02\x01\x84\x69\x74\x65\x6D\x05\x85\x76\x61\x6C\x75\x65\x06\x40\x10\x00\x80\x20\x01\x00\x01\x20\x03\x00\x05\x20\x1C\x40\x09\x05\x01\x01\x82\x5F\x31\x03\x80\x0D\x00\x02\x20\x0D\x00\x02\xA0\x19\x00\x03\x20\x0B\x02\x82\x5F\x33\xA0\x19\x00\x04\x20\x0D\x00\x04\x20\x19\x00\xFF\x10\x00\x00\x01\x7B\x13\x34\xC3\xB2\x00\x00\x00\x00\x00\x00\x00\x05\xC3\x40\x56\x40\x61\x18\x61\x00\x00\x00\x24\x00\x05\x01\x00\x01\x02\x01\x84\x69\x74\x65\x6D\x05\x85\x76\x61\x6C\x75\x65\x06\x40\x10\x00\x00\x20\x01\x06\x01\x01\x82\x5F\x35\x03\x05\x20\x1E\x40\x0B\x03\x01\x01\x06\x01\x40\x0B\x03\x01\x01\xDF\xFB\x20\x05\x02\x82\x5F\x37\x60\x1A\x20\x0E\x00\xFC\x20\x05\x00\x08\xC0\x1B\x00\xFD\x20\x0C\x02\x82\x5F\x39\x20\x1B\x00\xFF\x10\x00\x00\x01\x7B\x13\x34\xC3\xB3\x00\x00\x00\x00\x00\x00\x00\x03\xC3\x3D\x40\x4A\x18\x4A\x00\x00\x00\x15\x00\x02\x01\x00\x01\x02\x01\x84\x69\x74\x65\x6D\x05\x85\x76\x61\x6C\x75\x65\x06\x40\x10\x00\x00\x20\x01\x40\x00\x00\x05\x60\x07\x02\xDF\xFD\x02\xC0\x23\x09\x01\x01\x86\x75\x6E\x69\x71\x75\x65\x07\xA0\x2D\x02\x08\x01\xFF\x0C\x81\x00\x00\x01\x7B\x13\x34\xC3\xB4\x00\x00\x09\x00\x9D\xBD\xD5\xB9\x33\xC4\xC5\xFF"} err
#catch {r XINFO STREAM _streambig FULL }
assert_match "*Bad data format*" $err
r ping
}
......@@ -782,5 +780,15 @@ test {corrupt payload: fuzzer findings - zset zslInsert with a NAN score} {
}
}
test {corrupt payload: fuzzer findings - streamLastValidID panic} {
start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
r config set sanitize-dump-payload yes
r debug set-skip-checksum-validation 1
catch {r restore _streambig 0 "\x13\xC0\x10\x00\x00\x01\x80\x20\x48\xA0\x33\x00\x00\x00\x00\x00\x00\x00\x00\xC3\x40\x4F\x40\x5C\x18\x5C\x00\x00\x00\x24\x00\x05\x01\x00\x01\x02\x01\x84\x69\x74\x65\x6D\x05\x85\x76\x61\x6C\x75\x65\x06\x40\x10\x00\x00\x20\x01\x00\x01\x20\x03\x00\x05\x20\x1C\x40\x09\x05\x01\x01\x82\x5F\x31\x03\x80\x0D\x00\x02\x20\x0D\x00\x02\xA0\x19\x00\x03\x20\x0B\x02\x82\x5F\x33\x60\x19\x40\x2F\x02\x01\x01\x04\x20\x19\x00\xFF\x10\x00\x00\x01\x80\x20\x48\xA0\x34\x00\x00\x00\x00\x00\x00\x00\x01\xC3\x40\x51\x40\x5E\x18\x5E\x00\x00\x00\x24\x00\x05\x01\x00\x01\x02\x01\x84\x69\x74\x65\x6D\x05\x85\x76\x61\x6C\x75\x65\x06\x40\x10\x00\x00\x20\x01\x06\x01\x01\x82\x5F\x35\x03\x05\x20\x1E\x40\x0B\x03\x01\x01\x06\x01\x80\x0B\x00\x02\x20\x0B\x02\x82\x5F\x37\xA0\x19\x00\x03\x20\x0D\x00\x08\xA0\x19\x00\x04\x20\x0B\x02\x82\x5F\x39\x20\x19\x00\xFF\x10\x00\x00\x01\x80\x20\x48\xA0\x34\x00\x00\x00\x00\x00\x00\x00\x06\xC3\x3D\x40\x4A\x18\x4A\x00\x00\x00\x15\x00\x02\x01\x00\x01\x02\x01\x84\x69\x74\x65\x6D\x05\x85\x76\x61\x6C\x75\x65\x06\x40\x10\x00\x00\x20\x01\x40\x00\x00\x05\x60\x07\x02\xDF\xFA\x02\xC0\x23\x09\x01\x01\x86\x75\x6E\x69\x71\x75\x65\x07\xA0\x2D\x02\x08\x01\xFF\x0C\x81\x00\x00\x01\x80\x20\x48\xA0\x35\x00\x81\x00\x00\x01\x80\x20\x48\xA0\x33\x00\x00\x00\x0C\x00\x0A\x00\x34\x8B\x0E\x5B\x42\xCD\xD6\x08"} err
assert_match "*Bad data format*" $err
r ping
}
}
} ;# tags
This diff is collapsed.
This diff is collapsed.
......@@ -92,7 +92,7 @@ int rm_call_aclcheck_cmd(RedisModuleCtx *ctx, RedisModuleUser *user, RedisModule
if (ret != 0) {
RedisModule_ReplyWithError(ctx, "DENIED CMD");
/* Add entry to ACL log */
RedisModule_ACLAddLogEntry(ctx, user, argv[1]);
RedisModule_ACLAddLogEntry(ctx, user, argv[1], REDISMODULE_ACL_LOG_CMD);
return REDISMODULE_OK;
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment