Unverified Commit d375595d authored by Oran Agra's avatar Oran Agra Committed by GitHub
Browse files

Merge pull request #10652 from oranagra/redis-7.0.0

Redis 7.0.0
parents fb4e0d40 c1f30206
...@@ -64,6 +64,7 @@ ...@@ -64,6 +64,7 @@
#define SCRIPT_READ_ONLY (1ULL<<5) /* indicate that the current script should only perform read commands */ #define SCRIPT_READ_ONLY (1ULL<<5) /* indicate that the current script should only perform read commands */
#define SCRIPT_ALLOW_OOM (1ULL<<6) /* indicate to allow any command even if OOM reached */ #define SCRIPT_ALLOW_OOM (1ULL<<6) /* indicate to allow any command even if OOM reached */
#define SCRIPT_EVAL_MODE (1ULL<<7) /* Indicate that the current script called from legacy Lua */ #define SCRIPT_EVAL_MODE (1ULL<<7) /* Indicate that the current script called from legacy Lua */
#define SCRIPT_ALLOW_CROSS_SLOT (1ULL<<8) /* Indicate that the current script may access keys from multiple slots */
typedef struct scriptRunCtx scriptRunCtx; typedef struct scriptRunCtx scriptRunCtx;
struct scriptRunCtx { struct scriptRunCtx {
...@@ -82,6 +83,7 @@ struct scriptRunCtx { ...@@ -82,6 +83,7 @@ struct scriptRunCtx {
#define SCRIPT_FLAG_ALLOW_STALE (1ULL<<2) #define SCRIPT_FLAG_ALLOW_STALE (1ULL<<2)
#define SCRIPT_FLAG_NO_CLUSTER (1ULL<<3) #define SCRIPT_FLAG_NO_CLUSTER (1ULL<<3)
#define SCRIPT_FLAG_EVAL_COMPAT_MODE (1ULL<<4) /* EVAL Script backwards compatible behavior, no shebang provided */ #define SCRIPT_FLAG_EVAL_COMPAT_MODE (1ULL<<4) /* EVAL Script backwards compatible behavior, no shebang provided */
#define SCRIPT_FLAG_ALLOW_CROSS_SLOT (1ULL<<5)
/* Defines a script flags */ /* Defines a script flags */
typedef struct scriptFlag { typedef struct scriptFlag {
......
...@@ -41,6 +41,97 @@ ...@@ -41,6 +41,97 @@
#include <ctype.h> #include <ctype.h>
#include <math.h> #include <math.h>
/* Globals that are added by the Lua libraries */
static char *libraries_allow_list[] = {
"string",
"cjson",
"bit",
"cmsgpack",
"math",
"table",
"struct",
NULL,
};
/* Redis Lua API globals */
static char *redis_api_allow_list[] = {
"redis",
"__redis__err__handler", /* error handler for eval, currently located on globals.
Should move to registry. */
NULL,
};
/* Lua builtins */
static char *lua_builtins_allow_list[] = {
"xpcall",
"tostring",
"getfenv",
"setmetatable",
"next",
"assert",
"tonumber",
"rawequal",
"collectgarbage",
"getmetatable",
"rawset",
"pcall",
"coroutine",
"type",
"_G",
"select",
"unpack",
"gcinfo",
"pairs",
"rawget",
"loadstring",
"ipairs",
"_VERSION",
"setfenv",
"load",
"error",
NULL,
};
/* Lua builtins which are not documented on the Lua documentation */
static char *lua_builtins_not_documented_allow_list[] = {
"newproxy",
NULL,
};
/* Lua builtins which are allowed on initialization but will be removed right after */
static char *lua_builtins_removed_after_initialization_allow_list[] = {
"debug", /* debug will be set to nil after the error handler will be created */
NULL,
};
/* Those allow lists was created from the globals that was
* available to the user when the allow lists was first introduce.
* Because we do not want to break backward compatibility we keep
* all the globals. The allow lists will prevent us from accidentally
* creating unwanted globals in the future.
*
* Also notice that the allow list is only checked on start time,
* after that the global table is locked so not need to check anything.*/
static char **allow_lists[] = {
libraries_allow_list,
redis_api_allow_list,
lua_builtins_allow_list,
lua_builtins_not_documented_allow_list,
lua_builtins_removed_after_initialization_allow_list,
NULL,
};
/* Deny list contains elements which we know we do not want to add to globals
* and there is no need to print a warning message form them. We will print a
* log message only if an element was added to the globals and the element is
* not on the allow list nor on the back list. */
static char *deny_list[] = {
"dofile",
"loadfile",
"print",
NULL,
};
static int redis_math_random (lua_State *L); static int redis_math_random (lua_State *L);
static int redis_math_randomseed (lua_State *L); static int redis_math_randomseed (lua_State *L);
static void redisProtocolToLuaType_Int(void *ctx, long long val, const char *proto, size_t proto_len); static void redisProtocolToLuaType_Int(void *ctx, long long val, const char *proto, size_t proto_len);
...@@ -1113,15 +1204,6 @@ static void luaLoadLibraries(lua_State *lua) { ...@@ -1113,15 +1204,6 @@ static void luaLoadLibraries(lua_State *lua) {
#endif #endif
} }
/* Remove a functions that we don't want to expose to the Redis scripting
* environment. */
static void luaRemoveUnsupportedFunctions(lua_State *lua) {
lua_pushnil(lua);
lua_setglobal(lua,"loadfile");
lua_pushnil(lua);
lua_setglobal(lua,"dofile");
}
/* Return sds of the string value located on stack at the given index. /* Return sds of the string value located on stack at the given index.
* Return NULL if the value is not a string. */ * Return NULL if the value is not a string. */
sds luaGetStringSds(lua_State *lua, int index) { sds luaGetStringSds(lua_State *lua, int index) {
...@@ -1135,107 +1217,120 @@ sds luaGetStringSds(lua_State *lua, int index) { ...@@ -1135,107 +1217,120 @@ sds luaGetStringSds(lua_State *lua, int index) {
return str_sds; return str_sds;
} }
/* This function installs metamethods in the global table _G that prevent static int luaProtectedTableError(lua_State *lua) {
* the creation of globals accidentally. int argc = lua_gettop(lua);
* if (argc != 2) {
* It should be the last to be called in the scripting engine initialization serverLog(LL_WARNING, "malicious code trying to call luaProtectedTableError with wrong arguments");
* sequence, because it may interact with creation of globals. luaL_error(lua, "Wrong number of arguments to luaProtectedTableError");
* }
* On Legacy Lua (eval) we need to check 'w ~= \"main\"' otherwise we will not be able if (!lua_isstring(lua, -1) && !lua_isnumber(lua, -1)) {
* to create the global 'function <sha> ()' variable. On Functions Lua engine we do not use luaL_error(lua, "Second argument to luaProtectedTableError must be a string or number");
* this trick so it's not needed. */ }
void luaEnableGlobalsProtection(lua_State *lua, int is_eval) { const char *variable_name = lua_tostring(lua, -1);
char *s[32]; luaL_error(lua, "Script attempted to access nonexistent global variable '%s'", variable_name);
sds code = sdsempty(); return 0;
int j = 0;
/* strict.lua from: http://metalua.luaforge.net/src/lib/strict.lua.html.
* Modified to be adapted to Redis. */
s[j++]="local dbg=debug\n";
s[j++]="local mt = {}\n";
s[j++]="setmetatable(_G, mt)\n";
s[j++]="mt.__newindex = function (t, n, v)\n";
s[j++]=" if dbg.getinfo(2) then\n";
s[j++]=" local w = dbg.getinfo(2, \"S\").what\n";
s[j++]= is_eval ? " if w ~= \"main\" and w ~= \"C\" then\n" : " if w ~= \"C\" then\n";
s[j++]=" error(\"Script attempted to create global variable '\"..tostring(n)..\"'\", 2)\n";
s[j++]=" end\n";
s[j++]=" end\n";
s[j++]=" rawset(t, n, v)\n";
s[j++]="end\n";
s[j++]="mt.__index = function (t, n)\n";
s[j++]=" if dbg.getinfo(2) and dbg.getinfo(2, \"S\").what ~= \"C\" then\n";
s[j++]=" error(\"Script attempted to access nonexistent global variable '\"..tostring(n)..\"'\", 2)\n";
s[j++]=" end\n";
s[j++]=" return rawget(t, n)\n";
s[j++]="end\n";
s[j++]="debug = nil\n";
s[j++]=NULL;
for (j = 0; s[j] != NULL; j++) code = sdscatlen(code,s[j],strlen(s[j]));
luaL_loadbuffer(lua,code,sdslen(code),"@enable_strict_lua");
lua_pcall(lua,0,0,0);
sdsfree(code);
} }
/* Create a global protection function and put it to registry. /* Set a special metatable on the table on the top of the stack.
* This need to be called once in the lua_State lifetime. * The metatable will raise an error if the user tries to fetch
* After called it is possible to use luaSetGlobalProtection * an un-existing value.
* to set global protection on a give table.
* *
* The function assumes the Lua stack have a least enough * The function assumes the Lua stack have a least enough
* space to push 2 element, its up to the caller to verify * space to push 2 element, its up to the caller to verify
* this before calling this function. * this before calling this function. */
* void luaSetErrorMetatable(lua_State *lua) {
* Notice, the difference between this and luaEnableGlobalsProtection lua_newtable(lua); /* push metatable */
* is that luaEnableGlobalsProtection is enabling global protection lua_pushcfunction(lua, luaProtectedTableError); /* push get error handler */
* on the current Lua globals. This registering a global protection lua_setfield(lua, -2, "__index");
* function that later can be applied on any table. */ lua_setmetatable(lua, -2);
void luaRegisterGlobalProtectionFunction(lua_State *lua) {
lua_pushstring(lua, REGISTRY_SET_GLOBALS_PROTECTION_NAME);
char *global_protection_func = "local dbg = debug\n"
"local globals_protection = function (t)\n"
" local mt = {}\n"
" setmetatable(t, mt)\n"
" mt.__newindex = function (t, n, v)\n"
" if dbg.getinfo(2) then\n"
" local w = dbg.getinfo(2, \"S\").what\n"
" if w ~= \"C\" then\n"
" error(\"Script attempted to create global variable '\"..tostring(n)..\"'\", 2)\n"
" end"
" end"
" rawset(t, n, v)\n"
" end\n"
" mt.__index = function (t, n)\n"
" if dbg.getinfo(2) and dbg.getinfo(2, \"S\").what ~= \"C\" then\n"
" error(\"Script attempted to access nonexistent global variable '\"..tostring(n)..\"'\", 2)\n"
" end\n"
" return rawget(t, n)\n"
" end\n"
"end\n"
"return globals_protection";
int res = luaL_loadbuffer(lua, global_protection_func, strlen(global_protection_func), "@global_protection_def");
serverAssert(res == 0);
res = lua_pcall(lua,0,1,0);
serverAssert(res == 0);
lua_settable(lua, LUA_REGISTRYINDEX);
} }
/* Set global protection on a given table. static int luaNewIndexAllowList(lua_State *lua) {
* The table need to be located on the top of the lua stack. int argc = lua_gettop(lua);
* After called, it will no longer be possible to set if (argc != 3) {
* new items on the table. The function is not removing serverLog(LL_WARNING, "malicious code trying to call luaProtectedTableError with wrong arguments");
* the table from the top of the stack! luaL_error(lua, "Wrong number of arguments to luaNewIndexAllowList");
}
if (!lua_istable(lua, -3)) {
luaL_error(lua, "first argument to luaNewIndexAllowList must be a table");
}
if (!lua_isstring(lua, -2) && !lua_isnumber(lua, -2)) {
luaL_error(lua, "Second argument to luaNewIndexAllowList must be a string or number");
}
const char *variable_name = lua_tostring(lua, -2);
/* check if the key is in our allow list */
char ***allow_l = allow_lists;
for (; *allow_l ; ++allow_l){
char **c = *allow_l;
for (; *c ; ++c) {
if (strcmp(*c, variable_name) == 0) {
break;
}
}
if (*c) {
break;
}
}
if (!*allow_l) {
/* Search the value on the back list, if its there we know that it was removed
* on purpose and there is no need to print a warning. */
char **c = deny_list;
for ( ; *c ; ++c) {
if (strcmp(*c, variable_name) == 0) {
break;
}
}
if (!*c) {
serverLog(LL_WARNING, "A key '%s' was added to Lua globals which is not on the globals allow list nor listed on the deny list.", variable_name);
}
} else {
lua_rawset(lua, -3);
}
return 0;
}
/* Set a metatable with '__newindex' function that verify that
* the new index appears on our globals while list.
* *
* The function assumes the Lua stack have a least enough * The metatable is set on the table which located on the top
* space to push 2 element, its up to the caller to verify * of the stack.
* this before calling this function. */ */
void luaSetGlobalProtection(lua_State *lua) { void luaSetAllowListProtection(lua_State *lua) {
lua_pushstring(lua, REGISTRY_SET_GLOBALS_PROTECTION_NAME); lua_newtable(lua); /* push metatable */
lua_gettable(lua, LUA_REGISTRYINDEX); lua_pushcfunction(lua, luaNewIndexAllowList); /* push get error handler */
lua_pushvalue(lua, -2); lua_setfield(lua, -2, "__newindex");
int res = lua_pcall(lua, 1, 0, 0); lua_setmetatable(lua, -2);
serverAssert(res == 0); }
/* Set the readonly flag on the table located on the top of the stack
* and recursively call this function on each table located on the original
* table. Also, recursively call this function on the metatables.*/
void luaSetTableProtectionRecursively(lua_State *lua) {
/* This protect us from a loop in case we already visited the table
* For example, globals has '_G' key which is pointing back to globals. */
if (lua_isreadonlytable(lua, -1)) {
return;
}
/* protect the current table */
lua_enablereadonlytable(lua, -1, 1);
lua_checkstack(lua, 2);
lua_pushnil(lua); /* Use nil to start iteration. */
while (lua_next(lua,-2)) {
/* Stack now: table, key, value */
if (lua_istable(lua, -1)) {
luaSetTableProtectionRecursively(lua);
}
lua_pop(lua, 1);
}
/* protect the metatable if exists */
if (lua_getmetatable(lua, -1)) {
luaSetTableProtectionRecursively(lua);
lua_pop(lua, 1); /* pop the metatable */
}
} }
void luaRegisterVersion(lua_State* lua) { void luaRegisterVersion(lua_State* lua) {
...@@ -1272,8 +1367,11 @@ void luaRegisterLogFunction(lua_State* lua) { ...@@ -1272,8 +1367,11 @@ void luaRegisterLogFunction(lua_State* lua) {
} }
void luaRegisterRedisAPI(lua_State* lua) { void luaRegisterRedisAPI(lua_State* lua) {
lua_pushvalue(lua, LUA_GLOBALSINDEX);
luaSetAllowListProtection(lua);
lua_pop(lua, 1);
luaLoadLibraries(lua); luaLoadLibraries(lua);
luaRemoveUnsupportedFunctions(lua);
lua_pushcfunction(lua,luaRedisPcall); lua_pushcfunction(lua,luaRedisPcall);
lua_setglobal(lua, "pcall"); lua_setglobal(lua, "pcall");
...@@ -1504,9 +1602,19 @@ void luaCallFunction(scriptRunCtx* run_ctx, lua_State *lua, robj** keys, size_t ...@@ -1504,9 +1602,19 @@ void luaCallFunction(scriptRunCtx* run_ctx, lua_State *lua, robj** keys, size_t
* EVAL received. */ * EVAL received. */
luaCreateArray(lua,keys,nkeys); luaCreateArray(lua,keys,nkeys);
/* On eval, keys and arguments are globals. */ /* On eval, keys and arguments are globals. */
if (run_ctx->flags & SCRIPT_EVAL_MODE) lua_setglobal(lua,"KEYS"); if (run_ctx->flags & SCRIPT_EVAL_MODE){
/* open global protection to set KEYS */
lua_enablereadonlytable(lua, LUA_GLOBALSINDEX, 0);
lua_setglobal(lua,"KEYS");
lua_enablereadonlytable(lua, LUA_GLOBALSINDEX, 1);
}
luaCreateArray(lua,args,nargs); luaCreateArray(lua,args,nargs);
if (run_ctx->flags & SCRIPT_EVAL_MODE) lua_setglobal(lua,"ARGV"); if (run_ctx->flags & SCRIPT_EVAL_MODE){
/* open global protection to set ARGV */
lua_enablereadonlytable(lua, LUA_GLOBALSINDEX, 0);
lua_setglobal(lua,"ARGV");
lua_enablereadonlytable(lua, LUA_GLOBALSINDEX, 1);
}
/* At this point whether this script was never seen before or if it was /* At this point whether this script was never seen before or if it was
* already defined, we can call it. * already defined, we can call it.
......
...@@ -67,9 +67,10 @@ typedef struct errorInfo { ...@@ -67,9 +67,10 @@ typedef struct errorInfo {
void luaRegisterRedisAPI(lua_State* lua); void luaRegisterRedisAPI(lua_State* lua);
sds luaGetStringSds(lua_State *lua, int index); sds luaGetStringSds(lua_State *lua, int index);
void luaEnableGlobalsProtection(lua_State *lua, int is_eval);
void luaRegisterGlobalProtectionFunction(lua_State *lua); void luaRegisterGlobalProtectionFunction(lua_State *lua);
void luaSetGlobalProtection(lua_State *lua); void luaSetErrorMetatable(lua_State *lua);
void luaSetAllowListProtection(lua_State *lua);
void luaSetTableProtectionRecursively(lua_State *lua);
void luaRegisterLogFunction(lua_State* lua); void luaRegisterLogFunction(lua_State* lua);
void luaRegisterVersion(lua_State* lua); void luaRegisterVersion(lua_State* lua);
void luaPushErrorBuff(lua_State *lua, sds err_buff); void luaPushErrorBuff(lua_State *lua, sds err_buff);
......
...@@ -705,7 +705,7 @@ void sentinelEvent(int level, char *type, sentinelRedisInstance *ri, ...@@ -705,7 +705,7 @@ void sentinelEvent(int level, char *type, sentinelRedisInstance *ri,
if (level != LL_DEBUG) { if (level != LL_DEBUG) {
channel = createStringObject(type,strlen(type)); channel = createStringObject(type,strlen(type));
payload = createStringObject(msg,strlen(msg)); payload = createStringObject(msg,strlen(msg));
pubsubPublishMessage(channel,payload); pubsubPublishMessage(channel,payload,0);
decrRefCount(channel); decrRefCount(channel);
decrRefCount(payload); decrRefCount(payload);
} }
......
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
...@@ -38,7 +38,7 @@ int getGenericCommand(client *c); ...@@ -38,7 +38,7 @@ int getGenericCommand(client *c);
*----------------------------------------------------------------------------*/ *----------------------------------------------------------------------------*/
static int checkStringLength(client *c, long long size) { static int checkStringLength(client *c, long long size) {
if (!(c->flags & CLIENT_MASTER) && size > server.proto_max_bulk_len) { if (!mustObeyClient(c) && size > server.proto_max_bulk_len) {
addReplyError(c,"string exceeds maximum allowed size (proto-max-bulk-len)"); addReplyError(c,"string exceeds maximum allowed size (proto-max-bulk-len)");
return C_ERR; return C_ERR;
} }
...@@ -792,7 +792,7 @@ void lcsCommand(client *c) { ...@@ -792,7 +792,7 @@ void lcsCommand(client *c) {
/* Setup an uint32_t array to store at LCS[i,j] the length of the /* Setup an uint32_t array to store at LCS[i,j] the length of the
* LCS A0..i-1, B0..j-1. Note that we have a linear array here, so * LCS A0..i-1, B0..j-1. Note that we have a linear array here, so
* we index it as LCS[j+(blen+1)*j] */ * we index it as LCS[j+(blen+1)*i] */
#define LCS(A,B) lcs[(B)+((A)*(blen+1))] #define LCS(A,B) lcs[(B)+((A)*(blen+1))]
/* Try to allocate the LCS table, and abort on overflow or insufficient memory. */ /* Try to allocate the LCS table, and abort on overflow or insufficient memory. */
......
This diff is collapsed.
...@@ -58,7 +58,7 @@ int clientsCronHandleTimeout(client *c, mstime_t now_ms) { ...@@ -58,7 +58,7 @@ int clientsCronHandleTimeout(client *c, mstime_t now_ms) {
if (server.maxidletime && if (server.maxidletime &&
/* This handles the idle clients connection timeout if set. */ /* This handles the idle clients connection timeout if set. */
!(c->flags & CLIENT_SLAVE) && /* No timeout for slaves and monitors */ !(c->flags & CLIENT_SLAVE) && /* No timeout for slaves and monitors */
!(c->flags & CLIENT_MASTER) && /* No timeout for masters */ !mustObeyClient(c) && /* No timeout for masters and AOF */
!(c->flags & CLIENT_BLOCKED) && /* No timeout for BLPOP */ !(c->flags & CLIENT_BLOCKED) && /* No timeout for BLPOP */
!(c->flags & CLIENT_PUBSUB) && /* No timeout for Pub/Sub clients */ !(c->flags & CLIENT_PUBSUB) && /* No timeout for Pub/Sub clients */
(now - c->lastinteraction > server.maxidletime)) (now - c->lastinteraction > server.maxidletime))
......
This diff is collapsed.
...@@ -75,6 +75,7 @@ int string2d(const char *s, size_t slen, double *dp); ...@@ -75,6 +75,7 @@ int string2d(const char *s, size_t slen, double *dp);
int trimDoubleString(char *buf, size_t len); int trimDoubleString(char *buf, size_t len);
int d2string(char *buf, size_t len, double value); int d2string(char *buf, size_t len, double value);
int ld2string(char *buf, size_t len, long double value, ld2string_mode mode); int ld2string(char *buf, size_t len, long double value, ld2string_mode mode);
int double2ll(double d, long long *out);
int yesnotoi(char *s); int yesnotoi(char *s);
sds getAbsolutePath(char *filename); sds getAbsolutePath(char *filename);
long getTimeZone(void); long getTimeZone(void);
......
#define REDIS_VERSION "6.9.242" #define REDIS_VERSION "7.0.0"
#define REDIS_VERSION_NUM 0x000609f2 #define REDIS_VERSION_NUM 0x00070000
...@@ -30,3 +30,5 @@ activerehashing yes ...@@ -30,3 +30,5 @@ activerehashing yes
enable-protected-configs yes enable-protected-configs yes
enable-debug-command yes enable-debug-command yes
enable-module-command yes enable-module-command yes
propagation-error-behavior panic
\ No newline at end of file
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment