Commit 941cdc27 authored by Oran Agra's avatar Oran Agra
Browse files

Merge branch 'unstable' into 7.2 to prepare for 7.2.0 GA

parents 1a589818 f4549d1c
{ {
"GEOHASH": { "GEOHASH": {
"summary": "Returns members from a geospatial index as geohash strings.", "summary": "Returns members from a geospatial index as geohash strings.",
"complexity": "O(log(N)) for each member requested, where N is the number of elements in the sorted set.", "complexity": "O(1) for each member requested.",
"group": "geo", "group": "geo",
"since": "3.2.0", "since": "3.2.0",
"arity": -2, "arity": -2,
......
{ {
"GEOPOS": { "GEOPOS": {
"summary": "Returns the longitude and latitude of members from a geospatial index.", "summary": "Returns the longitude and latitude of members from a geospatial index.",
"complexity": "O(N) where N is the number of members requested.", "complexity": "O(1) for each member requested.",
"group": "geo", "group": "geo",
"since": "3.2.0", "since": "3.2.0",
"arity": -2, "arity": -2,
......
...@@ -13,10 +13,6 @@ ...@@ -13,10 +13,6 @@
"acl_categories": [ "acl_categories": [
"STRING" "STRING"
], ],
"command_tips": [
"REQUEST_POLICY:MULTI_SHARD",
"RESPONSE_POLICY:AGG_MIN"
],
"key_specs": [ "key_specs": [
{ {
"flags": [ "flags": [
......
...@@ -15,6 +15,7 @@ ...@@ -15,6 +15,7 @@
], ],
"command_tips": [ "command_tips": [
"REQUEST_POLICY:ALL_SHARDS", "REQUEST_POLICY:ALL_SHARDS",
"RESPONSE_POLICY:SPECIAL",
"NONDETERMINISTIC_OUTPUT" "NONDETERMINISTIC_OUTPUT"
], ],
"reply_schema": { "reply_schema": {
......
...@@ -21,7 +21,8 @@ ...@@ -21,7 +21,8 @@
], ],
"command_tips": [ "command_tips": [
"NONDETERMINISTIC_OUTPUT", "NONDETERMINISTIC_OUTPUT",
"REQUEST_POLICY:SPECIAL" "REQUEST_POLICY:SPECIAL",
"RESPONSE_POLICY:SPECIAL"
], ],
"arguments": [ "arguments": [
{ {
......
...@@ -51,7 +51,6 @@ ...@@ -51,7 +51,6 @@
#define REGISTRY_LOAD_CTX_NAME "__LIBRARY_CTX__" #define REGISTRY_LOAD_CTX_NAME "__LIBRARY_CTX__"
#define LIBRARY_API_NAME "__LIBRARY_API__" #define LIBRARY_API_NAME "__LIBRARY_API__"
#define GLOBALS_API_NAME "__GLOBALS_API__" #define GLOBALS_API_NAME "__GLOBALS_API__"
#define LOAD_TIMEOUT_MS 500
/* Lua engine ctx */ /* Lua engine ctx */
typedef struct luaEngineCtx { typedef struct luaEngineCtx {
...@@ -67,6 +66,7 @@ typedef struct luaFunctionCtx { ...@@ -67,6 +66,7 @@ typedef struct luaFunctionCtx {
typedef struct loadCtx { typedef struct loadCtx {
functionLibInfo *li; functionLibInfo *li;
monotime start_time; monotime start_time;
size_t timeout;
} loadCtx; } loadCtx;
typedef struct registerFunctionArgs { typedef struct registerFunctionArgs {
...@@ -85,7 +85,7 @@ static void luaEngineLoadHook(lua_State *lua, lua_Debug *ar) { ...@@ -85,7 +85,7 @@ static void luaEngineLoadHook(lua_State *lua, lua_Debug *ar) {
loadCtx *load_ctx = luaGetFromRegistry(lua, REGISTRY_LOAD_CTX_NAME); loadCtx *load_ctx = luaGetFromRegistry(lua, REGISTRY_LOAD_CTX_NAME);
serverAssert(load_ctx); /* Only supported inside script invocation */ serverAssert(load_ctx); /* Only supported inside script invocation */
uint64_t duration = elapsedMs(load_ctx->start_time); uint64_t duration = elapsedMs(load_ctx->start_time);
if (duration > LOAD_TIMEOUT_MS) { if (load_ctx->timeout > 0 && duration > load_ctx->timeout) {
lua_sethook(lua, luaEngineLoadHook, LUA_MASKLINE, 0); lua_sethook(lua, luaEngineLoadHook, LUA_MASKLINE, 0);
luaPushError(lua,"FUNCTION LOAD timeout"); luaPushError(lua,"FUNCTION LOAD timeout");
...@@ -100,7 +100,7 @@ static void luaEngineLoadHook(lua_State *lua, lua_Debug *ar) { ...@@ -100,7 +100,7 @@ static void luaEngineLoadHook(lua_State *lua, lua_Debug *ar) {
* *
* Return NULL on compilation error and set the error to the err variable * Return NULL on compilation error and set the error to the err variable
*/ */
static int luaEngineCreate(void *engine_ctx, functionLibInfo *li, sds blob, sds *err) { static int luaEngineCreate(void *engine_ctx, functionLibInfo *li, sds blob, size_t timeout, sds *err) {
int ret = C_ERR; int ret = C_ERR;
luaEngineCtx *lua_engine_ctx = engine_ctx; luaEngineCtx *lua_engine_ctx = engine_ctx;
lua_State *lua = lua_engine_ctx->lua; lua_State *lua = lua_engine_ctx->lua;
...@@ -124,6 +124,7 @@ static int luaEngineCreate(void *engine_ctx, functionLibInfo *li, sds blob, sds ...@@ -124,6 +124,7 @@ static int luaEngineCreate(void *engine_ctx, functionLibInfo *li, sds blob, sds
loadCtx load_ctx = { loadCtx load_ctx = {
.li = li, .li = li,
.start_time = getMonotonicUs(), .start_time = getMonotonicUs(),
.timeout = timeout,
}; };
luaSaveOnRegistry(lua, REGISTRY_LOAD_CTX_NAME, &load_ctx); luaSaveOnRegistry(lua, REGISTRY_LOAD_CTX_NAME, &load_ctx);
......
...@@ -33,6 +33,8 @@ ...@@ -33,6 +33,8 @@
#include "adlist.h" #include "adlist.h"
#include "atomicvar.h" #include "atomicvar.h"
#define LOAD_TIMEOUT_MS 500
typedef enum { typedef enum {
restorePolicy_Flush, restorePolicy_Append, restorePolicy_Replace restorePolicy_Flush, restorePolicy_Append, restorePolicy_Replace
} restorePolicy; } restorePolicy;
...@@ -961,7 +963,7 @@ void functionFreeLibMetaData(functionsLibMataData *md) { ...@@ -961,7 +963,7 @@ void functionFreeLibMetaData(functionsLibMataData *md) {
/* Compile and save the given library, return the loaded library name on success /* Compile and save the given library, return the loaded library name on success
* and NULL on failure. In case on failure the err out param is set with relevant error message */ * and NULL on failure. In case on failure the err out param is set with relevant error message */
sds functionsCreateWithLibraryCtx(sds code, int replace, sds* err, functionsLibCtx *lib_ctx) { sds functionsCreateWithLibraryCtx(sds code, int replace, sds* err, functionsLibCtx *lib_ctx, size_t timeout) {
dictIterator *iter = NULL; dictIterator *iter = NULL;
dictEntry *entry = NULL; dictEntry *entry = NULL;
functionLibInfo *new_li = NULL; functionLibInfo *new_li = NULL;
...@@ -995,7 +997,7 @@ sds functionsCreateWithLibraryCtx(sds code, int replace, sds* err, functionsLibC ...@@ -995,7 +997,7 @@ sds functionsCreateWithLibraryCtx(sds code, int replace, sds* err, functionsLibC
} }
new_li = engineLibraryCreate(md.name, ei, code); new_li = engineLibraryCreate(md.name, ei, code);
if (engine->create(engine->engine_ctx, new_li, md.code, err) != C_OK) { if (engine->create(engine->engine_ctx, new_li, md.code, timeout, err) != C_OK) {
goto error; goto error;
} }
...@@ -1063,7 +1065,11 @@ void functionLoadCommand(client *c) { ...@@ -1063,7 +1065,11 @@ void functionLoadCommand(client *c) {
robj *code = c->argv[argc_pos]; robj *code = c->argv[argc_pos];
sds err = NULL; sds err = NULL;
sds library_name = NULL; sds library_name = NULL;
if (!(library_name = functionsCreateWithLibraryCtx(code->ptr, replace, &err, curr_functions_lib_ctx))) size_t timeout = LOAD_TIMEOUT_MS;
if (mustObeyClient(c)) {
timeout = 0;
}
if (!(library_name = functionsCreateWithLibraryCtx(code->ptr, replace, &err, curr_functions_lib_ctx, timeout)))
{ {
addReplyErrorSds(c, err); addReplyErrorSds(c, err);
return; return;
......
...@@ -53,9 +53,14 @@ typedef struct engine { ...@@ -53,9 +53,14 @@ typedef struct engine {
/* engine specific context */ /* engine specific context */
void *engine_ctx; void *engine_ctx;
/* Create function callback, get the engine_ctx, and function code. /* Create function callback, get the engine_ctx, and function code
* engine_ctx - opaque struct that was created on engine initialization
* li - library information that need to be provided and when add functions
* code - the library code
* timeout - timeout for the library creation (0 for no timeout)
* err - description of error (if occurred)
* returns NULL on error and set sds to be the error message */ * returns NULL on error and set sds to be the error message */
int (*create)(void *engine_ctx, functionLibInfo *li, sds code, sds *err); int (*create)(void *engine_ctx, functionLibInfo *li, sds code, size_t timeout, sds *err);
/* Invoking a function, r_ctx is an opaque object (from engine POV). /* Invoking a function, r_ctx is an opaque object (from engine POV).
* The r_ctx should be used by the engine to interaction with Redis, * The r_ctx should be used by the engine to interaction with Redis,
...@@ -109,7 +114,7 @@ struct functionLibInfo { ...@@ -109,7 +114,7 @@ struct functionLibInfo {
}; };
int functionsRegisterEngine(const char *engine_name, engine *engine_ctx); int functionsRegisterEngine(const char *engine_name, engine *engine_ctx);
sds functionsCreateWithLibraryCtx(sds code, int replace, sds* err, functionsLibCtx *lib_ctx); sds functionsCreateWithLibraryCtx(sds code, int replace, sds* err, functionsLibCtx *lib_ctx, size_t timeout);
unsigned long functionsMemory(void); unsigned long functionsMemory(void);
unsigned long functionsMemoryOverhead(void); unsigned long functionsMemoryOverhead(void);
unsigned long functionsNum(void); unsigned long functionsNum(void);
......
...@@ -3808,7 +3808,7 @@ size_t getClientMemoryUsage(client *c, size_t *output_buffer_mem_usage) { ...@@ -3808,7 +3808,7 @@ size_t getClientMemoryUsage(client *c, size_t *output_buffer_mem_usage) {
* classes of clients. * classes of clients.
* *
* The function will return one of the following: * The function will return one of the following:
* CLIENT_TYPE_NORMAL -> Normal client * CLIENT_TYPE_NORMAL -> Normal client, including MONITOR
* CLIENT_TYPE_SLAVE -> Slave * CLIENT_TYPE_SLAVE -> Slave
* CLIENT_TYPE_PUBSUB -> Client subscribed to Pub/Sub channels * CLIENT_TYPE_PUBSUB -> Client subscribed to Pub/Sub channels
* CLIENT_TYPE_MASTER -> The client representing our replication master. * CLIENT_TYPE_MASTER -> The client representing our replication master.
......
...@@ -2981,7 +2981,7 @@ int rdbFunctionLoad(rio *rdb, int ver, functionsLibCtx* lib_ctx, int rdbflags, s ...@@ -2981,7 +2981,7 @@ int rdbFunctionLoad(rio *rdb, int ver, functionsLibCtx* lib_ctx, int rdbflags, s
if (lib_ctx) { if (lib_ctx) {
sds library_name = NULL; sds library_name = NULL;
if (!(library_name = functionsCreateWithLibraryCtx(final_payload, rdbflags & RDBFLAGS_ALLOW_DUP, &error, lib_ctx))) { if (!(library_name = functionsCreateWithLibraryCtx(final_payload, rdbflags & RDBFLAGS_ALLOW_DUP, &error, lib_ctx, 0))) {
if (!error) { if (!error) {
error = sdsnew("Failed creating the library"); error = sdsnew("Failed creating the library");
} }
......
...@@ -2291,8 +2291,12 @@ static int cliReadReply(int output_raw_strings) { ...@@ -2291,8 +2291,12 @@ static int cliReadReply(int output_raw_strings) {
slot = atoi(s+1); slot = atoi(s+1);
s = strrchr(p+1,':'); /* MOVED 3999[P]127.0.0.1[S]6381 */ s = strrchr(p+1,':'); /* MOVED 3999[P]127.0.0.1[S]6381 */
*s = '\0'; *s = '\0';
sdsfree(config.conn_info.hostip); if (p+1 != s) {
config.conn_info.hostip = sdsnew(p+1); /* Host might be empty, like 'MOVED 3999 :6381', if endpoint type is unknown. Only update the
* host if it's non-empty. */
sdsfree(config.conn_info.hostip);
config.conn_info.hostip = sdsnew(p+1);
}
config.conn_info.hostport = atoi(s+1); config.conn_info.hostport = atoi(s+1);
if (config.interactive) if (config.interactive)
printf("-> Redirected to slot [%d] located at %s:%d\n", printf("-> Redirected to slot [%d] located at %s:%d\n",
......
...@@ -610,6 +610,7 @@ void replicationFeedMonitors(client *c, list *monitors, int dictid, robj **argv, ...@@ -610,6 +610,7 @@ void replicationFeedMonitors(client *c, list *monitors, int dictid, robj **argv,
while((ln = listNext(&li))) { while((ln = listNext(&li))) {
client *monitor = ln->value; client *monitor = ln->value;
addReply(monitor,cmdobj); addReply(monitor,cmdobj);
updateClientMemUsageAndBucket(monitor);
} }
decrRefCount(cmdobj); decrRefCount(cmdobj);
} }
......
...@@ -917,11 +917,9 @@ void removeClientFromMemUsageBucket(client *c, int allow_eviction) { ...@@ -917,11 +917,9 @@ void removeClientFromMemUsageBucket(client *c, int allow_eviction) {
* together clients consuming about the same amount of memory and can quickly * together clients consuming about the same amount of memory and can quickly
* free them in case we reach maxmemory-clients (client eviction). * free them in case we reach maxmemory-clients (client eviction).
* *
* Note: This function filters clients of type monitor, master or replica regardless * Note: This function filters clients of type no-evict, master or replica regardless
* of whether the eviction is enabled or not, so the memory usage we get from these * of whether the eviction is enabled or not, so the memory usage we get from these
* types of clients via the INFO command may be out of date. If someday we wanna * types of clients via the INFO command may be out of date.
* improve that to make monitors' memory usage more accurate, we need to re-add this
* function call to `replicationFeedMonitors()`.
* *
* returns 1 if client eviction for this client is allowed, 0 otherwise. * returns 1 if client eviction for this client is allowed, 0 otherwise.
*/ */
...@@ -4160,7 +4158,7 @@ int processCommand(client *c) { ...@@ -4160,7 +4158,7 @@ int processCommand(client *c) {
int flags = CMD_CALL_FULL; int flags = CMD_CALL_FULL;
if (client_reprocessing_command) flags |= CMD_CALL_REPROCESSING; if (client_reprocessing_command) flags |= CMD_CALL_REPROCESSING;
call(c,flags); call(c,flags);
if (listLength(server.ready_keys)) if (listLength(server.ready_keys) && !isInsideYieldingLongCommand())
handleClientsBlockedOnKeys(); handleClientsBlockedOnKeys();
} }
......
...@@ -195,8 +195,16 @@ test "New master [join $addr {:}] role matches" { ...@@ -195,8 +195,16 @@ test "New master [join $addr {:}] role matches" {
} }
test "SENTINEL RESET can resets the master" { test "SENTINEL RESET can resets the master" {
assert_equal 1 [S 0 SENTINEL RESET mymaster] # After SENTINEL RESET, sometimes the sentinel can sense the master again,
assert_equal 0 [llength [S 0 SENTINEL SENTINELS mymaster]] # causing the test to fail. Here we give it a few more chances.
assert_equal 0 [llength [S 0 SENTINEL SLAVES mymaster]] for {set j 0} {$j < 10} {incr j} {
assert_equal 0 [llength [S 0 SENTINEL REPLICAS mymaster]] assert_equal 1 [S 0 SENTINEL RESET mymaster]
set res1 [llength [S 0 SENTINEL SENTINELS mymaster]]
set res2 [llength [S 0 SENTINEL SLAVES mymaster]]
set res3 [llength [S 0 SENTINEL REPLICAS mymaster]]
if {$res1 eq 0 && $res2 eq 0 && $res3 eq 0} break
}
assert_equal 0 $res1
assert_equal 0 $res2
assert_equal 0 $res3
} }
...@@ -62,8 +62,8 @@ proc sanitizer_errors_from_file {filename} { ...@@ -62,8 +62,8 @@ proc sanitizer_errors_from_file {filename} {
} }
# GCC UBSAN output does not contain 'Sanitizer' but 'runtime error'. # GCC UBSAN output does not contain 'Sanitizer' but 'runtime error'.
if {[string match {*runtime error*} $log] || if {[string match {*runtime error*} $line] ||
[string match {*Sanitizer*} $log]} { [string match {*Sanitizer*} $line]} {
return $log return $log
} }
} }
......
...@@ -47,6 +47,15 @@ start_server {tags {"acl external:skip"}} { ...@@ -47,6 +47,15 @@ start_server {tags {"acl external:skip"}} {
catch {r ACL SETUSER selector-syntax on (&* &fail)} e catch {r ACL SETUSER selector-syntax on (&* &fail)} e
assert_match "*ERR Error in ACL SETUSER modifier '(*)*Adding a pattern after the*" $e assert_match "*ERR Error in ACL SETUSER modifier '(*)*Adding a pattern after the*" $e
catch {r ACL SETUSER selector-syntax on (+PING (+SELECT (+DEL} e
assert_match "*ERR Unmatched parenthesis in acl selector*" $e
catch {r ACL SETUSER selector-syntax on (+PING (+SELECT (+DEL ) ) ) } e
assert_match "*ERR Error in ACL SETUSER modifier*" $e
catch {r ACL SETUSER selector-syntax on (+PING (+SELECT (+DEL ) } e
assert_match "*ERR Error in ACL SETUSER modifier*" $e
assert_equal "" [r ACL GETUSER selector-syntax] assert_equal "" [r ACL GETUSER selector-syntax]
} }
......
...@@ -615,6 +615,10 @@ start_server {tags {"acl external:skip"}} { ...@@ -615,6 +615,10 @@ start_server {tags {"acl external:skip"}} {
# Unnecessary categories are retained for potentional future compatibility # Unnecessary categories are retained for potentional future compatibility
r ACL SETUSER adv-test -@all -@dangerous r ACL SETUSER adv-test -@all -@dangerous
assert_equal "-@all -@dangerous" [dict get [r ACL getuser adv-test] commands] assert_equal "-@all -@dangerous" [dict get [r ACL getuser adv-test] commands]
# Duplicate categories are compressed, regression test for #12470
r ACL SETUSER adv-test -@all +config +config|get -config|set +config
assert_equal "-@all +config" [dict get [r ACL getuser adv-test] commands]
} }
test "ACL CAT with illegal arguments" { test "ACL CAT with illegal arguments" {
......
...@@ -70,8 +70,8 @@ start_server {tags {"auth_binary_password external:skip"}} { ...@@ -70,8 +70,8 @@ start_server {tags {"auth_binary_password external:skip"}} {
# Configure the replica with masterauth # Configure the replica with masterauth
set loglines [count_log_lines 0] set loglines [count_log_lines 0]
$slave slaveof $master_host $master_port
$slave config set masterauth "abc" $slave config set masterauth "abc"
$slave slaveof $master_host $master_port
# Verify replica is not able to sync with master # Verify replica is not able to sync with master
wait_for_log_messages 0 {"*Unable to AUTH to MASTER*"} $loglines 1000 10 wait_for_log_messages 0 {"*Unable to AUTH to MASTER*"} $loglines 1000 10
......
...@@ -81,6 +81,23 @@ start_multiple_servers 3 [list overrides $base_conf] { ...@@ -81,6 +81,23 @@ start_multiple_servers 3 [list overrides $base_conf] {
set node1_rd [redis_deferring_client 0] set node1_rd [redis_deferring_client 0]
test "use previous hostip in \"cluster-preferred-endpoint-type unknown-endpoint\" mode" {
# backup and set cluster-preferred-endpoint-type unknown-endpoint
set endpoint_type_before_set [lindex [split [$node1 CONFIG GET cluster-preferred-endpoint-type] " "] 1]
$node1 CONFIG SET cluster-preferred-endpoint-type unknown-endpoint
# when redis-cli not in cluster mode, return MOVE with empty host
set slot_for_foo [$node1 CLUSTER KEYSLOT foo]
assert_error "*MOVED $slot_for_foo :*" {$node1 set foo bar}
# when in cluster mode, redirect using previous hostip
assert_equal "[exec src/redis-cli -h 127.0.0.1 -p [srv 0 port] -c set foo bar]" {OK}
assert_match "[exec src/redis-cli -h 127.0.0.1 -p [srv 0 port] -c get foo]" {bar}
assert_equal [$node1 CONFIG SET cluster-preferred-endpoint-type "$endpoint_type_before_set"] {OK}
}
test "Sanity test push cmd after resharding" { test "Sanity test push cmd after resharding" {
assert_error {*MOVED*} {$node3 lpush key9184688 v1} assert_error {*MOVED*} {$node3 lpush key9184688 v1}
......
...@@ -307,6 +307,13 @@ start_server {tags {"scripting"}} { ...@@ -307,6 +307,13 @@ start_server {tags {"scripting"}} {
set e set e
} {*against a key*} } {*against a key*}
test {EVAL - JSON string encoding a string larger than 2GB} {
run_script {
local s = string.rep("a", 1024 * 1024 * 1024)
return #cjson.encode(s..s..s)
} 0
} {3221225474} {large-memory} ;# length includes two double quotes at both ends
test {EVAL - JSON numeric decoding} { test {EVAL - JSON numeric decoding} {
# We must return the table as a string because otherwise # We must return the table as a string because otherwise
# Redis converts floats to ints and we get 0 and 1023 instead # Redis converts floats to ints and we get 0 and 1023 instead
...@@ -1112,6 +1119,53 @@ start_server {tags {"scripting"}} { ...@@ -1112,6 +1119,53 @@ start_server {tags {"scripting"}} {
r ping r ping
} {PONG} } {PONG}
test {Timedout scripts and unblocked command} {
# make sure a command that's allowed during BUSY doesn't trigger an unblocked command
# enable AOF to also expose an assertion if the bug would happen
r flushall
r config set appendonly yes
# create clients, and set one to block waiting for key 'x'
set rd [redis_deferring_client]
set rd2 [redis_deferring_client]
set r3 [redis_client]
$rd2 blpop x 0
wait_for_blocked_clients_count 1
# hack: allow the script to use client list command so that we can control when it aborts
r DEBUG set-disable-deny-scripts 1
r config set lua-time-limit 10
run_script_on_connection $rd {
local clients
redis.call('lpush',KEYS[1],'y');
while true do
clients = redis.call('client','list')
if string.find(clients, 'abortscript') ~= nil then break end
end
redis.call('lpush',KEYS[1],'z');
return clients
} 1 x
# wait for the script to be busy
after 200
catch {r ping} e
assert_match {BUSY*} $e
# run cause the script to abort, and run a command that could have processed
# unblocked clients (due to a bug)
$r3 hello 2 setname abortscript
# make sure the script completed before the pop was processed
assert_equal [$rd2 read] {x z}
assert_match {*abortscript*} [$rd read]
$rd close
$rd2 close
$r3 close
r DEBUG set-disable-deny-scripts 0
} {OK} {external:skip needs:debug}
test {Timedout scripts that modified data can't be killed by SCRIPT KILL} { test {Timedout scripts that modified data can't be killed by SCRIPT KILL} {
set rd [redis_deferring_client] set rd [redis_deferring_client]
r config set lua-time-limit 10 r config set lua-time-limit 10
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment