Commit 941cdc27 authored by Oran Agra's avatar Oran Agra
Browse files

Merge branch 'unstable' into 7.2 to prepare for 7.2.0 GA

parents 1a589818 f4549d1c
{
"GEOHASH": {
"summary": "Returns members from a geospatial index as geohash strings.",
"complexity": "O(log(N)) for each member requested, where N is the number of elements in the sorted set.",
"complexity": "O(1) for each member requested.",
"group": "geo",
"since": "3.2.0",
"arity": -2,
......
{
"GEOPOS": {
"summary": "Returns the longitude and latitude of members from a geospatial index.",
"complexity": "O(N) where N is the number of members requested.",
"complexity": "O(1) for each member requested.",
"group": "geo",
"since": "3.2.0",
"arity": -2,
......
......@@ -13,10 +13,6 @@
"acl_categories": [
"STRING"
],
"command_tips": [
"REQUEST_POLICY:MULTI_SHARD",
"RESPONSE_POLICY:AGG_MIN"
],
"key_specs": [
{
"flags": [
......
......@@ -15,6 +15,7 @@
],
"command_tips": [
"REQUEST_POLICY:ALL_SHARDS",
"RESPONSE_POLICY:SPECIAL",
"NONDETERMINISTIC_OUTPUT"
],
"reply_schema": {
......
......@@ -21,7 +21,8 @@
],
"command_tips": [
"NONDETERMINISTIC_OUTPUT",
"REQUEST_POLICY:SPECIAL"
"REQUEST_POLICY:SPECIAL",
"RESPONSE_POLICY:SPECIAL"
],
"arguments": [
{
......
......@@ -51,7 +51,6 @@
#define REGISTRY_LOAD_CTX_NAME "__LIBRARY_CTX__"
#define LIBRARY_API_NAME "__LIBRARY_API__"
#define GLOBALS_API_NAME "__GLOBALS_API__"
#define LOAD_TIMEOUT_MS 500
/* Lua engine ctx */
typedef struct luaEngineCtx {
......@@ -67,6 +66,7 @@ typedef struct luaFunctionCtx {
typedef struct loadCtx {
functionLibInfo *li;
monotime start_time;
size_t timeout;
} loadCtx;
typedef struct registerFunctionArgs {
......@@ -85,7 +85,7 @@ static void luaEngineLoadHook(lua_State *lua, lua_Debug *ar) {
loadCtx *load_ctx = luaGetFromRegistry(lua, REGISTRY_LOAD_CTX_NAME);
serverAssert(load_ctx); /* Only supported inside script invocation */
uint64_t duration = elapsedMs(load_ctx->start_time);
if (duration > LOAD_TIMEOUT_MS) {
if (load_ctx->timeout > 0 && duration > load_ctx->timeout) {
lua_sethook(lua, luaEngineLoadHook, LUA_MASKLINE, 0);
luaPushError(lua,"FUNCTION LOAD timeout");
......@@ -100,7 +100,7 @@ static void luaEngineLoadHook(lua_State *lua, lua_Debug *ar) {
*
* Return NULL on compilation error and set the error to the err variable
*/
static int luaEngineCreate(void *engine_ctx, functionLibInfo *li, sds blob, sds *err) {
static int luaEngineCreate(void *engine_ctx, functionLibInfo *li, sds blob, size_t timeout, sds *err) {
int ret = C_ERR;
luaEngineCtx *lua_engine_ctx = engine_ctx;
lua_State *lua = lua_engine_ctx->lua;
......@@ -124,6 +124,7 @@ static int luaEngineCreate(void *engine_ctx, functionLibInfo *li, sds blob, sds
loadCtx load_ctx = {
.li = li,
.start_time = getMonotonicUs(),
.timeout = timeout,
};
luaSaveOnRegistry(lua, REGISTRY_LOAD_CTX_NAME, &load_ctx);
......
......@@ -33,6 +33,8 @@
#include "adlist.h"
#include "atomicvar.h"
#define LOAD_TIMEOUT_MS 500
typedef enum {
restorePolicy_Flush, restorePolicy_Append, restorePolicy_Replace
} restorePolicy;
......@@ -961,7 +963,7 @@ void functionFreeLibMetaData(functionsLibMataData *md) {
/* Compile and save the given library, return the loaded library name on success
* and NULL on failure. In case on failure the err out param is set with relevant error message */
sds functionsCreateWithLibraryCtx(sds code, int replace, sds* err, functionsLibCtx *lib_ctx) {
sds functionsCreateWithLibraryCtx(sds code, int replace, sds* err, functionsLibCtx *lib_ctx, size_t timeout) {
dictIterator *iter = NULL;
dictEntry *entry = NULL;
functionLibInfo *new_li = NULL;
......@@ -995,7 +997,7 @@ sds functionsCreateWithLibraryCtx(sds code, int replace, sds* err, functionsLibC
}
new_li = engineLibraryCreate(md.name, ei, code);
if (engine->create(engine->engine_ctx, new_li, md.code, err) != C_OK) {
if (engine->create(engine->engine_ctx, new_li, md.code, timeout, err) != C_OK) {
goto error;
}
......@@ -1063,7 +1065,11 @@ void functionLoadCommand(client *c) {
robj *code = c->argv[argc_pos];
sds err = NULL;
sds library_name = NULL;
if (!(library_name = functionsCreateWithLibraryCtx(code->ptr, replace, &err, curr_functions_lib_ctx)))
size_t timeout = LOAD_TIMEOUT_MS;
if (mustObeyClient(c)) {
timeout = 0;
}
if (!(library_name = functionsCreateWithLibraryCtx(code->ptr, replace, &err, curr_functions_lib_ctx, timeout)))
{
addReplyErrorSds(c, err);
return;
......
......@@ -53,9 +53,14 @@ typedef struct engine {
/* engine specific context */
void *engine_ctx;
/* Create function callback, get the engine_ctx, and function code.
/* Create function callback, get the engine_ctx, and function code
* engine_ctx - opaque struct that was created on engine initialization
* li - library information that need to be provided and when add functions
* code - the library code
* timeout - timeout for the library creation (0 for no timeout)
* err - description of error (if occurred)
* returns NULL on error and set sds to be the error message */
int (*create)(void *engine_ctx, functionLibInfo *li, sds code, sds *err);
int (*create)(void *engine_ctx, functionLibInfo *li, sds code, size_t timeout, sds *err);
/* Invoking a function, r_ctx is an opaque object (from engine POV).
* The r_ctx should be used by the engine to interaction with Redis,
......@@ -109,7 +114,7 @@ struct functionLibInfo {
};
int functionsRegisterEngine(const char *engine_name, engine *engine_ctx);
sds functionsCreateWithLibraryCtx(sds code, int replace, sds* err, functionsLibCtx *lib_ctx);
sds functionsCreateWithLibraryCtx(sds code, int replace, sds* err, functionsLibCtx *lib_ctx, size_t timeout);
unsigned long functionsMemory(void);
unsigned long functionsMemoryOverhead(void);
unsigned long functionsNum(void);
......
......@@ -3808,7 +3808,7 @@ size_t getClientMemoryUsage(client *c, size_t *output_buffer_mem_usage) {
* classes of clients.
*
* The function will return one of the following:
* CLIENT_TYPE_NORMAL -> Normal client
* CLIENT_TYPE_NORMAL -> Normal client, including MONITOR
* CLIENT_TYPE_SLAVE -> Slave
* CLIENT_TYPE_PUBSUB -> Client subscribed to Pub/Sub channels
* CLIENT_TYPE_MASTER -> The client representing our replication master.
......
......@@ -2981,7 +2981,7 @@ int rdbFunctionLoad(rio *rdb, int ver, functionsLibCtx* lib_ctx, int rdbflags, s
if (lib_ctx) {
sds library_name = NULL;
if (!(library_name = functionsCreateWithLibraryCtx(final_payload, rdbflags & RDBFLAGS_ALLOW_DUP, &error, lib_ctx))) {
if (!(library_name = functionsCreateWithLibraryCtx(final_payload, rdbflags & RDBFLAGS_ALLOW_DUP, &error, lib_ctx, 0))) {
if (!error) {
error = sdsnew("Failed creating the library");
}
......
......@@ -2291,8 +2291,12 @@ static int cliReadReply(int output_raw_strings) {
slot = atoi(s+1);
s = strrchr(p+1,':'); /* MOVED 3999[P]127.0.0.1[S]6381 */
*s = '\0';
if (p+1 != s) {
/* Host might be empty, like 'MOVED 3999 :6381', if endpoint type is unknown. Only update the
* host if it's non-empty. */
sdsfree(config.conn_info.hostip);
config.conn_info.hostip = sdsnew(p+1);
}
config.conn_info.hostport = atoi(s+1);
if (config.interactive)
printf("-> Redirected to slot [%d] located at %s:%d\n",
......
......@@ -610,6 +610,7 @@ void replicationFeedMonitors(client *c, list *monitors, int dictid, robj **argv,
while((ln = listNext(&li))) {
client *monitor = ln->value;
addReply(monitor,cmdobj);
updateClientMemUsageAndBucket(monitor);
}
decrRefCount(cmdobj);
}
......
......@@ -917,11 +917,9 @@ void removeClientFromMemUsageBucket(client *c, int allow_eviction) {
* together clients consuming about the same amount of memory and can quickly
* free them in case we reach maxmemory-clients (client eviction).
*
* Note: This function filters clients of type monitor, master or replica regardless
* Note: This function filters clients of type no-evict, master or replica regardless
* of whether the eviction is enabled or not, so the memory usage we get from these
* types of clients via the INFO command may be out of date. If someday we wanna
* improve that to make monitors' memory usage more accurate, we need to re-add this
* function call to `replicationFeedMonitors()`.
* types of clients via the INFO command may be out of date.
*
* returns 1 if client eviction for this client is allowed, 0 otherwise.
*/
......@@ -4160,7 +4158,7 @@ int processCommand(client *c) {
int flags = CMD_CALL_FULL;
if (client_reprocessing_command) flags |= CMD_CALL_REPROCESSING;
call(c,flags);
if (listLength(server.ready_keys))
if (listLength(server.ready_keys) && !isInsideYieldingLongCommand())
handleClientsBlockedOnKeys();
}
......
......@@ -195,8 +195,16 @@ test "New master [join $addr {:}] role matches" {
}
test "SENTINEL RESET can resets the master" {
# After SENTINEL RESET, sometimes the sentinel can sense the master again,
# causing the test to fail. Here we give it a few more chances.
for {set j 0} {$j < 10} {incr j} {
assert_equal 1 [S 0 SENTINEL RESET mymaster]
assert_equal 0 [llength [S 0 SENTINEL SENTINELS mymaster]]
assert_equal 0 [llength [S 0 SENTINEL SLAVES mymaster]]
assert_equal 0 [llength [S 0 SENTINEL REPLICAS mymaster]]
set res1 [llength [S 0 SENTINEL SENTINELS mymaster]]
set res2 [llength [S 0 SENTINEL SLAVES mymaster]]
set res3 [llength [S 0 SENTINEL REPLICAS mymaster]]
if {$res1 eq 0 && $res2 eq 0 && $res3 eq 0} break
}
assert_equal 0 $res1
assert_equal 0 $res2
assert_equal 0 $res3
}
......@@ -62,8 +62,8 @@ proc sanitizer_errors_from_file {filename} {
}
# GCC UBSAN output does not contain 'Sanitizer' but 'runtime error'.
if {[string match {*runtime error*} $log] ||
[string match {*Sanitizer*} $log]} {
if {[string match {*runtime error*} $line] ||
[string match {*Sanitizer*} $line]} {
return $log
}
}
......
......@@ -47,6 +47,15 @@ start_server {tags {"acl external:skip"}} {
catch {r ACL SETUSER selector-syntax on (&* &fail)} e
assert_match "*ERR Error in ACL SETUSER modifier '(*)*Adding a pattern after the*" $e
catch {r ACL SETUSER selector-syntax on (+PING (+SELECT (+DEL} e
assert_match "*ERR Unmatched parenthesis in acl selector*" $e
catch {r ACL SETUSER selector-syntax on (+PING (+SELECT (+DEL ) ) ) } e
assert_match "*ERR Error in ACL SETUSER modifier*" $e
catch {r ACL SETUSER selector-syntax on (+PING (+SELECT (+DEL ) } e
assert_match "*ERR Error in ACL SETUSER modifier*" $e
assert_equal "" [r ACL GETUSER selector-syntax]
}
......
......@@ -615,6 +615,10 @@ start_server {tags {"acl external:skip"}} {
# Unnecessary categories are retained for potentional future compatibility
r ACL SETUSER adv-test -@all -@dangerous
assert_equal "-@all -@dangerous" [dict get [r ACL getuser adv-test] commands]
# Duplicate categories are compressed, regression test for #12470
r ACL SETUSER adv-test -@all +config +config|get -config|set +config
assert_equal "-@all +config" [dict get [r ACL getuser adv-test] commands]
}
test "ACL CAT with illegal arguments" {
......
......@@ -70,8 +70,8 @@ start_server {tags {"auth_binary_password external:skip"}} {
# Configure the replica with masterauth
set loglines [count_log_lines 0]
$slave slaveof $master_host $master_port
$slave config set masterauth "abc"
$slave slaveof $master_host $master_port
# Verify replica is not able to sync with master
wait_for_log_messages 0 {"*Unable to AUTH to MASTER*"} $loglines 1000 10
......
......@@ -81,6 +81,23 @@ start_multiple_servers 3 [list overrides $base_conf] {
set node1_rd [redis_deferring_client 0]
test "use previous hostip in \"cluster-preferred-endpoint-type unknown-endpoint\" mode" {
# backup and set cluster-preferred-endpoint-type unknown-endpoint
set endpoint_type_before_set [lindex [split [$node1 CONFIG GET cluster-preferred-endpoint-type] " "] 1]
$node1 CONFIG SET cluster-preferred-endpoint-type unknown-endpoint
# when redis-cli not in cluster mode, return MOVE with empty host
set slot_for_foo [$node1 CLUSTER KEYSLOT foo]
assert_error "*MOVED $slot_for_foo :*" {$node1 set foo bar}
# when in cluster mode, redirect using previous hostip
assert_equal "[exec src/redis-cli -h 127.0.0.1 -p [srv 0 port] -c set foo bar]" {OK}
assert_match "[exec src/redis-cli -h 127.0.0.1 -p [srv 0 port] -c get foo]" {bar}
assert_equal [$node1 CONFIG SET cluster-preferred-endpoint-type "$endpoint_type_before_set"] {OK}
}
test "Sanity test push cmd after resharding" {
assert_error {*MOVED*} {$node3 lpush key9184688 v1}
......
......@@ -307,6 +307,13 @@ start_server {tags {"scripting"}} {
set e
} {*against a key*}
test {EVAL - JSON string encoding a string larger than 2GB} {
run_script {
local s = string.rep("a", 1024 * 1024 * 1024)
return #cjson.encode(s..s..s)
} 0
} {3221225474} {large-memory} ;# length includes two double quotes at both ends
test {EVAL - JSON numeric decoding} {
# We must return the table as a string because otherwise
# Redis converts floats to ints and we get 0 and 1023 instead
......@@ -1112,6 +1119,53 @@ start_server {tags {"scripting"}} {
r ping
} {PONG}
test {Timedout scripts and unblocked command} {
# make sure a command that's allowed during BUSY doesn't trigger an unblocked command
# enable AOF to also expose an assertion if the bug would happen
r flushall
r config set appendonly yes
# create clients, and set one to block waiting for key 'x'
set rd [redis_deferring_client]
set rd2 [redis_deferring_client]
set r3 [redis_client]
$rd2 blpop x 0
wait_for_blocked_clients_count 1
# hack: allow the script to use client list command so that we can control when it aborts
r DEBUG set-disable-deny-scripts 1
r config set lua-time-limit 10
run_script_on_connection $rd {
local clients
redis.call('lpush',KEYS[1],'y');
while true do
clients = redis.call('client','list')
if string.find(clients, 'abortscript') ~= nil then break end
end
redis.call('lpush',KEYS[1],'z');
return clients
} 1 x
# wait for the script to be busy
after 200
catch {r ping} e
assert_match {BUSY*} $e
# run cause the script to abort, and run a command that could have processed
# unblocked clients (due to a bug)
$r3 hello 2 setname abortscript
# make sure the script completed before the pop was processed
assert_equal [$rd2 read] {x z}
assert_match {*abortscript*} [$rd read]
$rd close
$rd2 close
$r3 close
r DEBUG set-disable-deny-scripts 0
} {OK} {external:skip needs:debug}
test {Timedout scripts that modified data can't be killed by SCRIPT KILL} {
set rd [redis_deferring_client]
r config set lua-time-limit 10
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment