Unverified Commit 61297585 authored by Salvatore Sanfilippo's avatar Salvatore Sanfilippo Committed by GitHub
Browse files

Merge branch 'unstable' into modules_fork

parents 83e87bac fddc4757
...@@ -92,6 +92,7 @@ static const rio rioBufferIO = { ...@@ -92,6 +92,7 @@ static const rio rioBufferIO = {
rioBufferFlush, rioBufferFlush,
NULL, /* update_checksum */ NULL, /* update_checksum */
0, /* current checksum */ 0, /* current checksum */
0, /* flags */
0, /* bytes read or written */ 0, /* bytes read or written */
0, /* read/write chunk size */ 0, /* read/write chunk size */
{ { NULL, 0 } } /* union for io-specific vars */ { { NULL, 0 } } /* union for io-specific vars */
...@@ -145,6 +146,7 @@ static const rio rioFileIO = { ...@@ -145,6 +146,7 @@ static const rio rioFileIO = {
rioFileFlush, rioFileFlush,
NULL, /* update_checksum */ NULL, /* update_checksum */
0, /* current checksum */ 0, /* current checksum */
0, /* flags */
0, /* bytes read or written */ 0, /* bytes read or written */
0, /* read/write chunk size */ 0, /* read/write chunk size */
{ { NULL, 0 } } /* union for io-specific vars */ { { NULL, 0 } } /* union for io-specific vars */
...@@ -239,6 +241,7 @@ static const rio rioFdIO = { ...@@ -239,6 +241,7 @@ static const rio rioFdIO = {
rioFdFlush, rioFdFlush,
NULL, /* update_checksum */ NULL, /* update_checksum */
0, /* current checksum */ 0, /* current checksum */
0, /* flags */
0, /* bytes read or written */ 0, /* bytes read or written */
0, /* read/write chunk size */ 0, /* read/write chunk size */
{ { NULL, 0 } } /* union for io-specific vars */ { { NULL, 0 } } /* union for io-specific vars */
...@@ -374,6 +377,7 @@ static const rio rioFdsetIO = { ...@@ -374,6 +377,7 @@ static const rio rioFdsetIO = {
rioFdsetFlush, rioFdsetFlush,
NULL, /* update_checksum */ NULL, /* update_checksum */
0, /* current checksum */ 0, /* current checksum */
0, /* flags */
0, /* bytes read or written */ 0, /* bytes read or written */
0, /* read/write chunk size */ 0, /* read/write chunk size */
{ { NULL, 0 } } /* union for io-specific vars */ { { NULL, 0 } } /* union for io-specific vars */
......
/* /*
* Copyright (c) 2009-2012, Pieter Noordhuis <pcnoordhuis at gmail dot com> * Copyright (c) 2009-2012, Pieter Noordhuis <pcnoordhuis at gmail dot com>
* Copyright (c) 2009-2012, Salvatore Sanfilippo <antirez at gmail dot com> * Copyright (c) 2009-2019, Salvatore Sanfilippo <antirez at gmail dot com>
* All rights reserved. * All rights reserved.
* *
* Redistribution and use in source and binary forms, with or without * Redistribution and use in source and binary forms, with or without
...@@ -36,6 +36,9 @@ ...@@ -36,6 +36,9 @@
#include <stdint.h> #include <stdint.h>
#include "sds.h" #include "sds.h"
#define RIO_FLAG_READ_ERROR (1<<0)
#define RIO_FLAG_WRITE_ERROR (1<<1)
struct _rio { struct _rio {
/* Backend functions. /* Backend functions.
* Since this functions do not tolerate short writes or reads the return * Since this functions do not tolerate short writes or reads the return
...@@ -51,8 +54,8 @@ struct _rio { ...@@ -51,8 +54,8 @@ struct _rio {
* computation. */ * computation. */
void (*update_cksum)(struct _rio *, const void *buf, size_t len); void (*update_cksum)(struct _rio *, const void *buf, size_t len);
/* The current checksum */ /* The current checksum and flags (see RIO_FLAG_*) */
uint64_t cksum; uint64_t cksum, flags;
/* number of bytes read or written */ /* number of bytes read or written */
size_t processed_bytes; size_t processed_bytes;
...@@ -99,11 +102,14 @@ typedef struct _rio rio; ...@@ -99,11 +102,14 @@ typedef struct _rio rio;
* if needed. */ * if needed. */
static inline size_t rioWrite(rio *r, const void *buf, size_t len) { static inline size_t rioWrite(rio *r, const void *buf, size_t len) {
if (r->flags & RIO_FLAG_WRITE_ERROR) return 0;
while (len) { while (len) {
size_t bytes_to_write = (r->max_processing_chunk && r->max_processing_chunk < len) ? r->max_processing_chunk : len; size_t bytes_to_write = (r->max_processing_chunk && r->max_processing_chunk < len) ? r->max_processing_chunk : len;
if (r->update_cksum) r->update_cksum(r,buf,bytes_to_write); if (r->update_cksum) r->update_cksum(r,buf,bytes_to_write);
if (r->write(r,buf,bytes_to_write) == 0) if (r->write(r,buf,bytes_to_write) == 0) {
r->flags |= RIO_FLAG_WRITE_ERROR;
return 0; return 0;
}
buf = (char*)buf + bytes_to_write; buf = (char*)buf + bytes_to_write;
len -= bytes_to_write; len -= bytes_to_write;
r->processed_bytes += bytes_to_write; r->processed_bytes += bytes_to_write;
...@@ -112,10 +118,13 @@ static inline size_t rioWrite(rio *r, const void *buf, size_t len) { ...@@ -112,10 +118,13 @@ static inline size_t rioWrite(rio *r, const void *buf, size_t len) {
} }
static inline size_t rioRead(rio *r, void *buf, size_t len) { static inline size_t rioRead(rio *r, void *buf, size_t len) {
if (r->flags & RIO_FLAG_READ_ERROR) return 0;
while (len) { while (len) {
size_t bytes_to_read = (r->max_processing_chunk && r->max_processing_chunk < len) ? r->max_processing_chunk : len; size_t bytes_to_read = (r->max_processing_chunk && r->max_processing_chunk < len) ? r->max_processing_chunk : len;
if (r->read(r,buf,bytes_to_read) == 0) if (r->read(r,buf,bytes_to_read) == 0) {
r->flags |= RIO_FLAG_READ_ERROR;
return 0; return 0;
}
if (r->update_cksum) r->update_cksum(r,buf,bytes_to_read); if (r->update_cksum) r->update_cksum(r,buf,bytes_to_read);
buf = (char*)buf + bytes_to_read; buf = (char*)buf + bytes_to_read;
len -= bytes_to_read; len -= bytes_to_read;
...@@ -132,6 +141,22 @@ static inline int rioFlush(rio *r) { ...@@ -132,6 +141,22 @@ static inline int rioFlush(rio *r) {
return r->flush(r); return r->flush(r);
} }
/* This function allows to know if there was a read error in any past
* operation, since the rio stream was created or since the last call
* to rioClearError(). */
static inline int rioGetReadError(rio *r) {
return (r->flags & RIO_FLAG_READ_ERROR) != 0;
}
/* Like rioGetReadError() but for write errors. */
static inline int rioGetWriteError(rio *r) {
return (r->flags & RIO_FLAG_WRITE_ERROR) != 0;
}
static inline void rioClearErrors(rio *r) {
r->flags &= ~(RIO_FLAG_READ_ERROR|RIO_FLAG_WRITE_ERROR);
}
void rioInitWithFile(rio *r, FILE *fp); void rioInitWithFile(rio *r, FILE *fp);
void rioInitWithBuffer(rio *r, sds s); void rioInitWithBuffer(rio *r, sds s);
void rioInitWithFd(rio *r, int fd, size_t read_limit); void rioInitWithFd(rio *r, int fd, size_t read_limit);
......
...@@ -42,7 +42,10 @@ char *redisProtocolToLuaType_Int(lua_State *lua, char *reply); ...@@ -42,7 +42,10 @@ char *redisProtocolToLuaType_Int(lua_State *lua, char *reply);
char *redisProtocolToLuaType_Bulk(lua_State *lua, char *reply); char *redisProtocolToLuaType_Bulk(lua_State *lua, char *reply);
char *redisProtocolToLuaType_Status(lua_State *lua, char *reply); char *redisProtocolToLuaType_Status(lua_State *lua, char *reply);
char *redisProtocolToLuaType_Error(lua_State *lua, char *reply); char *redisProtocolToLuaType_Error(lua_State *lua, char *reply);
char *redisProtocolToLuaType_MultiBulk(lua_State *lua, char *reply, int atype); char *redisProtocolToLuaType_Aggregate(lua_State *lua, char *reply, int atype);
char *redisProtocolToLuaType_Null(lua_State *lua, char *reply);
char *redisProtocolToLuaType_Bool(lua_State *lua, char *reply, int tf);
char *redisProtocolToLuaType_Double(lua_State *lua, char *reply);
int redis_math_random (lua_State *L); int redis_math_random (lua_State *L);
int redis_math_randomseed (lua_State *L); int redis_math_randomseed (lua_State *L);
void ldbInit(void); void ldbInit(void);
...@@ -132,9 +135,12 @@ char *redisProtocolToLuaType(lua_State *lua, char* reply) { ...@@ -132,9 +135,12 @@ char *redisProtocolToLuaType(lua_State *lua, char* reply) {
case '$': p = redisProtocolToLuaType_Bulk(lua,reply); break; case '$': p = redisProtocolToLuaType_Bulk(lua,reply); break;
case '+': p = redisProtocolToLuaType_Status(lua,reply); break; case '+': p = redisProtocolToLuaType_Status(lua,reply); break;
case '-': p = redisProtocolToLuaType_Error(lua,reply); break; case '-': p = redisProtocolToLuaType_Error(lua,reply); break;
case '*': p = redisProtocolToLuaType_MultiBulk(lua,reply,*p); break; case '*': p = redisProtocolToLuaType_Aggregate(lua,reply,*p); break;
case '%': p = redisProtocolToLuaType_MultiBulk(lua,reply,*p); break; case '%': p = redisProtocolToLuaType_Aggregate(lua,reply,*p); break;
case '~': p = redisProtocolToLuaType_MultiBulk(lua,reply,*p); break; case '~': p = redisProtocolToLuaType_Aggregate(lua,reply,*p); break;
case '_': p = redisProtocolToLuaType_Null(lua,reply); break;
case '#': p = redisProtocolToLuaType_Bool(lua,reply,p[1]); break;
case ',': p = redisProtocolToLuaType_Double(lua,reply); break;
} }
return p; return p;
} }
...@@ -182,13 +188,13 @@ char *redisProtocolToLuaType_Error(lua_State *lua, char *reply) { ...@@ -182,13 +188,13 @@ char *redisProtocolToLuaType_Error(lua_State *lua, char *reply) {
return p+2; return p+2;
} }
char *redisProtocolToLuaType_MultiBulk(lua_State *lua, char *reply, int atype) { char *redisProtocolToLuaType_Aggregate(lua_State *lua, char *reply, int atype) {
char *p = strchr(reply+1,'\r'); char *p = strchr(reply+1,'\r');
long long mbulklen; long long mbulklen;
int j = 0; int j = 0;
string2ll(reply+1,p-reply-1,&mbulklen); string2ll(reply+1,p-reply-1,&mbulklen);
if (server.lua_caller->resp == 2 || atype == '*') { if (server.lua_client->resp == 2 || atype == '*') {
p += 2; p += 2;
if (mbulklen == -1) { if (mbulklen == -1) {
lua_pushboolean(lua,0); lua_pushboolean(lua,0);
...@@ -200,11 +206,15 @@ char *redisProtocolToLuaType_MultiBulk(lua_State *lua, char *reply, int atype) { ...@@ -200,11 +206,15 @@ char *redisProtocolToLuaType_MultiBulk(lua_State *lua, char *reply, int atype) {
p = redisProtocolToLuaType(lua,p); p = redisProtocolToLuaType(lua,p);
lua_settable(lua,-3); lua_settable(lua,-3);
} }
} else if (server.lua_caller->resp == 3) { } else if (server.lua_client->resp == 3) {
/* Here we handle only Set and Map replies in RESP3 mode, since arrays /* Here we handle only Set and Map replies in RESP3 mode, since arrays
* follow the above RESP2 code path. */ * follow the above RESP2 code path. Note that those are represented
* as a table with the "map" or "set" field populated with the actual
* table representing the set or the map type. */
p += 2; p += 2;
lua_newtable(lua); lua_newtable(lua);
lua_pushstring(lua,atype == '%' ? "map" : "set");
lua_newtable(lua);
for (j = 0; j < mbulklen; j++) { for (j = 0; j < mbulklen; j++) {
p = redisProtocolToLuaType(lua,p); p = redisProtocolToLuaType(lua,p);
if (atype == '%') { if (atype == '%') {
...@@ -214,10 +224,44 @@ char *redisProtocolToLuaType_MultiBulk(lua_State *lua, char *reply, int atype) { ...@@ -214,10 +224,44 @@ char *redisProtocolToLuaType_MultiBulk(lua_State *lua, char *reply, int atype) {
} }
lua_settable(lua,-3); lua_settable(lua,-3);
} }
lua_settable(lua,-3);
} }
return p; return p;
} }
char *redisProtocolToLuaType_Null(lua_State *lua, char *reply) {
char *p = strchr(reply+1,'\r');
lua_pushnil(lua);
return p+2;
}
char *redisProtocolToLuaType_Bool(lua_State *lua, char *reply, int tf) {
char *p = strchr(reply+1,'\r');
lua_pushboolean(lua,tf == 't');
return p+2;
}
char *redisProtocolToLuaType_Double(lua_State *lua, char *reply) {
char *p = strchr(reply+1,'\r');
char buf[MAX_LONG_DOUBLE_CHARS+1];
size_t len = p-reply-1;
double d;
if (len <= MAX_LONG_DOUBLE_CHARS) {
memcpy(buf,reply+1,len);
buf[len] = '\0';
d = strtod(buf,NULL); /* We expect a valid representation. */
} else {
d = 0;
}
lua_newtable(lua);
lua_pushstring(lua,"double");
lua_pushnumber(lua,d);
lua_settable(lua,-3);
return p+2;
}
/* This function is used in order to push an error on the Lua stack in the /* This function is used in order to push an error on the Lua stack in the
* format used by redis.pcall to return errors, which is a lua table * format used by redis.pcall to return errors, which is a lua table
* with a single "err" field set to the error string. Note that this * with a single "err" field set to the error string. Note that this
...@@ -292,6 +336,8 @@ void luaSortArray(lua_State *lua) { ...@@ -292,6 +336,8 @@ void luaSortArray(lua_State *lua) {
* Lua reply to Redis reply conversion functions. * Lua reply to Redis reply conversion functions.
* ------------------------------------------------------------------------- */ * ------------------------------------------------------------------------- */
/* Reply to client 'c' converting the top element in the Lua stack to a
* Redis reply. As a side effect the element is consumed from the stack. */
void luaReplyToRedisReply(client *c, lua_State *lua) { void luaReplyToRedisReply(client *c, lua_State *lua) {
int t = lua_type(lua,-1); int t = lua_type(lua,-1);
...@@ -300,7 +346,11 @@ void luaReplyToRedisReply(client *c, lua_State *lua) { ...@@ -300,7 +346,11 @@ void luaReplyToRedisReply(client *c, lua_State *lua) {
addReplyBulkCBuffer(c,(char*)lua_tostring(lua,-1),lua_strlen(lua,-1)); addReplyBulkCBuffer(c,(char*)lua_tostring(lua,-1),lua_strlen(lua,-1));
break; break;
case LUA_TBOOLEAN: case LUA_TBOOLEAN:
addReply(c,lua_toboolean(lua,-1) ? shared.cone : shared.null[c->resp]); if (server.lua_client->resp == 2)
addReply(c,lua_toboolean(lua,-1) ? shared.cone :
shared.null[c->resp]);
else
addReplyBool(c,lua_toboolean(lua,-1));
break; break;
case LUA_TNUMBER: case LUA_TNUMBER:
addReplyLongLong(c,(long long)lua_tonumber(lua,-1)); addReplyLongLong(c,(long long)lua_tonumber(lua,-1));
...@@ -310,6 +360,8 @@ void luaReplyToRedisReply(client *c, lua_State *lua) { ...@@ -310,6 +360,8 @@ void luaReplyToRedisReply(client *c, lua_State *lua) {
* Error are returned as a single element table with 'err' field. * Error are returned as a single element table with 'err' field.
* Status replies are returned as single element table with 'ok' * Status replies are returned as single element table with 'ok'
* field. */ * field. */
/* Handle error reply. */
lua_pushstring(lua,"err"); lua_pushstring(lua,"err");
lua_gettable(lua,-2); lua_gettable(lua,-2);
t = lua_type(lua,-1); t = lua_type(lua,-1);
...@@ -321,8 +373,9 @@ void luaReplyToRedisReply(client *c, lua_State *lua) { ...@@ -321,8 +373,9 @@ void luaReplyToRedisReply(client *c, lua_State *lua) {
lua_pop(lua,2); lua_pop(lua,2);
return; return;
} }
lua_pop(lua,1); /* Discard field name pushed before. */
lua_pop(lua,1); /* Handle status reply. */
lua_pushstring(lua,"ok"); lua_pushstring(lua,"ok");
lua_gettable(lua,-2); lua_gettable(lua,-2);
t = lua_type(lua,-1); t = lua_type(lua,-1);
...@@ -331,25 +384,81 @@ void luaReplyToRedisReply(client *c, lua_State *lua) { ...@@ -331,25 +384,81 @@ void luaReplyToRedisReply(client *c, lua_State *lua) {
sdsmapchars(ok,"\r\n"," ",2); sdsmapchars(ok,"\r\n"," ",2);
addReplySds(c,sdscatprintf(sdsempty(),"+%s\r\n",ok)); addReplySds(c,sdscatprintf(sdsempty(),"+%s\r\n",ok));
sdsfree(ok); sdsfree(ok);
lua_pop(lua,1); lua_pop(lua,2);
} else { return;
}
lua_pop(lua,1); /* Discard field name pushed before. */
/* Handle double reply. */
lua_pushstring(lua,"double");
lua_gettable(lua,-2);
t = lua_type(lua,-1);
if (t == LUA_TNUMBER) {
addReplyDouble(c,lua_tonumber(lua,-1));
lua_pop(lua,2);
return;
}
lua_pop(lua,1); /* Discard field name pushed before. */
/* Handle map reply. */
lua_pushstring(lua,"map");
lua_gettable(lua,-2);
t = lua_type(lua,-1);
if (t == LUA_TTABLE) {
int maplen = 0;
void *replylen = addReplyDeferredLen(c); void *replylen = addReplyDeferredLen(c);
int j = 1, mbulklen = 0; lua_pushnil(lua); /* Use nil to start iteration. */
while (lua_next(lua,-2)) {
lua_pop(lua,1); /* Discard the 'ok' field value we popped */ /* Stack now: table, key, value */
while(1) { luaReplyToRedisReply(c, lua); /* Return value. */
lua_pushnumber(lua,j++); lua_pushvalue(lua,-1); /* Dup key before consuming. */
lua_gettable(lua,-2); luaReplyToRedisReply(c, lua); /* Return key. */
t = lua_type(lua,-1); /* Stack now: table, key. */
if (t == LUA_TNIL) { maplen++;
lua_pop(lua,1); }
break; setDeferredMapLen(c,replylen,maplen);
} lua_pop(lua,2);
luaReplyToRedisReply(c, lua); return;
mbulklen++; }
lua_pop(lua,1); /* Discard field name pushed before. */
/* Handle set reply. */
lua_pushstring(lua,"set");
lua_gettable(lua,-2);
t = lua_type(lua,-1);
if (t == LUA_TTABLE) {
int setlen = 0;
void *replylen = addReplyDeferredLen(c);
lua_pushnil(lua); /* Use nil to start iteration. */
while (lua_next(lua,-2)) {
/* Stack now: table, key, true */
lua_pop(lua,1); /* Discard the boolean value. */
lua_pushvalue(lua,-1); /* Dup key before consuming. */
luaReplyToRedisReply(c, lua); /* Return key. */
/* Stack now: table, key. */
setlen++;
} }
setDeferredArrayLen(c,replylen,mbulklen); setDeferredSetLen(c,replylen,setlen);
lua_pop(lua,2);
return;
} }
lua_pop(lua,1); /* Discard field name pushed before. */
/* Handle the array reply. */
void *replylen = addReplyDeferredLen(c);
int j = 1, mbulklen = 0;
while(1) {
lua_pushnumber(lua,j++);
lua_gettable(lua,-2);
t = lua_type(lua,-1);
if (t == LUA_TNIL) {
lua_pop(lua,1);
break;
}
luaReplyToRedisReply(c, lua);
mbulklen++;
}
setDeferredArrayLen(c,replylen,mbulklen);
break; break;
default: default:
addReplyNull(c); addReplyNull(c);
...@@ -859,6 +968,25 @@ int luaLogCommand(lua_State *lua) { ...@@ -859,6 +968,25 @@ int luaLogCommand(lua_State *lua) {
return 0; return 0;
} }
/* redis.setresp() */
int luaSetResp(lua_State *lua) {
int argc = lua_gettop(lua);
if (argc != 1) {
lua_pushstring(lua, "redis.setresp() requires one argument.");
return lua_error(lua);
}
int resp = lua_tonumber(lua,-argc);
if (resp != 2 && resp != 3) {
lua_pushstring(lua, "RESP version must be 2 or 3.");
return lua_error(lua);
}
server.lua_client->resp = resp;
return 0;
}
/* --------------------------------------------------------------------------- /* ---------------------------------------------------------------------------
* Lua engine initialization and reset. * Lua engine initialization and reset.
* ------------------------------------------------------------------------- */ * ------------------------------------------------------------------------- */
...@@ -986,6 +1114,11 @@ void scriptingInit(int setup) { ...@@ -986,6 +1114,11 @@ void scriptingInit(int setup) {
lua_pushcfunction(lua,luaLogCommand); lua_pushcfunction(lua,luaLogCommand);
lua_settable(lua,-3); lua_settable(lua,-3);
/* redis.setresp */
lua_pushstring(lua,"setresp");
lua_pushcfunction(lua,luaSetResp);
lua_settable(lua,-3);
lua_pushstring(lua,"LOG_DEBUG"); lua_pushstring(lua,"LOG_DEBUG");
lua_pushnumber(lua,LL_DEBUG); lua_pushnumber(lua,LL_DEBUG);
lua_settable(lua,-3); lua_settable(lua,-3);
...@@ -1379,8 +1512,9 @@ void evalGenericCommand(client *c, int evalsha) { ...@@ -1379,8 +1512,9 @@ void evalGenericCommand(client *c, int evalsha) {
luaSetGlobalArray(lua,"KEYS",c->argv+3,numkeys); luaSetGlobalArray(lua,"KEYS",c->argv+3,numkeys);
luaSetGlobalArray(lua,"ARGV",c->argv+3+numkeys,c->argc-3-numkeys); luaSetGlobalArray(lua,"ARGV",c->argv+3+numkeys,c->argc-3-numkeys);
/* Select the right DB in the context of the Lua client */ /* Set the Lua client database and protocol. */
selectDb(server.lua_client,c->db->id); selectDb(server.lua_client,c->db->id);
server.lua_client->resp = 2; /* Default is RESP2, scripts can change it. */
/* Set a hook in order to be able to stop the script execution if it /* Set a hook in order to be able to stop the script execution if it
* is running for too much time. * is running for too much time.
...@@ -2052,6 +2186,11 @@ char *ldbRedisProtocolToHuman_Int(sds *o, char *reply); ...@@ -2052,6 +2186,11 @@ char *ldbRedisProtocolToHuman_Int(sds *o, char *reply);
char *ldbRedisProtocolToHuman_Bulk(sds *o, char *reply); char *ldbRedisProtocolToHuman_Bulk(sds *o, char *reply);
char *ldbRedisProtocolToHuman_Status(sds *o, char *reply); char *ldbRedisProtocolToHuman_Status(sds *o, char *reply);
char *ldbRedisProtocolToHuman_MultiBulk(sds *o, char *reply); char *ldbRedisProtocolToHuman_MultiBulk(sds *o, char *reply);
char *ldbRedisProtocolToHuman_Set(sds *o, char *reply);
char *ldbRedisProtocolToHuman_Map(sds *o, char *reply);
char *ldbRedisProtocolToHuman_Null(sds *o, char *reply);
char *ldbRedisProtocolToHuman_Bool(sds *o, char *reply);
char *ldbRedisProtocolToHuman_Double(sds *o, char *reply);
/* Get Redis protocol from 'reply' and appends it in human readable form to /* Get Redis protocol from 'reply' and appends it in human readable form to
* the passed SDS string 'o'. * the passed SDS string 'o'.
...@@ -2066,6 +2205,11 @@ char *ldbRedisProtocolToHuman(sds *o, char *reply) { ...@@ -2066,6 +2205,11 @@ char *ldbRedisProtocolToHuman(sds *o, char *reply) {
case '+': p = ldbRedisProtocolToHuman_Status(o,reply); break; case '+': p = ldbRedisProtocolToHuman_Status(o,reply); break;
case '-': p = ldbRedisProtocolToHuman_Status(o,reply); break; case '-': p = ldbRedisProtocolToHuman_Status(o,reply); break;
case '*': p = ldbRedisProtocolToHuman_MultiBulk(o,reply); break; case '*': p = ldbRedisProtocolToHuman_MultiBulk(o,reply); break;
case '~': p = ldbRedisProtocolToHuman_Set(o,reply); break;
case '%': p = ldbRedisProtocolToHuman_Map(o,reply); break;
case '_': p = ldbRedisProtocolToHuman_Null(o,reply); break;
case '#': p = ldbRedisProtocolToHuman_Bool(o,reply); break;
case ',': p = ldbRedisProtocolToHuman_Double(o,reply); break;
} }
return p; return p;
} }
...@@ -2120,6 +2264,62 @@ char *ldbRedisProtocolToHuman_MultiBulk(sds *o, char *reply) { ...@@ -2120,6 +2264,62 @@ char *ldbRedisProtocolToHuman_MultiBulk(sds *o, char *reply) {
return p; return p;
} }
char *ldbRedisProtocolToHuman_Set(sds *o, char *reply) {
char *p = strchr(reply+1,'\r');
long long mbulklen;
int j = 0;
string2ll(reply+1,p-reply-1,&mbulklen);
p += 2;
*o = sdscatlen(*o,"~(",2);
for (j = 0; j < mbulklen; j++) {
p = ldbRedisProtocolToHuman(o,p);
if (j != mbulklen-1) *o = sdscatlen(*o,",",1);
}
*o = sdscatlen(*o,")",1);
return p;
}
char *ldbRedisProtocolToHuman_Map(sds *o, char *reply) {
char *p = strchr(reply+1,'\r');
long long mbulklen;
int j = 0;
string2ll(reply+1,p-reply-1,&mbulklen);
p += 2;
*o = sdscatlen(*o,"{",1);
for (j = 0; j < mbulklen; j++) {
p = ldbRedisProtocolToHuman(o,p);
*o = sdscatlen(*o," => ",4);
p = ldbRedisProtocolToHuman(o,p);
if (j != mbulklen-1) *o = sdscatlen(*o,",",1);
}
*o = sdscatlen(*o,"}",1);
return p;
}
char *ldbRedisProtocolToHuman_Null(sds *o, char *reply) {
char *p = strchr(reply+1,'\r');
*o = sdscatlen(*o,"(null)",6);
return p+2;
}
char *ldbRedisProtocolToHuman_Bool(sds *o, char *reply) {
char *p = strchr(reply+1,'\r');
if (reply[1] == 't')
*o = sdscatlen(*o,"#true",5);
else
*o = sdscatlen(*o,"#false",6);
return p+2;
}
char *ldbRedisProtocolToHuman_Double(sds *o, char *reply) {
char *p = strchr(reply+1,'\r');
*o = sdscatlen(*o,"(double) ",9);
*o = sdscatlen(*o,reply+1,p-reply-1);
return p+2;
}
/* Log a Redis reply as debugger output, in an human readable format. /* Log a Redis reply as debugger output, in an human readable format.
* If the resulting string is longer than 'len' plus a few more chars * If the resulting string is longer than 'len' plus a few more chars
* used as prefix, it gets truncated. */ * used as prefix, it gets truncated. */
......
...@@ -146,6 +146,8 @@ volatile unsigned long lru_clock; /* Server global current LRU time. */ ...@@ -146,6 +146,8 @@ volatile unsigned long lru_clock; /* Server global current LRU time. */
* in this condition but just a few. * in this condition but just a few.
* *
* no-monitor: Do not automatically propagate the command on MONITOR. * no-monitor: Do not automatically propagate the command on MONITOR.
*
* no-slowlog: Do not automatically propagate the command to the slowlog.
* *
* cluster-asking: Perform an implicit ASKING for this command, so the * cluster-asking: Perform an implicit ASKING for this command, so the
* command will be accepted in cluster mode if the slot is marked * command will be accepted in cluster mode if the slot is marked
...@@ -627,7 +629,7 @@ struct redisCommand redisCommandTable[] = { ...@@ -627,7 +629,7 @@ struct redisCommand redisCommandTable[] = {
0,NULL,0,0,0,0,0,0}, 0,NULL,0,0,0,0,0,0},
{"auth",authCommand,-2, {"auth",authCommand,-2,
"no-script ok-loading ok-stale fast @connection", "no-script ok-loading ok-stale fast no-monitor no-slowlog @connection",
0,NULL,0,0,0,0,0,0}, 0,NULL,0,0,0,0,0,0},
/* We don't allow PING during loading since in Redis PING is used as /* We don't allow PING during loading since in Redis PING is used as
...@@ -670,7 +672,7 @@ struct redisCommand redisCommandTable[] = { ...@@ -670,7 +672,7 @@ struct redisCommand redisCommandTable[] = {
0,NULL,0,0,0,0,0,0}, 0,NULL,0,0,0,0,0,0},
{"exec",execCommand,1, {"exec",execCommand,1,
"no-script no-monitor @transaction", "no-script no-monitor no-slowlog @transaction",
0,NULL,0,0,0,0,0,0}, 0,NULL,0,0,0,0,0,0},
{"discard",discardCommand,1, {"discard",discardCommand,1,
...@@ -822,7 +824,7 @@ struct redisCommand redisCommandTable[] = { ...@@ -822,7 +824,7 @@ struct redisCommand redisCommandTable[] = {
0,NULL,0,0,0,0,0,0}, 0,NULL,0,0,0,0,0,0},
{"hello",helloCommand,-2, {"hello",helloCommand,-2,
"no-script fast @connection", "no-script fast no-monitor no-slowlog @connection",
0,NULL,0,0,0,0,0,0}, 0,NULL,0,0,0,0,0,0},
/* EVAL can modify the dataset, however it is not flagged as a write /* EVAL can modify the dataset, however it is not flagged as a write
...@@ -2174,6 +2176,16 @@ void createSharedObjects(void) { ...@@ -2174,6 +2176,16 @@ void createSharedObjects(void) {
shared.nullarray[2] = createObject(OBJ_STRING,sdsnew("*-1\r\n")); shared.nullarray[2] = createObject(OBJ_STRING,sdsnew("*-1\r\n"));
shared.nullarray[3] = createObject(OBJ_STRING,sdsnew("_\r\n")); shared.nullarray[3] = createObject(OBJ_STRING,sdsnew("_\r\n"));
shared.emptymap[0] = NULL;
shared.emptymap[1] = NULL;
shared.emptymap[2] = createObject(OBJ_STRING,sdsnew("*0\r\n"));
shared.emptymap[3] = createObject(OBJ_STRING,sdsnew("%0\r\n"));
shared.emptyset[0] = NULL;
shared.emptyset[1] = NULL;
shared.emptyset[2] = createObject(OBJ_STRING,sdsnew("*0\r\n"));
shared.emptyset[3] = createObject(OBJ_STRING,sdsnew("~0\r\n"));
for (j = 0; j < PROTO_SHARED_SELECT_CMDS; j++) { for (j = 0; j < PROTO_SHARED_SELECT_CMDS; j++) {
char dictid_str[64]; char dictid_str[64];
int dictid_len; int dictid_len;
...@@ -2418,6 +2430,9 @@ void initServerConfig(void) { ...@@ -2418,6 +2430,9 @@ void initServerConfig(void) {
/* Latency monitor */ /* Latency monitor */
server.latency_monitor_threshold = CONFIG_DEFAULT_LATENCY_MONITOR_THRESHOLD; server.latency_monitor_threshold = CONFIG_DEFAULT_LATENCY_MONITOR_THRESHOLD;
/* Tracking. */
server.tracking_table_max_fill = CONFIG_DEFAULT_TRACKING_TABLE_MAX_FILL;
/* Debugging */ /* Debugging */
server.assert_failed = "<no assertion failed>"; server.assert_failed = "<no assertion failed>";
server.assert_file = "<no file>"; server.assert_file = "<no file>";
...@@ -2926,6 +2941,8 @@ int populateCommandTableParseFlags(struct redisCommand *c, char *strflags) { ...@@ -2926,6 +2941,8 @@ int populateCommandTableParseFlags(struct redisCommand *c, char *strflags) {
c->flags |= CMD_STALE; c->flags |= CMD_STALE;
} else if (!strcasecmp(flag,"no-monitor")) { } else if (!strcasecmp(flag,"no-monitor")) {
c->flags |= CMD_SKIP_MONITOR; c->flags |= CMD_SKIP_MONITOR;
} else if (!strcasecmp(flag,"no-slowlog")) {
c->flags |= CMD_SKIP_SLOWLOG;
} else if (!strcasecmp(flag,"cluster-asking")) { } else if (!strcasecmp(flag,"cluster-asking")) {
c->flags |= CMD_ASKING; c->flags |= CMD_ASKING;
} else if (!strcasecmp(flag,"fast")) { } else if (!strcasecmp(flag,"fast")) {
...@@ -3210,7 +3227,7 @@ void call(client *c, int flags) { ...@@ -3210,7 +3227,7 @@ void call(client *c, int flags) {
/* Log the command into the Slow log if needed, and populate the /* Log the command into the Slow log if needed, and populate the
* per-command statistics that we show in INFO commandstats. */ * per-command statistics that we show in INFO commandstats. */
if (flags & CMD_CALL_SLOWLOG && c->cmd->proc != execCommand) { if (flags & CMD_CALL_SLOWLOG && !(c->cmd->flags & CMD_SKIP_SLOWLOG)) {
char *latency_event = (c->cmd->flags & CMD_FAST) ? char *latency_event = (c->cmd->flags & CMD_FAST) ?
"fast-command" : "command"; "fast-command" : "command";
latencyAddSampleIfNeeded(latency_event,duration/1000); latencyAddSampleIfNeeded(latency_event,duration/1000);
...@@ -3341,9 +3358,10 @@ int processCommand(client *c) { ...@@ -3341,9 +3358,10 @@ int processCommand(client *c) {
/* Check if the user is authenticated. This check is skipped in case /* Check if the user is authenticated. This check is skipped in case
* the default user is flagged as "nopass" and is active. */ * the default user is flagged as "nopass" and is active. */
int auth_required = !(DefaultUser->flags & USER_FLAG_NOPASS) && int auth_required = (!(DefaultUser->flags & USER_FLAG_NOPASS) ||
DefaultUser->flags & USER_FLAG_DISABLED) &&
!c->authenticated; !c->authenticated;
if (auth_required || DefaultUser->flags & USER_FLAG_DISABLED) { if (auth_required) {
/* AUTH and HELLO are valid even in non authenticated state. */ /* AUTH and HELLO are valid even in non authenticated state. */
if (c->cmd->proc != authCommand || c->cmd->proc == helloCommand) { if (c->cmd->proc != authCommand || c->cmd->proc == helloCommand) {
flagTransaction(c); flagTransaction(c);
...@@ -3411,13 +3429,20 @@ int processCommand(client *c) { ...@@ -3411,13 +3429,20 @@ int processCommand(client *c) {
* is in MULTI/EXEC context? Error. */ * is in MULTI/EXEC context? Error. */
if (out_of_memory && if (out_of_memory &&
(c->cmd->flags & CMD_DENYOOM || (c->cmd->flags & CMD_DENYOOM ||
(c->flags & CLIENT_MULTI && c->cmd->proc != execCommand))) { (c->flags & CLIENT_MULTI &&
c->cmd->proc != execCommand &&
c->cmd->proc != discardCommand)))
{
flagTransaction(c); flagTransaction(c);
addReply(c, shared.oomerr); addReply(c, shared.oomerr);
return C_OK; return C_OK;
} }
} }
/* Make sure to use a reasonable amount of memory for client side
* caching metadata. */
if (server.tracking_clients) trackingLimitUsedSlots();
/* Don't accept write commands if there are problems persisting on disk /* Don't accept write commands if there are problems persisting on disk
* and if this is a master instance. */ * and if this is a master instance. */
int deny_write_type = writeCommandsDeniedByDiskError(); int deny_write_type = writeCommandsDeniedByDiskError();
...@@ -3718,6 +3743,7 @@ void addReplyCommand(client *c, struct redisCommand *cmd) { ...@@ -3718,6 +3743,7 @@ void addReplyCommand(client *c, struct redisCommand *cmd) {
flagcount += addReplyCommandFlag(c,cmd,CMD_LOADING, "loading"); flagcount += addReplyCommandFlag(c,cmd,CMD_LOADING, "loading");
flagcount += addReplyCommandFlag(c,cmd,CMD_STALE, "stale"); flagcount += addReplyCommandFlag(c,cmd,CMD_STALE, "stale");
flagcount += addReplyCommandFlag(c,cmd,CMD_SKIP_MONITOR, "skip_monitor"); flagcount += addReplyCommandFlag(c,cmd,CMD_SKIP_MONITOR, "skip_monitor");
flagcount += addReplyCommandFlag(c,cmd,CMD_SKIP_SLOWLOG, "skip_slowlog");
flagcount += addReplyCommandFlag(c,cmd,CMD_ASKING, "asking"); flagcount += addReplyCommandFlag(c,cmd,CMD_ASKING, "asking");
flagcount += addReplyCommandFlag(c,cmd,CMD_FAST, "fast"); flagcount += addReplyCommandFlag(c,cmd,CMD_FAST, "fast");
if ((cmd->getkeys_proc && !(cmd->flags & CMD_MODULE)) || if ((cmd->getkeys_proc && !(cmd->flags & CMD_MODULE)) ||
...@@ -4167,7 +4193,8 @@ sds genRedisInfoString(char *section) { ...@@ -4167,7 +4193,8 @@ sds genRedisInfoString(char *section) {
"active_defrag_hits:%lld\r\n" "active_defrag_hits:%lld\r\n"
"active_defrag_misses:%lld\r\n" "active_defrag_misses:%lld\r\n"
"active_defrag_key_hits:%lld\r\n" "active_defrag_key_hits:%lld\r\n"
"active_defrag_key_misses:%lld\r\n", "active_defrag_key_misses:%lld\r\n"
"tracking_used_slots:%lld\r\n",
server.stat_numconnections, server.stat_numconnections,
server.stat_numcommands, server.stat_numcommands,
getInstantaneousMetric(STATS_METRIC_COMMAND), getInstantaneousMetric(STATS_METRIC_COMMAND),
...@@ -4193,7 +4220,8 @@ sds genRedisInfoString(char *section) { ...@@ -4193,7 +4220,8 @@ sds genRedisInfoString(char *section) {
server.stat_active_defrag_hits, server.stat_active_defrag_hits,
server.stat_active_defrag_misses, server.stat_active_defrag_misses,
server.stat_active_defrag_key_hits, server.stat_active_defrag_key_hits,
server.stat_active_defrag_key_misses); server.stat_active_defrag_key_misses,
trackingGetUsedSlots());
} }
/* Replication */ /* Replication */
...@@ -4339,6 +4367,13 @@ sds genRedisInfoString(char *section) { ...@@ -4339,6 +4367,13 @@ sds genRedisInfoString(char *section) {
(long)c_ru.ru_utime.tv_sec, (long)c_ru.ru_utime.tv_usec); (long)c_ru.ru_utime.tv_sec, (long)c_ru.ru_utime.tv_usec);
} }
/* Modules */
if (allsections || defsections || !strcasecmp(section,"modules")) {
if (sections++) info = sdscat(info,"\r\n");
info = sdscatprintf(info,"# Modules\r\n");
info = genModulesInfoString(info);
}
/* Command statistics */ /* Command statistics */
if (allsections || !strcasecmp(section,"commandstats")) { if (allsections || !strcasecmp(section,"commandstats")) {
if (sections++) info = sdscat(info,"\r\n"); if (sections++) info = sdscat(info,"\r\n");
...@@ -4394,7 +4429,9 @@ void infoCommand(client *c) { ...@@ -4394,7 +4429,9 @@ void infoCommand(client *c) {
addReply(c,shared.syntaxerr); addReply(c,shared.syntaxerr);
return; return;
} }
addReplyBulkSds(c, genRedisInfoString(section)); sds info = genRedisInfoString(section);
addReplyVerbatim(c,info,sdslen(info),"txt");
sdsfree(info);
} }
void monitorCommand(client *c) { void monitorCommand(client *c) {
...@@ -4840,9 +4877,9 @@ int main(int argc, char **argv) { ...@@ -4840,9 +4877,9 @@ int main(int argc, char **argv) {
srand(time(NULL)^getpid()); srand(time(NULL)^getpid());
gettimeofday(&tv,NULL); gettimeofday(&tv,NULL);
char hashseed[16]; uint8_t hashseed[16];
getRandomHexChars(hashseed,sizeof(hashseed)); getRandomBytes(hashseed,sizeof(hashseed));
dictSetHashFunctionSeed((uint8_t*)hashseed); dictSetHashFunctionSeed(hashseed);
server.sentinel_mode = checkForSentinelMode(argc,argv); server.sentinel_mode = checkForSentinelMode(argc,argv);
initServerConfig(); initServerConfig();
ACLInit(); /* The ACL subsystem must be initialized ASAP because the ACLInit(); /* The ACL subsystem must be initialized ASAP because the
......
...@@ -171,6 +171,7 @@ typedef long long mstime_t; /* millisecond time type. */ ...@@ -171,6 +171,7 @@ typedef long long mstime_t; /* millisecond time type. */
#define CONFIG_DEFAULT_DEFRAG_CYCLE_MAX 75 /* 75% CPU max (at upper threshold) */ #define CONFIG_DEFAULT_DEFRAG_CYCLE_MAX 75 /* 75% CPU max (at upper threshold) */
#define CONFIG_DEFAULT_DEFRAG_MAX_SCAN_FIELDS 1000 /* keys with more than 1000 fields will be processed separately */ #define CONFIG_DEFAULT_DEFRAG_MAX_SCAN_FIELDS 1000 /* keys with more than 1000 fields will be processed separately */
#define CONFIG_DEFAULT_PROTO_MAX_BULK_LEN (512ll*1024*1024) /* Bulk request max size */ #define CONFIG_DEFAULT_PROTO_MAX_BULK_LEN (512ll*1024*1024) /* Bulk request max size */
#define CONFIG_DEFAULT_TRACKING_TABLE_MAX_FILL 10 /* 10% tracking table max fill. */
#define ACTIVE_EXPIRE_CYCLE_LOOKUPS_PER_LOOP 20 /* Loopkups per loop. */ #define ACTIVE_EXPIRE_CYCLE_LOOKUPS_PER_LOOP 20 /* Loopkups per loop. */
#define ACTIVE_EXPIRE_CYCLE_FAST_DURATION 1000 /* Microseconds */ #define ACTIVE_EXPIRE_CYCLE_FAST_DURATION 1000 /* Microseconds */
...@@ -219,35 +220,36 @@ typedef long long mstime_t; /* millisecond time type. */ ...@@ -219,35 +220,36 @@ typedef long long mstime_t; /* millisecond time type. */
#define CMD_LOADING (1ULL<<9) /* "ok-loading" flag */ #define CMD_LOADING (1ULL<<9) /* "ok-loading" flag */
#define CMD_STALE (1ULL<<10) /* "ok-stale" flag */ #define CMD_STALE (1ULL<<10) /* "ok-stale" flag */
#define CMD_SKIP_MONITOR (1ULL<<11) /* "no-monitor" flag */ #define CMD_SKIP_MONITOR (1ULL<<11) /* "no-monitor" flag */
#define CMD_ASKING (1ULL<<12) /* "cluster-asking" flag */ #define CMD_SKIP_SLOWLOG (1ULL<<12) /* "no-slowlog" flag */
#define CMD_FAST (1ULL<<13) /* "fast" flag */ #define CMD_ASKING (1ULL<<13) /* "cluster-asking" flag */
#define CMD_FAST (1ULL<<14) /* "fast" flag */
/* Command flags used by the module system. */ /* Command flags used by the module system. */
#define CMD_MODULE_GETKEYS (1ULL<<14) /* Use the modules getkeys interface. */ #define CMD_MODULE_GETKEYS (1ULL<<15) /* Use the modules getkeys interface. */
#define CMD_MODULE_NO_CLUSTER (1ULL<<15) /* Deny on Redis Cluster. */ #define CMD_MODULE_NO_CLUSTER (1ULL<<16) /* Deny on Redis Cluster. */
/* Command flags that describe ACLs categories. */ /* Command flags that describe ACLs categories. */
#define CMD_CATEGORY_KEYSPACE (1ULL<<16) #define CMD_CATEGORY_KEYSPACE (1ULL<<17)
#define CMD_CATEGORY_READ (1ULL<<17) #define CMD_CATEGORY_READ (1ULL<<18)
#define CMD_CATEGORY_WRITE (1ULL<<18) #define CMD_CATEGORY_WRITE (1ULL<<19)
#define CMD_CATEGORY_SET (1ULL<<19) #define CMD_CATEGORY_SET (1ULL<<20)
#define CMD_CATEGORY_SORTEDSET (1ULL<<20) #define CMD_CATEGORY_SORTEDSET (1ULL<<21)
#define CMD_CATEGORY_LIST (1ULL<<21) #define CMD_CATEGORY_LIST (1ULL<<22)
#define CMD_CATEGORY_HASH (1ULL<<22) #define CMD_CATEGORY_HASH (1ULL<<23)
#define CMD_CATEGORY_STRING (1ULL<<23) #define CMD_CATEGORY_STRING (1ULL<<24)
#define CMD_CATEGORY_BITMAP (1ULL<<24) #define CMD_CATEGORY_BITMAP (1ULL<<25)
#define CMD_CATEGORY_HYPERLOGLOG (1ULL<<25) #define CMD_CATEGORY_HYPERLOGLOG (1ULL<<26)
#define CMD_CATEGORY_GEO (1ULL<<26) #define CMD_CATEGORY_GEO (1ULL<<27)
#define CMD_CATEGORY_STREAM (1ULL<<27) #define CMD_CATEGORY_STREAM (1ULL<<28)
#define CMD_CATEGORY_PUBSUB (1ULL<<28) #define CMD_CATEGORY_PUBSUB (1ULL<<29)
#define CMD_CATEGORY_ADMIN (1ULL<<29) #define CMD_CATEGORY_ADMIN (1ULL<<30)
#define CMD_CATEGORY_FAST (1ULL<<30) #define CMD_CATEGORY_FAST (1ULL<<31)
#define CMD_CATEGORY_SLOW (1ULL<<31) #define CMD_CATEGORY_SLOW (1ULL<<32)
#define CMD_CATEGORY_BLOCKING (1ULL<<32) #define CMD_CATEGORY_BLOCKING (1ULL<<33)
#define CMD_CATEGORY_DANGEROUS (1ULL<<33) #define CMD_CATEGORY_DANGEROUS (1ULL<<34)
#define CMD_CATEGORY_CONNECTION (1ULL<<34) #define CMD_CATEGORY_CONNECTION (1ULL<<35)
#define CMD_CATEGORY_TRANSACTION (1ULL<<35) #define CMD_CATEGORY_TRANSACTION (1ULL<<36)
#define CMD_CATEGORY_SCRIPTING (1ULL<<36) #define CMD_CATEGORY_SCRIPTING (1ULL<<37)
/* AOF states */ /* AOF states */
#define AOF_OFF 0 /* AOF is off */ #define AOF_OFF 0 /* AOF is off */
...@@ -536,6 +538,10 @@ typedef long long mstime_t; /* millisecond time type. */ ...@@ -536,6 +538,10 @@ typedef long long mstime_t; /* millisecond time type. */
#define REDISMODULE_TYPE_ENCVER(id) (id & REDISMODULE_TYPE_ENCVER_MASK) #define REDISMODULE_TYPE_ENCVER(id) (id & REDISMODULE_TYPE_ENCVER_MASK)
#define REDISMODULE_TYPE_SIGN(id) ((id & ~((uint64_t)REDISMODULE_TYPE_ENCVER_MASK)) >>REDISMODULE_TYPE_ENCVER_BITS) #define REDISMODULE_TYPE_SIGN(id) ((id & ~((uint64_t)REDISMODULE_TYPE_ENCVER_MASK)) >>REDISMODULE_TYPE_ENCVER_BITS)
/* Bit flags for moduleTypeAuxSaveFunc */
#define REDISMODULE_AUX_BEFORE_RDB (1<<0)
#define REDISMODULE_AUX_AFTER_RDB (1<<1)
struct RedisModule; struct RedisModule;
struct RedisModuleIO; struct RedisModuleIO;
struct RedisModuleDigest; struct RedisModuleDigest;
...@@ -548,6 +554,8 @@ struct redisObject; ...@@ -548,6 +554,8 @@ struct redisObject;
* is deleted. */ * is deleted. */
typedef void *(*moduleTypeLoadFunc)(struct RedisModuleIO *io, int encver); typedef void *(*moduleTypeLoadFunc)(struct RedisModuleIO *io, int encver);
typedef void (*moduleTypeSaveFunc)(struct RedisModuleIO *io, void *value); typedef void (*moduleTypeSaveFunc)(struct RedisModuleIO *io, void *value);
typedef int (*moduleTypeAuxLoadFunc)(struct RedisModuleIO *rdb, int encver, int when);
typedef void (*moduleTypeAuxSaveFunc)(struct RedisModuleIO *rdb, int when);
typedef void (*moduleTypeRewriteFunc)(struct RedisModuleIO *io, struct redisObject *key, void *value); typedef void (*moduleTypeRewriteFunc)(struct RedisModuleIO *io, struct redisObject *key, void *value);
typedef void (*moduleTypeDigestFunc)(struct RedisModuleDigest *digest, void *value); typedef void (*moduleTypeDigestFunc)(struct RedisModuleDigest *digest, void *value);
typedef size_t (*moduleTypeMemUsageFunc)(const void *value); typedef size_t (*moduleTypeMemUsageFunc)(const void *value);
...@@ -564,6 +572,9 @@ typedef struct RedisModuleType { ...@@ -564,6 +572,9 @@ typedef struct RedisModuleType {
moduleTypeMemUsageFunc mem_usage; moduleTypeMemUsageFunc mem_usage;
moduleTypeDigestFunc digest; moduleTypeDigestFunc digest;
moduleTypeFreeFunc free; moduleTypeFreeFunc free;
moduleTypeAuxLoadFunc aux_load;
moduleTypeAuxSaveFunc aux_save;
int aux_save_triggers;
char name[10]; /* 9 bytes name + null term. Charset: A-Z a-z 0-9 _- */ char name[10]; /* 9 bytes name + null term. Charset: A-Z a-z 0-9 _- */
} moduleType; } moduleType;
...@@ -837,7 +848,7 @@ typedef struct client { ...@@ -837,7 +848,7 @@ typedef struct client {
uint64_t flags; /* Client flags: CLIENT_* macros. */ uint64_t flags; /* Client flags: CLIENT_* macros. */
int authenticated; /* Needed when the default user requires auth. */ int authenticated; /* Needed when the default user requires auth. */
int replstate; /* Replication state if this is a slave. */ int replstate; /* Replication state if this is a slave. */
int repl_put_online_on_ack; /* Install slave write handler on ACK. */ int repl_put_online_on_ack; /* Install slave write handler on first ACK. */
int repldbfd; /* Replication DB file descriptor. */ int repldbfd; /* Replication DB file descriptor. */
off_t repldboff; /* Replication DB file offset. */ off_t repldboff; /* Replication DB file offset. */
off_t repldbsize; /* Replication DB file size. */ off_t repldbsize; /* Replication DB file size. */
...@@ -886,7 +897,7 @@ struct moduleLoadQueueEntry { ...@@ -886,7 +897,7 @@ struct moduleLoadQueueEntry {
struct sharedObjectsStruct { struct sharedObjectsStruct {
robj *crlf, *ok, *err, *emptybulk, *czero, *cone, *pong, *space, robj *crlf, *ok, *err, *emptybulk, *czero, *cone, *pong, *space,
*colon, *queued, *null[4], *nullarray[4], *colon, *queued, *null[4], *nullarray[4], *emptymap[4], *emptyset[4],
*emptyarray, *wrongtypeerr, *nokeyerr, *syntaxerr, *sameobjecterr, *emptyarray, *wrongtypeerr, *nokeyerr, *syntaxerr, *sameobjecterr,
*outofrangeerr, *noscripterr, *loadingerr, *slowscripterr, *bgsaveerr, *outofrangeerr, *noscripterr, *loadingerr, *slowscripterr, *bgsaveerr,
*masterdownerr, *roslaveerr, *execaborterr, *noautherr, *noreplicaserr, *masterdownerr, *roslaveerr, *execaborterr, *noautherr, *noreplicaserr,
...@@ -1319,6 +1330,7 @@ struct redisServer { ...@@ -1319,6 +1330,7 @@ struct redisServer {
list *ready_keys; /* List of readyList structures for BLPOP & co */ list *ready_keys; /* List of readyList structures for BLPOP & co */
/* Client side caching. */ /* Client side caching. */
unsigned int tracking_clients; /* # of clients with tracking enabled.*/ unsigned int tracking_clients; /* # of clients with tracking enabled.*/
int tracking_table_max_fill; /* Max fill percentage. */
/* Sort parameters - qsort_r() is only available under BSD so we /* Sort parameters - qsort_r() is only available under BSD so we
* have to take this state global, in order to pass it to sortCompare() */ * have to take this state global, in order to pass it to sortCompare() */
int sort_desc; int sort_desc;
...@@ -1533,6 +1545,8 @@ void moduleNotifyKeyspaceEvent(int type, const char *event, robj *key, int dbid) ...@@ -1533,6 +1545,8 @@ void moduleNotifyKeyspaceEvent(int type, const char *event, robj *key, int dbid)
void moduleCallCommandFilters(client *c); void moduleCallCommandFilters(client *c);
void ModuleForkDoneHandler(int exitcode, int bysignal); void ModuleForkDoneHandler(int exitcode, int bysignal);
void TerminateModuleForkChild(int wait); void TerminateModuleForkChild(int wait);
ssize_t rdbSaveModulesAux(rio *rdb, int when);
int moduleAllDatatypesHandleErrors();
/* Utils */ /* Utils */
long long ustime(void); long long ustime(void);
...@@ -1643,6 +1657,9 @@ void enableTracking(client *c, uint64_t redirect_to); ...@@ -1643,6 +1657,9 @@ void enableTracking(client *c, uint64_t redirect_to);
void disableTracking(client *c); void disableTracking(client *c);
void trackingRememberKeys(client *c); void trackingRememberKeys(client *c);
void trackingInvalidateKey(robj *keyobj); void trackingInvalidateKey(robj *keyobj);
void trackingInvalidateKeysOnFlush(int dbid);
void trackingLimitUsedSlots(void);
unsigned long long trackingGetUsedSlots(void);
/* List data type */ /* List data type */
void listTypeTryConversion(robj *subject, robj *value); void listTypeTryConversion(robj *subject, robj *value);
...@@ -2328,6 +2345,7 @@ void bugReportStart(void); ...@@ -2328,6 +2345,7 @@ void bugReportStart(void);
void serverLogObjectDebugInfo(const robj *o); void serverLogObjectDebugInfo(const robj *o);
void sigsegvHandler(int sig, siginfo_t *info, void *secret); void sigsegvHandler(int sig, siginfo_t *info, void *secret);
sds genRedisInfoString(char *section); sds genRedisInfoString(char *section);
sds genModulesInfoString(sds info);
void enableWatchdog(int period); void enableWatchdog(int period);
void disableWatchdog(void); void disableWatchdog(void);
void watchdogScheduleSignal(int period); void watchdogScheduleSignal(int period);
......
/*********************************************************************
* Filename: sha256.c
* Author: Brad Conte (brad AT bradconte.com)
* Copyright:
* Disclaimer: This code is presented "as is" without any guarantees.
* Details: Implementation of the SHA-256 hashing algorithm.
SHA-256 is one of the three algorithms in the SHA2
specification. The others, SHA-384 and SHA-512, are not
offered in this implementation.
Algorithm specification can be found here:
* http://csrc.nist.gov/publications/fips/fips180-2/fips180-2withchangenotice.pdf
This implementation uses little endian byte order.
*********************************************************************/
/*************************** HEADER FILES ***************************/
#include <stdlib.h>
#include <string.h>
#include "sha256.h"
/****************************** MACROS ******************************/
#define ROTLEFT(a,b) (((a) << (b)) | ((a) >> (32-(b))))
#define ROTRIGHT(a,b) (((a) >> (b)) | ((a) << (32-(b))))
#define CH(x,y,z) (((x) & (y)) ^ (~(x) & (z)))
#define MAJ(x,y,z) (((x) & (y)) ^ ((x) & (z)) ^ ((y) & (z)))
#define EP0(x) (ROTRIGHT(x,2) ^ ROTRIGHT(x,13) ^ ROTRIGHT(x,22))
#define EP1(x) (ROTRIGHT(x,6) ^ ROTRIGHT(x,11) ^ ROTRIGHT(x,25))
#define SIG0(x) (ROTRIGHT(x,7) ^ ROTRIGHT(x,18) ^ ((x) >> 3))
#define SIG1(x) (ROTRIGHT(x,17) ^ ROTRIGHT(x,19) ^ ((x) >> 10))
/**************************** VARIABLES *****************************/
static const WORD k[64] = {
0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5,0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5,
0xd807aa98,0x12835b01,0x243185be,0x550c7dc3,0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174,
0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc,0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da,
0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7,0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967,
0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13,0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85,
0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3,0xd192e819,0xd6990624,0xf40e3585,0x106aa070,
0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5,0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3,
0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208,0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
};
/*********************** FUNCTION DEFINITIONS ***********************/
void sha256_transform(SHA256_CTX *ctx, const BYTE data[])
{
WORD a, b, c, d, e, f, g, h, i, j, t1, t2, m[64];
for (i = 0, j = 0; i < 16; ++i, j += 4)
m[i] = (data[j] << 24) | (data[j + 1] << 16) | (data[j + 2] << 8) | (data[j + 3]);
for ( ; i < 64; ++i)
m[i] = SIG1(m[i - 2]) + m[i - 7] + SIG0(m[i - 15]) + m[i - 16];
a = ctx->state[0];
b = ctx->state[1];
c = ctx->state[2];
d = ctx->state[3];
e = ctx->state[4];
f = ctx->state[5];
g = ctx->state[6];
h = ctx->state[7];
for (i = 0; i < 64; ++i) {
t1 = h + EP1(e) + CH(e,f,g) + k[i] + m[i];
t2 = EP0(a) + MAJ(a,b,c);
h = g;
g = f;
f = e;
e = d + t1;
d = c;
c = b;
b = a;
a = t1 + t2;
}
ctx->state[0] += a;
ctx->state[1] += b;
ctx->state[2] += c;
ctx->state[3] += d;
ctx->state[4] += e;
ctx->state[5] += f;
ctx->state[6] += g;
ctx->state[7] += h;
}
void sha256_init(SHA256_CTX *ctx)
{
ctx->datalen = 0;
ctx->bitlen = 0;
ctx->state[0] = 0x6a09e667;
ctx->state[1] = 0xbb67ae85;
ctx->state[2] = 0x3c6ef372;
ctx->state[3] = 0xa54ff53a;
ctx->state[4] = 0x510e527f;
ctx->state[5] = 0x9b05688c;
ctx->state[6] = 0x1f83d9ab;
ctx->state[7] = 0x5be0cd19;
}
void sha256_update(SHA256_CTX *ctx, const BYTE data[], size_t len)
{
WORD i;
for (i = 0; i < len; ++i) {
ctx->data[ctx->datalen] = data[i];
ctx->datalen++;
if (ctx->datalen == 64) {
sha256_transform(ctx, ctx->data);
ctx->bitlen += 512;
ctx->datalen = 0;
}
}
}
void sha256_final(SHA256_CTX *ctx, BYTE hash[])
{
WORD i;
i = ctx->datalen;
// Pad whatever data is left in the buffer.
if (ctx->datalen < 56) {
ctx->data[i++] = 0x80;
while (i < 56)
ctx->data[i++] = 0x00;
}
else {
ctx->data[i++] = 0x80;
while (i < 64)
ctx->data[i++] = 0x00;
sha256_transform(ctx, ctx->data);
memset(ctx->data, 0, 56);
}
// Append to the padding the total message's length in bits and transform.
ctx->bitlen += ctx->datalen * 8;
ctx->data[63] = ctx->bitlen;
ctx->data[62] = ctx->bitlen >> 8;
ctx->data[61] = ctx->bitlen >> 16;
ctx->data[60] = ctx->bitlen >> 24;
ctx->data[59] = ctx->bitlen >> 32;
ctx->data[58] = ctx->bitlen >> 40;
ctx->data[57] = ctx->bitlen >> 48;
ctx->data[56] = ctx->bitlen >> 56;
sha256_transform(ctx, ctx->data);
// Since this implementation uses little endian byte ordering and SHA uses big endian,
// reverse all the bytes when copying the final state to the output hash.
for (i = 0; i < 4; ++i) {
hash[i] = (ctx->state[0] >> (24 - i * 8)) & 0x000000ff;
hash[i + 4] = (ctx->state[1] >> (24 - i * 8)) & 0x000000ff;
hash[i + 8] = (ctx->state[2] >> (24 - i * 8)) & 0x000000ff;
hash[i + 12] = (ctx->state[3] >> (24 - i * 8)) & 0x000000ff;
hash[i + 16] = (ctx->state[4] >> (24 - i * 8)) & 0x000000ff;
hash[i + 20] = (ctx->state[5] >> (24 - i * 8)) & 0x000000ff;
hash[i + 24] = (ctx->state[6] >> (24 - i * 8)) & 0x000000ff;
hash[i + 28] = (ctx->state[7] >> (24 - i * 8)) & 0x000000ff;
}
}
/*********************************************************************
* Filename: sha256.h
* Author: Brad Conte (brad AT bradconte.com)
* Copyright:
* Disclaimer: This code is presented "as is" without any guarantees.
* Details: Defines the API for the corresponding SHA1 implementation.
*********************************************************************/
#ifndef SHA256_H
#define SHA256_H
/*************************** HEADER FILES ***************************/
#include <stddef.h>
#include <stdint.h>
/****************************** MACROS ******************************/
#define SHA256_BLOCK_SIZE 32 // SHA256 outputs a 32 byte digest
/**************************** DATA TYPES ****************************/
typedef uint8_t BYTE; // 8-bit byte
typedef uint32_t WORD; // 32-bit word
typedef struct {
BYTE data[64];
WORD datalen;
unsigned long long bitlen;
WORD state[8];
} SHA256_CTX;
/*********************** FUNCTION DECLARATIONS **********************/
void sha256_init(SHA256_CTX *ctx);
void sha256_update(SHA256_CTX *ctx, const BYTE data[], size_t len);
void sha256_final(SHA256_CTX *ctx, BYTE hash[]);
#endif // SHA256_H
...@@ -58,7 +58,8 @@ int siptlw(int c) { ...@@ -58,7 +58,8 @@ int siptlw(int c) {
/* Test of the CPU is Little Endian and supports not aligned accesses. /* Test of the CPU is Little Endian and supports not aligned accesses.
* Two interesting conditions to speedup the function that happen to be * Two interesting conditions to speedup the function that happen to be
* in most of x86 servers. */ * in most of x86 servers. */
#if defined(__X86_64__) || defined(__x86_64__) || defined (__i386__) #if defined(__X86_64__) || defined(__x86_64__) || defined (__i386__) \
|| defined (__aarch64__) || defined (__arm64__)
#define UNALIGNED_LE_CPU #define UNALIGNED_LE_CPU
#endif #endif
......
...@@ -109,5 +109,6 @@ streamCG *streamCreateCG(stream *s, char *name, size_t namelen, streamID *id); ...@@ -109,5 +109,6 @@ streamCG *streamCreateCG(stream *s, char *name, size_t namelen, streamID *id);
streamNACK *streamCreateNACK(streamConsumer *consumer); streamNACK *streamCreateNACK(streamConsumer *consumer);
void streamDecodeID(void *buf, streamID *id); void streamDecodeID(void *buf, streamID *id);
int streamCompareID(streamID *a, streamID *b); int streamCompareID(streamID *a, streamID *b);
void streamFreeNACK(streamNACK *na);
#endif #endif
...@@ -772,8 +772,8 @@ void genericHgetallCommand(client *c, int flags) { ...@@ -772,8 +772,8 @@ void genericHgetallCommand(client *c, int flags) {
hashTypeIterator *hi; hashTypeIterator *hi;
int length, count = 0; int length, count = 0;
if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.null[c->resp])) == NULL if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.emptymap[c->resp]))
|| checkType(c,o,OBJ_HASH)) return; == NULL || checkType(c,o,OBJ_HASH)) return;
/* We return a map if the user requested keys and values, like in the /* We return a map if the user requested keys and values, like in the
* HGETALL case. Otherwise to use a flat array makes more sense. */ * HGETALL case. Otherwise to use a flat array makes more sense. */
......
...@@ -402,7 +402,7 @@ void lrangeCommand(client *c) { ...@@ -402,7 +402,7 @@ void lrangeCommand(client *c) {
if ((getLongFromObjectOrReply(c, c->argv[2], &start, NULL) != C_OK) || if ((getLongFromObjectOrReply(c, c->argv[2], &start, NULL) != C_OK) ||
(getLongFromObjectOrReply(c, c->argv[3], &end, NULL) != C_OK)) return; (getLongFromObjectOrReply(c, c->argv[3], &end, NULL) != C_OK)) return;
if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.null[c->resp])) == NULL if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.emptyarray)) == NULL
|| checkType(c,o,OBJ_LIST)) return; || checkType(c,o,OBJ_LIST)) return;
llen = listTypeLength(o); llen = listTypeLength(o);
...@@ -414,7 +414,7 @@ void lrangeCommand(client *c) { ...@@ -414,7 +414,7 @@ void lrangeCommand(client *c) {
/* Invariant: start >= 0, so this test will be true when end < 0. /* Invariant: start >= 0, so this test will be true when end < 0.
* The range is empty when start > end or start >= length. */ * The range is empty when start > end or start >= length. */
if (start > end || start >= llen) { if (start > end || start >= llen) {
addReplyNull(c); addReply(c,shared.emptyarray);
return; return;
} }
if (end >= llen) end = llen-1; if (end >= llen) end = llen-1;
...@@ -606,7 +606,7 @@ void rpoplpushCommand(client *c) { ...@@ -606,7 +606,7 @@ void rpoplpushCommand(client *c) {
* Blocking POP operations * Blocking POP operations
*----------------------------------------------------------------------------*/ *----------------------------------------------------------------------------*/
/* This is a helper function for handleClientsBlockedOnLists(). It's work /* This is a helper function for handleClientsBlockedOnKeys(). It's work
* is to serve a specific client (receiver) that is blocked on 'key' * is to serve a specific client (receiver) that is blocked on 'key'
* in the context of the specified 'db', doing the following: * in the context of the specified 'db', doing the following:
* *
......
...@@ -418,10 +418,10 @@ void spopWithCountCommand(client *c) { ...@@ -418,10 +418,10 @@ void spopWithCountCommand(client *c) {
if ((set = lookupKeyWriteOrReply(c,c->argv[1],shared.null[c->resp])) if ((set = lookupKeyWriteOrReply(c,c->argv[1],shared.null[c->resp]))
== NULL || checkType(c,set,OBJ_SET)) return; == NULL || checkType(c,set,OBJ_SET)) return;
/* If count is zero, serve an empty multibulk ASAP to avoid special /* If count is zero, serve an empty set ASAP to avoid special
* cases later. */ * cases later. */
if (count == 0) { if (count == 0) {
addReplyNull(c); addReply(c,shared.emptyset[c->resp]);
return; return;
} }
...@@ -632,13 +632,13 @@ void srandmemberWithCountCommand(client *c) { ...@@ -632,13 +632,13 @@ void srandmemberWithCountCommand(client *c) {
uniq = 0; uniq = 0;
} }
if ((set = lookupKeyReadOrReply(c,c->argv[1],shared.null[c->resp])) if ((set = lookupKeyReadOrReply(c,c->argv[1],shared.emptyset[c->resp]))
== NULL || checkType(c,set,OBJ_SET)) return; == NULL || checkType(c,set,OBJ_SET)) return;
size = setTypeSize(set); size = setTypeSize(set);
/* If count is zero, serve it ASAP to avoid special cases later. */ /* If count is zero, serve it ASAP to avoid special cases later. */
if (count == 0) { if (count == 0) {
addReplyNull(c); addReply(c,shared.emptyset[c->resp]);
return; return;
} }
...@@ -813,7 +813,7 @@ void sinterGenericCommand(client *c, robj **setkeys, ...@@ -813,7 +813,7 @@ void sinterGenericCommand(client *c, robj **setkeys,
} }
addReply(c,shared.czero); addReply(c,shared.czero);
} else { } else {
addReplyNull(c); addReply(c,shared.emptyset[c->resp]);
} }
return; return;
} }
......
...@@ -1357,9 +1357,8 @@ int zsetAdd(robj *zobj, double score, sds ele, int *flags, double *newscore) { ...@@ -1357,9 +1357,8 @@ int zsetAdd(robj *zobj, double score, sds ele, int *flags, double *newscore) {
/* Optimize: check if the element is too large or the list /* Optimize: check if the element is too large or the list
* becomes too long *before* executing zzlInsert. */ * becomes too long *before* executing zzlInsert. */
zobj->ptr = zzlInsert(zobj->ptr,ele,score); zobj->ptr = zzlInsert(zobj->ptr,ele,score);
if (zzlLength(zobj->ptr) > server.zset_max_ziplist_entries) if (zzlLength(zobj->ptr) > server.zset_max_ziplist_entries ||
zsetConvert(zobj,OBJ_ENCODING_SKIPLIST); sdslen(ele) > server.zset_max_ziplist_value)
if (sdslen(ele) > server.zset_max_ziplist_value)
zsetConvert(zobj,OBJ_ENCODING_SKIPLIST); zsetConvert(zobj,OBJ_ENCODING_SKIPLIST);
if (newscore) *newscore = score; if (newscore) *newscore = score;
*flags |= ZADD_ADDED; *flags |= ZADD_ADDED;
...@@ -2427,7 +2426,7 @@ void zrangeGenericCommand(client *c, int reverse) { ...@@ -2427,7 +2426,7 @@ void zrangeGenericCommand(client *c, int reverse) {
return; return;
} }
if ((zobj = lookupKeyReadOrReply(c,key,shared.null[c->resp])) == NULL if ((zobj = lookupKeyReadOrReply(c,key,shared.emptyarray)) == NULL
|| checkType(c,zobj,OBJ_ZSET)) return; || checkType(c,zobj,OBJ_ZSET)) return;
/* Sanitize indexes. */ /* Sanitize indexes. */
...@@ -2439,7 +2438,7 @@ void zrangeGenericCommand(client *c, int reverse) { ...@@ -2439,7 +2438,7 @@ void zrangeGenericCommand(client *c, int reverse) {
/* Invariant: start >= 0, so this test will be true when end < 0. /* Invariant: start >= 0, so this test will be true when end < 0.
* The range is empty when start > end or start >= length. */ * The range is empty when start > end or start >= length. */
if (start > end || start >= llen) { if (start > end || start >= llen) {
addReplyNull(c); addReply(c,shared.emptyarray);
return; return;
} }
if (end >= llen) end = llen-1; if (end >= llen) end = llen-1;
...@@ -2575,7 +2574,7 @@ void genericZrangebyscoreCommand(client *c, int reverse) { ...@@ -2575,7 +2574,7 @@ void genericZrangebyscoreCommand(client *c, int reverse) {
} }
/* Ok, lookup the key and get the range */ /* Ok, lookup the key and get the range */
if ((zobj = lookupKeyReadOrReply(c,key,shared.null[c->resp])) == NULL || if ((zobj = lookupKeyReadOrReply(c,key,shared.emptyarray)) == NULL ||
checkType(c,zobj,OBJ_ZSET)) return; checkType(c,zobj,OBJ_ZSET)) return;
if (zobj->encoding == OBJ_ENCODING_ZIPLIST) { if (zobj->encoding == OBJ_ENCODING_ZIPLIST) {
...@@ -2595,7 +2594,7 @@ void genericZrangebyscoreCommand(client *c, int reverse) { ...@@ -2595,7 +2594,7 @@ void genericZrangebyscoreCommand(client *c, int reverse) {
/* No "first" element in the specified interval. */ /* No "first" element in the specified interval. */
if (eptr == NULL) { if (eptr == NULL) {
addReplyNull(c); addReply(c,shared.emptyarray);
return; return;
} }
...@@ -2662,7 +2661,7 @@ void genericZrangebyscoreCommand(client *c, int reverse) { ...@@ -2662,7 +2661,7 @@ void genericZrangebyscoreCommand(client *c, int reverse) {
/* No "first" element in the specified interval. */ /* No "first" element in the specified interval. */
if (ln == NULL) { if (ln == NULL) {
addReplyNull(c); addReply(c,shared.emptyarray);
return; return;
} }
...@@ -2920,7 +2919,7 @@ void genericZrangebylexCommand(client *c, int reverse) { ...@@ -2920,7 +2919,7 @@ void genericZrangebylexCommand(client *c, int reverse) {
} }
/* Ok, lookup the key and get the range */ /* Ok, lookup the key and get the range */
if ((zobj = lookupKeyReadOrReply(c,key,shared.null[c->resp])) == NULL || if ((zobj = lookupKeyReadOrReply(c,key,shared.emptyarray)) == NULL ||
checkType(c,zobj,OBJ_ZSET)) checkType(c,zobj,OBJ_ZSET))
{ {
zslFreeLexRange(&range); zslFreeLexRange(&range);
...@@ -2943,7 +2942,7 @@ void genericZrangebylexCommand(client *c, int reverse) { ...@@ -2943,7 +2942,7 @@ void genericZrangebylexCommand(client *c, int reverse) {
/* No "first" element in the specified interval. */ /* No "first" element in the specified interval. */
if (eptr == NULL) { if (eptr == NULL) {
addReplyNull(c); addReply(c,shared.emptyarray);
zslFreeLexRange(&range); zslFreeLexRange(&range);
return; return;
} }
...@@ -3007,7 +3006,7 @@ void genericZrangebylexCommand(client *c, int reverse) { ...@@ -3007,7 +3006,7 @@ void genericZrangebylexCommand(client *c, int reverse) {
/* No "first" element in the specified interval. */ /* No "first" element in the specified interval. */
if (ln == NULL) { if (ln == NULL) {
addReplyNull(c); addReply(c,shared.emptyarray);
zslFreeLexRange(&range); zslFreeLexRange(&range);
return; return;
} }
...@@ -3161,7 +3160,7 @@ void genericZpopCommand(client *c, robj **keyv, int keyc, int where, int emitkey ...@@ -3161,7 +3160,7 @@ void genericZpopCommand(client *c, robj **keyv, int keyc, int where, int emitkey
/* No candidate for zpopping, return empty. */ /* No candidate for zpopping, return empty. */
if (!zobj) { if (!zobj) {
addReplyNull(c); addReply(c,shared.emptyarray);
return; return;
} }
......
...@@ -60,6 +60,7 @@ ...@@ -60,6 +60,7 @@
* use the most significant bits instead of the full 24 bits. */ * use the most significant bits instead of the full 24 bits. */
#define TRACKING_TABLE_SIZE (1<<24) #define TRACKING_TABLE_SIZE (1<<24)
rax **TrackingTable = NULL; rax **TrackingTable = NULL;
unsigned long TrackingTableUsedSlots = 0;
robj *TrackingChannelName; robj *TrackingChannelName;
/* Remove the tracking state from the client 'c'. Note that there is not much /* Remove the tracking state from the client 'c'. Note that there is not much
...@@ -109,67 +110,187 @@ void trackingRememberKeys(client *c) { ...@@ -109,67 +110,187 @@ void trackingRememberKeys(client *c) {
sds sdskey = c->argv[idx]->ptr; sds sdskey = c->argv[idx]->ptr;
uint64_t hash = crc64(0, uint64_t hash = crc64(0,
(unsigned char*)sdskey,sdslen(sdskey))&(TRACKING_TABLE_SIZE-1); (unsigned char*)sdskey,sdslen(sdskey))&(TRACKING_TABLE_SIZE-1);
if (TrackingTable[hash] == NULL) if (TrackingTable[hash] == NULL) {
TrackingTable[hash] = raxNew(); TrackingTable[hash] = raxNew();
TrackingTableUsedSlots++;
}
raxTryInsert(TrackingTable[hash], raxTryInsert(TrackingTable[hash],
(unsigned char*)&c->id,sizeof(c->id),NULL,NULL); (unsigned char*)&c->id,sizeof(c->id),NULL,NULL);
} }
getKeysFreeResult(keys); getKeysFreeResult(keys);
} }
/* This function is called from signalModifiedKey() or other places in Redis void sendTrackingMessage(client *c, long long hash) {
* when a key changes value. In the context of keys tracking, our task here is int using_redirection = 0;
* to send a notification to every client that may have keys about such . */ if (c->client_tracking_redirection) {
void trackingInvalidateKey(robj *keyobj) { client *redir = lookupClientByID(c->client_tracking_redirection);
sds sdskey = keyobj->ptr; if (!redir) {
uint64_t hash = crc64(0, /* We need to signal to the original connection that we
(unsigned char*)sdskey,sdslen(sdskey))&(TRACKING_TABLE_SIZE-1); * are unable to send invalidation messages to the redirected
if (TrackingTable == NULL || TrackingTable[hash] == NULL) return; * connection, because the client no longer exist. */
if (c->resp > 2) {
addReplyPushLen(c,3);
addReplyBulkCBuffer(c,"tracking-redir-broken",21);
addReplyLongLong(c,c->client_tracking_redirection);
}
return;
}
c = redir;
using_redirection = 1;
}
/* Only send such info for clients in RESP version 3 or more. However
* if redirection is active, and the connection we redirect to is
* in Pub/Sub mode, we can support the feature with RESP 2 as well,
* by sending Pub/Sub messages in the __redis__:invalidate channel. */
if (c->resp > 2) {
addReplyPushLen(c,2);
addReplyBulkCBuffer(c,"invalidate",10);
addReplyLongLong(c,hash);
} else if (using_redirection && c->flags & CLIENT_PUBSUB) {
robj *msg = createStringObjectFromLongLong(hash);
addReplyPubsubMessage(c,TrackingChannelName,msg);
decrRefCount(msg);
}
}
/* Invalidates a caching slot: this is actually the low level implementation
* of the API that Redis calls externally, that is trackingInvalidateKey(). */
void trackingInvalidateSlot(uint64_t slot) {
if (TrackingTable == NULL || TrackingTable[slot] == NULL) return;
raxIterator ri; raxIterator ri;
raxStart(&ri,TrackingTable[hash]); raxStart(&ri,TrackingTable[slot]);
raxSeek(&ri,"^",NULL,0); raxSeek(&ri,"^",NULL,0);
while(raxNext(&ri)) { while(raxNext(&ri)) {
uint64_t id; uint64_t id;
memcpy(&id,ri.key,ri.key_len); memcpy(&id,ri.key,ri.key_len);
client *c = lookupClientByID(id); client *c = lookupClientByID(id);
if (c == NULL) continue; if (c == NULL || !(c->flags & CLIENT_TRACKING)) continue;
int using_redirection = 0; sendTrackingMessage(c,slot);
if (c->client_tracking_redirection) { }
client *redir = lookupClientByID(c->client_tracking_redirection); raxStop(&ri);
if (!redir) {
/* We need to signal to the original connection that we /* Free the tracking table: we'll create the radix tree and populate it
* are unable to send invalidation messages to the redirected * again if more keys will be modified in this caching slot. */
* connection, because the client no longer exist. */ raxFree(TrackingTable[slot]);
if (c->resp > 2) { TrackingTable[slot] = NULL;
addReplyPushLen(c,3); TrackingTableUsedSlots--;
addReplyBulkCBuffer(c,"tracking-redir-broken",21); }
addReplyLongLong(c,c->client_tracking_redirection);
} /* This function is called from signalModifiedKey() or other places in Redis
continue; * when a key changes value. In the context of keys tracking, our task here is
* to send a notification to every client that may have keys about such caching
* slot. */
void trackingInvalidateKey(robj *keyobj) {
if (TrackingTable == NULL || TrackingTableUsedSlots == 0) return;
sds sdskey = keyobj->ptr;
uint64_t hash = crc64(0,
(unsigned char*)sdskey,sdslen(sdskey))&(TRACKING_TABLE_SIZE-1);
trackingInvalidateSlot(hash);
}
/* This function is called when one or all the Redis databases are flushed
* (dbid == -1 in case of FLUSHALL). Caching slots are not specific for
* each DB but are global: currently what we do is sending a special
* notification to clients with tracking enabled, invalidating the caching
* slot "-1", which means, "all the keys", in order to avoid flooding clients
* with many invalidation messages for all the keys they may hold.
*
* However trying to flush the tracking table here is very costly:
* we need scanning 16 million caching slots in the table to check
* if they are used, this introduces a big delay. So what we do is to really
* flush the table in the case of FLUSHALL. When a FLUSHDB is called instead
* we just send the invalidation message to all the clients, but don't
* flush the table: it will slowly get garbage collected as more keys
* are modified in the used caching slots. */
void trackingInvalidateKeysOnFlush(int dbid) {
if (server.tracking_clients) {
listNode *ln;
listIter li;
listRewind(server.clients,&li);
while ((ln = listNext(&li)) != NULL) {
client *c = listNodeValue(ln);
if (c->flags & CLIENT_TRACKING) {
sendTrackingMessage(c,-1);
}
}
}
/* In case of FLUSHALL, reclaim all the memory used by tracking. */
if (dbid == -1 && TrackingTable) {
for (int j = 0; j < TRACKING_TABLE_SIZE && TrackingTableUsedSlots > 0; j++) {
if (TrackingTable[j] != NULL) {
raxFree(TrackingTable[j]);
TrackingTable[j] = NULL;
TrackingTableUsedSlots--;
} }
c = redir;
using_redirection = 1;
} }
/* Only send such info for clients in RESP version 3 or more. However /* If there are no clients with tracking enabled, we can even
* if redirection is active, and the connection we redirect to is * reclaim the memory used by the table itself. The code assumes
* in Pub/Sub mode, we can support the feature with RESP 2 as well, * the table is allocated only if there is at least one client alive
* by sending Pub/Sub messages in the __redis__:invalidate channel. */ * with tracking enabled. */
if (c->resp > 2) { if (server.tracking_clients == 0) {
addReplyPushLen(c,2); zfree(TrackingTable);
addReplyBulkCBuffer(c,"invalidate",10); TrackingTable = NULL;
addReplyLongLong(c,hash);
} else if (using_redirection && c->flags & CLIENT_PUBSUB) {
robj *msg = createStringObjectFromLongLong(hash);
addReplyPubsubMessage(c,TrackingChannelName,msg);
decrRefCount(msg);
} }
} }
raxStop(&ri); }
/* Free the tracking table: we'll create the radix tree and populate it /* Tracking forces Redis to remember information about which client may have
* again if more keys will be modified in this hash slot. */ * keys about certian caching slots. In workloads where there are a lot of
raxFree(TrackingTable[hash]); * reads, but keys are hardly modified, the amount of information we have
TrackingTable[hash] = NULL; * to remember server side could be a lot: for each 16 millions of caching
* slots we may end with a radix tree containing many entries.
*
* So Redis allows the user to configure a maximum fill rate for the
* invalidation table. This function makes sure that we don't go over the
* specified fill rate: if we are over, we can just evict informations about
* random caching slots, and send invalidation messages to clients like if
* the key was modified. */
void trackingLimitUsedSlots(void) {
static unsigned int timeout_counter = 0;
if (server.tracking_table_max_fill == 0) return; /* No limits set. */
unsigned int max_slots =
(TRACKING_TABLE_SIZE/100) * server.tracking_table_max_fill;
if (TrackingTableUsedSlots <= max_slots) {
timeout_counter = 0;
return; /* Limit not reached. */
}
/* We have to invalidate a few slots to reach the limit again. The effort
* we do here is proportional to the number of times we entered this
* function and found that we are still over the limit. */
int effort = 100 * (timeout_counter+1);
/* Let's start at a random position, and perform linear probing, in order
* to improve cache locality. However once we are able to find an used
* slot, jump again randomly, in order to avoid creating big holes in the
* table (that will make this funciton use more resourced later). */
while(effort > 0) {
unsigned int idx = rand() % TRACKING_TABLE_SIZE;
do {
effort--;
idx = (idx+1) % TRACKING_TABLE_SIZE;
if (TrackingTable[idx] != NULL) {
trackingInvalidateSlot(idx);
if (TrackingTableUsedSlots <= max_slots) {
timeout_counter = 0;
return; /* Return ASAP: we are again under the limit. */
} else {
break; /* Jump to next random position. */
}
}
} while(effort > 0);
}
timeout_counter++;
}
/* This is just used in order to access the amount of used slots in the
* tracking table. */
unsigned long long trackingGetUsedSlots(void) {
return TrackingTableUsedSlots;
} }
...@@ -294,6 +294,26 @@ size_t zmalloc_get_rss(void) { ...@@ -294,6 +294,26 @@ size_t zmalloc_get_rss(void) {
return t_info.resident_size; return t_info.resident_size;
} }
#elif defined(__FreeBSD__)
#include <sys/types.h>
#include <sys/sysctl.h>
#include <sys/user.h>
#include <unistd.h>
size_t zmalloc_get_rss(void) {
struct kinfo_proc info;
size_t infolen = sizeof(info);
int mib[4];
mib[0] = CTL_KERN;
mib[1] = KERN_PROC;
mib[2] = KERN_PROC_PID;
mib[3] = getpid();
if (sysctl(mib, 4, &info, &infolen, NULL, 0) == 0)
return (size_t)info.ki_rssize;
return 0L;
}
#else #else
size_t zmalloc_get_rss(void) { size_t zmalloc_get_rss(void) {
/* If we can't get the RSS in an OS-specific way for this system just /* If we can't get the RSS in an OS-specific way for this system just
......
...@@ -192,12 +192,6 @@ foreach mdl {no yes} { ...@@ -192,12 +192,6 @@ foreach mdl {no yes} {
set master_host [srv 0 host] set master_host [srv 0 host]
set master_port [srv 0 port] set master_port [srv 0 port]
set slaves {} set slaves {}
set load_handle0 [start_bg_complex_data $master_host $master_port 9 100000000]
set load_handle1 [start_bg_complex_data $master_host $master_port 11 100000000]
set load_handle2 [start_bg_complex_data $master_host $master_port 12 100000000]
set load_handle3 [start_write_load $master_host $master_port 8]
set load_handle4 [start_write_load $master_host $master_port 4]
after 5000 ;# wait for some data to accumulate so that we have RDB part for the fork
start_server {} { start_server {} {
lappend slaves [srv 0 client] lappend slaves [srv 0 client]
start_server {} { start_server {} {
...@@ -205,6 +199,14 @@ foreach mdl {no yes} { ...@@ -205,6 +199,14 @@ foreach mdl {no yes} {
start_server {} { start_server {} {
lappend slaves [srv 0 client] lappend slaves [srv 0 client]
test "Connect multiple replicas at the same time (issue #141), master diskless=$mdl, replica diskless=$sdl" { test "Connect multiple replicas at the same time (issue #141), master diskless=$mdl, replica diskless=$sdl" {
# start load handles only inside the test, so that the test can be skipped
set load_handle0 [start_bg_complex_data $master_host $master_port 9 100000000]
set load_handle1 [start_bg_complex_data $master_host $master_port 11 100000000]
set load_handle2 [start_bg_complex_data $master_host $master_port 12 100000000]
set load_handle3 [start_write_load $master_host $master_port 8]
set load_handle4 [start_write_load $master_host $master_port 4]
after 5000 ;# wait for some data to accumulate so that we have RDB part for the fork
# Send SLAVEOF commands to slaves # Send SLAVEOF commands to slaves
[lindex $slaves 0] config set repl-diskless-load $sdl [lindex $slaves 0] config set repl-diskless-load $sdl
[lindex $slaves 1] config set repl-diskless-load $sdl [lindex $slaves 1] config set repl-diskless-load $sdl
...@@ -278,9 +280,9 @@ start_server {tags {"repl"}} { ...@@ -278,9 +280,9 @@ start_server {tags {"repl"}} {
set master [srv 0 client] set master [srv 0 client]
set master_host [srv 0 host] set master_host [srv 0 host]
set master_port [srv 0 port] set master_port [srv 0 port]
set load_handle0 [start_write_load $master_host $master_port 3]
start_server {} { start_server {} {
test "Master stream is correctly processed while the replica has a script in -BUSY state" { test "Master stream is correctly processed while the replica has a script in -BUSY state" {
set load_handle0 [start_write_load $master_host $master_port 3]
set slave [srv 0 client] set slave [srv 0 client]
$slave config set lua-time-limit 500 $slave config set lua-time-limit 500
$slave slaveof $master_host $master_port $slave slaveof $master_host $master_port
...@@ -383,3 +385,84 @@ test {slave fails full sync and diskless load swapdb recoveres it} { ...@@ -383,3 +385,84 @@ test {slave fails full sync and diskless load swapdb recoveres it} {
} }
} }
} }
test {diskless loading short read} {
start_server {tags {"repl"}} {
set replica [srv 0 client]
set replica_host [srv 0 host]
set replica_port [srv 0 port]
start_server {} {
set master [srv 0 client]
set master_host [srv 0 host]
set master_port [srv 0 port]
# Set master and replica to use diskless replication
$master config set repl-diskless-sync yes
$master config set rdbcompression no
$replica config set repl-diskless-load swapdb
# Try to fill the master with all types of data types / encodings
for {set k 0} {$k < 3} {incr k} {
for {set i 0} {$i < 10} {incr i} {
r set "$k int_$i" [expr {int(rand()*10000)}]
r expire "$k int_$i" [expr {int(rand()*10000)}]
r set "$k string_$i" [string repeat A [expr {int(rand()*1000000)}]]
r hset "$k hash_small" [string repeat A [expr {int(rand()*10)}]] 0[string repeat A [expr {int(rand()*10)}]]
r hset "$k hash_large" [string repeat A [expr {int(rand()*10000)}]] [string repeat A [expr {int(rand()*1000000)}]]
r sadd "$k set_small" [string repeat A [expr {int(rand()*10)}]]
r sadd "$k set_large" [string repeat A [expr {int(rand()*1000000)}]]
r zadd "$k zset_small" [expr {rand()}] [string repeat A [expr {int(rand()*10)}]]
r zadd "$k zset_large" [expr {rand()}] [string repeat A [expr {int(rand()*1000000)}]]
r lpush "$k list_small" [string repeat A [expr {int(rand()*10)}]]
r lpush "$k list_large" [string repeat A [expr {int(rand()*1000000)}]]
for {set j 0} {$j < 10} {incr j} {
r xadd "$k stream" * foo "asdf" bar "1234"
}
r xgroup create "$k stream" "mygroup_$i" 0
r xreadgroup GROUP "mygroup_$i" Alice COUNT 1 STREAMS "$k stream" >
}
}
# Start the replication process...
$master config set repl-diskless-sync-delay 0
$replica replicaof $master_host $master_port
# kill the replication at various points
set attempts 3
if {$::accurate} { set attempts 10 }
for {set i 0} {$i < $attempts} {incr i} {
# wait for the replica to start reading the rdb
# using the log file since the replica only responds to INFO once in 2mb
wait_for_log_message -1 "*Loading DB in memory*" 5 2000 1
# add some additional random sleep so that we kill the master on a different place each time
after [expr {int(rand()*100)}]
# kill the replica connection on the master
set killed [$master client kill type replica]
if {[catch {
set res [wait_for_log_message -1 "*Internal error in RDB*" 5 100 10]
if {$::verbose} {
puts $res
}
}]} {
puts "failed triggering short read"
# force the replica to try another full sync
$master client kill type replica
$master set asdf asdf
# the side effect of resizing the backlog is that it is flushed (16k is the min size)
$master config set repl-backlog-size [expr {16384 + $i}]
}
# wait for loading to stop (fail)
wait_for_condition 100 10 {
[s -1 loading] eq 0
} else {
fail "Replica didn't disconnect"
}
}
# enable fast shutdown
$master config set rdb-key-save-delay 0
}
}
}
...@@ -13,16 +13,20 @@ endif ...@@ -13,16 +13,20 @@ endif
.SUFFIXES: .c .so .xo .o .SUFFIXES: .c .so .xo .o
all: commandfilter.so fork.so all: commandfilter.so testrdb.so fork.so
.c.xo: .c.xo:
$(CC) -I../../src $(CFLAGS) $(SHOBJ_CFLAGS) -fPIC -c $< -o $@ $(CC) -I../../src $(CFLAGS) $(SHOBJ_CFLAGS) -fPIC -c $< -o $@
commandfilter.xo: ../../src/redismodule.h commandfilter.xo: ../../src/redismodule.h
fork.xo: ../../src/redismodule.h fork.xo: ../../src/redismodule.h
testrdb.xo: ../../src/redismodule.h
commandfilter.so: commandfilter.xo commandfilter.so: commandfilter.xo
$(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LIBS) -lc $(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LIBS) -lc
fork.so: fork.xo fork.so: fork.xo
$(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LIBS) -lc $(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LIBS) -lc
testrdb.so: testrdb.xo
$(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LIBS) -lc
#include "redismodule.h"
#include <string.h>
#include <assert.h>
/* Module configuration, save aux or not? */
long long conf_aux_count = 0;
/* Registered type */
RedisModuleType *testrdb_type = NULL;
/* Global values to store and persist to aux */
RedisModuleString *before_str = NULL;
RedisModuleString *after_str = NULL;
void *testrdb_type_load(RedisModuleIO *rdb, int encver) {
int count = RedisModule_LoadSigned(rdb);
if (RedisModule_IsIOError(rdb))
return NULL;
assert(count==1);
assert(encver==1);
RedisModuleString *str = RedisModule_LoadString(rdb);
return str;
}
void testrdb_type_save(RedisModuleIO *rdb, void *value) {
RedisModuleString *str = (RedisModuleString*)value;
RedisModule_SaveSigned(rdb, 1);
RedisModule_SaveString(rdb, str);
}
void testrdb_aux_save(RedisModuleIO *rdb, int when) {
if (conf_aux_count==1) assert(when == REDISMODULE_AUX_AFTER_RDB);
if (conf_aux_count==0) assert(0);
if (when == REDISMODULE_AUX_BEFORE_RDB) {
if (before_str) {
RedisModule_SaveSigned(rdb, 1);
RedisModule_SaveString(rdb, before_str);
} else {
RedisModule_SaveSigned(rdb, 0);
}
} else {
if (after_str) {
RedisModule_SaveSigned(rdb, 1);
RedisModule_SaveString(rdb, after_str);
} else {
RedisModule_SaveSigned(rdb, 0);
}
}
}
int testrdb_aux_load(RedisModuleIO *rdb, int encver, int when) {
assert(encver == 1);
if (conf_aux_count==1) assert(when == REDISMODULE_AUX_AFTER_RDB);
if (conf_aux_count==0) assert(0);
RedisModuleCtx *ctx = RedisModule_GetContextFromIO(rdb);
if (when == REDISMODULE_AUX_BEFORE_RDB) {
if (before_str)
RedisModule_FreeString(ctx, before_str);
before_str = NULL;
int count = RedisModule_LoadSigned(rdb);
if (RedisModule_IsIOError(rdb))
return REDISMODULE_ERR;
if (count)
before_str = RedisModule_LoadString(rdb);
} else {
if (after_str)
RedisModule_FreeString(ctx, after_str);
after_str = NULL;
int count = RedisModule_LoadSigned(rdb);
if (RedisModule_IsIOError(rdb))
return REDISMODULE_ERR;
if (count)
after_str = RedisModule_LoadString(rdb);
}
if (RedisModule_IsIOError(rdb))
return REDISMODULE_ERR;
return REDISMODULE_OK;
}
void testrdb_type_free(void *value) {
if (value)
RedisModule_FreeString(NULL, (RedisModuleString*)value);
}
int testrdb_set_before(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
{
if (argc != 2) {
RedisModule_WrongArity(ctx);
return REDISMODULE_OK;
}
if (before_str)
RedisModule_FreeString(ctx, before_str);
before_str = argv[1];
RedisModule_RetainString(ctx, argv[1]);
RedisModule_ReplyWithLongLong(ctx, 1);
return REDISMODULE_OK;
}
int testrdb_get_before(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
{
REDISMODULE_NOT_USED(argv);
if (argc != 1){
RedisModule_WrongArity(ctx);
return REDISMODULE_OK;
}
if (before_str)
RedisModule_ReplyWithString(ctx, before_str);
else
RedisModule_ReplyWithStringBuffer(ctx, "", 0);
return REDISMODULE_OK;
}
int testrdb_set_after(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
{
if (argc != 2){
RedisModule_WrongArity(ctx);
return REDISMODULE_OK;
}
if (after_str)
RedisModule_FreeString(ctx, after_str);
after_str = argv[1];
RedisModule_RetainString(ctx, argv[1]);
RedisModule_ReplyWithLongLong(ctx, 1);
return REDISMODULE_OK;
}
int testrdb_get_after(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
{
REDISMODULE_NOT_USED(argv);
if (argc != 1){
RedisModule_WrongArity(ctx);
return REDISMODULE_OK;
}
if (after_str)
RedisModule_ReplyWithString(ctx, after_str);
else
RedisModule_ReplyWithStringBuffer(ctx, "", 0);
return REDISMODULE_OK;
}
int testrdb_set_key(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
{
if (argc != 3){
RedisModule_WrongArity(ctx);
return REDISMODULE_OK;
}
RedisModuleKey *key = RedisModule_OpenKey(ctx, argv[1], REDISMODULE_WRITE);
RedisModuleString *str = RedisModule_ModuleTypeGetValue(key);
if (str)
RedisModule_FreeString(ctx, str);
RedisModule_ModuleTypeSetValue(key, testrdb_type, argv[2]);
RedisModule_RetainString(ctx, argv[2]);
RedisModule_CloseKey(key);
RedisModule_ReplyWithLongLong(ctx, 1);
return REDISMODULE_OK;
}
int testrdb_get_key(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
{
if (argc != 2){
RedisModule_WrongArity(ctx);
return REDISMODULE_OK;
}
RedisModuleKey *key = RedisModule_OpenKey(ctx, argv[1], REDISMODULE_WRITE);
RedisModuleString *str = RedisModule_ModuleTypeGetValue(key);
RedisModule_CloseKey(key);
RedisModule_ReplyWithString(ctx, str);
return REDISMODULE_OK;
}
int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
REDISMODULE_NOT_USED(argv);
REDISMODULE_NOT_USED(argc);
if (RedisModule_Init(ctx,"testrdb",1,REDISMODULE_APIVER_1) == REDISMODULE_ERR)
return REDISMODULE_ERR;
RedisModule_SetModuleOptions(ctx, REDISMODULE_OPTIONS_HANDLE_IO_ERRORS);
if (argc > 0)
RedisModule_StringToLongLong(argv[0], &conf_aux_count);
if (conf_aux_count==0) {
RedisModuleTypeMethods datatype_methods = {
.version = 1,
.rdb_load = testrdb_type_load,
.rdb_save = testrdb_type_save,
.aof_rewrite = NULL,
.digest = NULL,
.free = testrdb_type_free,
};
testrdb_type = RedisModule_CreateDataType(ctx, "test__rdb", 1, &datatype_methods);
if (testrdb_type == NULL)
return REDISMODULE_ERR;
} else {
RedisModuleTypeMethods datatype_methods = {
.version = REDISMODULE_TYPE_METHOD_VERSION,
.rdb_load = testrdb_type_load,
.rdb_save = testrdb_type_save,
.aof_rewrite = NULL,
.digest = NULL,
.free = testrdb_type_free,
.aux_load = testrdb_aux_load,
.aux_save = testrdb_aux_save,
.aux_save_triggers = (conf_aux_count == 1 ?
REDISMODULE_AUX_AFTER_RDB :
REDISMODULE_AUX_BEFORE_RDB | REDISMODULE_AUX_AFTER_RDB)
};
testrdb_type = RedisModule_CreateDataType(ctx, "test__rdb", 1, &datatype_methods);
if (testrdb_type == NULL)
return REDISMODULE_ERR;
}
if (RedisModule_CreateCommand(ctx,"testrdb.set.before", testrdb_set_before,"deny-oom",0,0,0) == REDISMODULE_ERR)
return REDISMODULE_ERR;
if (RedisModule_CreateCommand(ctx,"testrdb.get.before", testrdb_get_before,"",0,0,0) == REDISMODULE_ERR)
return REDISMODULE_ERR;
if (RedisModule_CreateCommand(ctx,"testrdb.set.after", testrdb_set_after,"deny-oom",0,0,0) == REDISMODULE_ERR)
return REDISMODULE_ERR;
if (RedisModule_CreateCommand(ctx,"testrdb.get.after", testrdb_get_after,"",0,0,0) == REDISMODULE_ERR)
return REDISMODULE_ERR;
if (RedisModule_CreateCommand(ctx,"testrdb.set.key", testrdb_set_key,"deny-oom",1,1,1) == REDISMODULE_ERR)
return REDISMODULE_ERR;
if (RedisModule_CreateCommand(ctx,"testrdb.get.key", testrdb_get_key,"",1,1,1) == REDISMODULE_ERR)
return REDISMODULE_ERR;
return REDISMODULE_OK;
}
...@@ -99,6 +99,25 @@ proc wait_for_ofs_sync {r1 r2} { ...@@ -99,6 +99,25 @@ proc wait_for_ofs_sync {r1 r2} {
} }
} }
proc wait_for_log_message {srv_idx pattern last_lines maxtries delay} {
set retry $maxtries
set stdout [srv $srv_idx stdout]
while {$retry} {
set result [exec tail -$last_lines < $stdout]
set result [split $result "\n"]
foreach line $result {
if {[string match $pattern $line]} {
return $line
}
}
incr retry -1
after $delay
}
if {$retry == 0} {
fail "log message of '$pattern' not found"
}
}
# Random integer between 0 and max (excluded). # Random integer between 0 and max (excluded).
proc randomInt {max} { proc randomInt {max} {
expr {int(rand()*$max)} expr {int(rand()*$max)}
......
set testmodule [file normalize tests/modules/testrdb.so]
proc restart_and_wait {} {
catch {
r debug restart
}
# wait for the server to come back up
set retry 50
while {$retry} {
if {[catch { r ping }]} {
after 100
} else {
break
}
incr retry -1
}
}
tags "modules" {
start_server [list overrides [list loadmodule "$testmodule"]] {
test {modules are able to persist types} {
r testrdb.set.key key1 value1
assert_equal "value1" [r testrdb.get.key key1]
r debug reload
assert_equal "value1" [r testrdb.get.key key1]
}
test {modules global are lost without aux} {
r testrdb.set.before global1
assert_equal "global1" [r testrdb.get.before]
restart_and_wait
assert_equal "" [r testrdb.get.before]
}
}
start_server [list overrides [list loadmodule "$testmodule 2"]] {
test {modules are able to persist globals before and after} {
r testrdb.set.before global1
r testrdb.set.after global2
assert_equal "global1" [r testrdb.get.before]
assert_equal "global2" [r testrdb.get.after]
restart_and_wait
assert_equal "global1" [r testrdb.get.before]
assert_equal "global2" [r testrdb.get.after]
}
}
start_server [list overrides [list loadmodule "$testmodule 1"]] {
test {modules are able to persist globals just after} {
r testrdb.set.after global2
assert_equal "global2" [r testrdb.get.after]
restart_and_wait
assert_equal "global2" [r testrdb.get.after]
}
}
tags {repl} {
test {diskless loading short read with module} {
start_server [list overrides [list loadmodule "$testmodule"]] {
set replica [srv 0 client]
set replica_host [srv 0 host]
set replica_port [srv 0 port]
start_server [list overrides [list loadmodule "$testmodule"]] {
set master [srv 0 client]
set master_host [srv 0 host]
set master_port [srv 0 port]
# Set master and replica to use diskless replication
$master config set repl-diskless-sync yes
$master config set rdbcompression no
$replica config set repl-diskless-load swapdb
for {set k 0} {$k < 30} {incr k} {
r testrdb.set.key key$k [string repeat A [expr {int(rand()*1000000)}]]
}
# Start the replication process...
$master config set repl-diskless-sync-delay 0
$replica replicaof $master_host $master_port
# kill the replication at various points
set attempts 3
if {$::accurate} { set attempts 10 }
for {set i 0} {$i < $attempts} {incr i} {
# wait for the replica to start reading the rdb
# using the log file since the replica only responds to INFO once in 2mb
wait_for_log_message -1 "*Loading DB in memory*" 5 2000 1
# add some additional random sleep so that we kill the master on a different place each time
after [expr {int(rand()*100)}]
# kill the replica connection on the master
set killed [$master client kill type replica]
if {[catch {
set res [wait_for_log_message -1 "*Internal error in RDB*" 5 100 10]
if {$::verbose} {
puts $res
}
}]} {
puts "failed triggering short read"
# force the replica to try another full sync
$master client kill type replica
$master set asdf asdf
# the side effect of resizing the backlog is that it is flushed (16k is the min size)
$master config set repl-backlog-size [expr {16384 + $i}]
}
# wait for loading to stop (fail)
wait_for_condition 100 10 {
[s -1 loading] eq 0
} else {
fail "Replica didn't disconnect"
}
}
# enable fast shutdown
$master config set rdb-key-save-delay 0
}
}
}
}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment