Unverified Commit 959d6035 authored by Oran Agra's avatar Oran Agra Committed by GitHub
Browse files

Merge 6.2.2 release

Release 6.2.2
parents 92bde124 aa730ef1
......@@ -139,9 +139,9 @@ robj *lookupKeyReadWithFlags(redisDb *db, robj *key, int flags) {
keymiss:
if (!(flags & LOOKUP_NONOTIFY)) {
server.stat_keyspace_misses++;
notifyKeyspaceEvent(NOTIFY_KEY_MISS, "keymiss", key, db->id);
}
server.stat_keyspace_misses++;
return NULL;
}
......@@ -164,16 +164,24 @@ robj *lookupKeyWriteWithFlags(redisDb *db, robj *key, int flags) {
robj *lookupKeyWrite(redisDb *db, robj *key) {
return lookupKeyWriteWithFlags(db, key, LOOKUP_NONE);
}
static void SentReplyOnKeyMiss(client *c, robj *reply){
serverAssert(sdsEncodedObject(reply));
sds rep = reply->ptr;
if (sdslen(rep) > 1 && rep[0] == '-'){
addReplyErrorObject(c, reply);
} else {
addReply(c,reply);
}
}
robj *lookupKeyReadOrReply(client *c, robj *key, robj *reply) {
robj *o = lookupKeyRead(c->db, key);
if (!o) addReply(c,reply);
if (!o) SentReplyOnKeyMiss(c, reply);
return o;
}
robj *lookupKeyWriteOrReply(client *c, robj *key, robj *reply) {
robj *o = lookupKeyWrite(c->db, key);
if (!o) addReply(c,reply);
if (!o) SentReplyOnKeyMiss(c, reply);
return o;
}
......@@ -1445,7 +1453,12 @@ void propagateExpire(redisDb *db, robj *key, int lazy) {
incrRefCount(argv[0]);
incrRefCount(argv[1]);
/* If the master decided to expire a key we must propagate it to replicas no matter what..
* Even if module executed a command without asking for propagation. */
int prev_replication_allowed = server.replication_allowed;
server.replication_allowed = 1;
propagate(server.delCommand,db->id,argv,2,PROPAGATE_AOF|PROPAGATE_REPL);
server.replication_allowed = prev_replication_allowed;
decrRefCount(argv[0]);
decrRefCount(argv[1]);
......
......@@ -249,7 +249,7 @@ void xorObjectDigest(redisDb *db, robj *keyobj, unsigned char *digest, robj *o)
}
streamIteratorStop(&si);
} else if (o->type == OBJ_MODULE) {
RedisModuleDigest md;
RedisModuleDigest md = {{0},{0}};
moduleValue *mv = o->ptr;
moduleType *mt = mv->type;
moduleInitDigestContext(md);
......@@ -441,7 +441,7 @@ void debugCommand(client *c) {
" conflicting keys will generate an exception and kill the server."
" * NOSAVE: the database will be loaded from an existing RDB file.",
" Examples:",
" * DEBUG RELOAD: verify that the server is able to persist, flsuh and reload",
" * DEBUG RELOAD: verify that the server is able to persist, flush and reload",
" the database.",
" * DEBUG RELOAD NOSAVE: replace the current database with the contents of an",
" existing RDB file.",
......@@ -473,7 +473,7 @@ NULL
} else if (!strcasecmp(c->argv[1]->ptr,"segfault")) {
*((char*)-1) = 'x';
} else if (!strcasecmp(c->argv[1]->ptr,"panic")) {
serverPanic("DEBUG PANIC called at Unix time %ld", time(NULL));
serverPanic("DEBUG PANIC called at Unix time %lld", (long long)time(NULL));
} else if (!strcasecmp(c->argv[1]->ptr,"restart") ||
!strcasecmp(c->argv[1]->ptr,"crash-and-recover"))
{
......@@ -1809,7 +1809,7 @@ void sigsegvHandler(int sig, siginfo_t *info, void *secret) {
serverLog(LL_WARNING,
"Accessing address: %p", (void*)info->si_addr);
}
if (info->si_pid != -1) {
if (info->si_code <= SI_USER && info->si_pid != -1) {
serverLog(LL_WARNING, "Killed by PID: %ld, UID: %d", (long) info->si_pid, info->si_uid);
}
......
......@@ -347,7 +347,9 @@ long activeDefragSdsListAndDict(list *l, dict *d, int dict_val_type) {
if ((newsds = activeDefragSds(sdsele))) {
/* When defragging an sds value, we need to update the dict key */
uint64_t hash = dictGetHash(d, newsds);
replaceSatelliteDictKeyPtrAndOrDefragDictEntry(d, sdsele, newsds, hash, &defragged);
dictEntry **deref = dictFindEntryRefByPtrAndHash(d, sdsele, hash);
if (deref)
(*deref)->key = newsds;
ln->value = newsds;
defragged++;
}
......
......@@ -45,11 +45,7 @@
#include "dict.h"
#include "zmalloc.h"
#ifndef DICT_BENCHMARK_MAIN
#include "redisassert.h"
#else
#include <assert.h>
#endif
/* Using dictEnableResize() / dictDisableResize() we make possible to
* enable/disable resizing of the hash table as needed. This is very important
......@@ -1175,20 +1171,18 @@ void dictGetStats(char *buf, size_t bufsize, dict *d) {
/* ------------------------------- Benchmark ---------------------------------*/
#ifdef DICT_BENCHMARK_MAIN
#include "sds.h"
#ifdef REDIS_TEST
uint64_t hashCallback(const void *key) {
return dictGenHashFunction((unsigned char*)key, sdslen((char*)key));
return dictGenHashFunction((unsigned char*)key, strlen((char*)key));
}
int compareCallback(void *privdata, const void *key1, const void *key2) {
int l1,l2;
DICT_NOTUSED(privdata);
l1 = sdslen((sds)key1);
l2 = sdslen((sds)key2);
l1 = strlen((char*)key1);
l2 = strlen((char*)key2);
if (l1 != l2) return 0;
return memcmp(key1, key2, l1) == 0;
}
......@@ -1196,7 +1190,19 @@ int compareCallback(void *privdata, const void *key1, const void *key2) {
void freeCallback(void *privdata, void *val) {
DICT_NOTUSED(privdata);
sdsfree(val);
zfree(val);
}
char *stringFromLongLong(long long value) {
char buf[32];
int len;
char *s;
len = sprintf(buf,"%lld",value);
s = zmalloc(len+1);
memcpy(s, buf, len);
s[len] = '\0';
return s;
}
dictType BenchmarkDictType = {
......@@ -1215,22 +1221,26 @@ dictType BenchmarkDictType = {
printf(msg ": %ld items in %lld ms\n", count, elapsed); \
} while(0)
/* dict-benchmark [count] */
int main(int argc, char **argv) {
/* ./redis-server test dict [<count> | --accurate] */
int dictTest(int argc, char **argv, int accurate) {
long j;
long long start, elapsed;
dict *dict = dictCreate(&BenchmarkDictType,NULL);
long count = 0;
if (argc == 2) {
count = strtol(argv[1],NULL,10);
} else {
if (argc == 4) {
if (accurate) {
count = 5000000;
} else {
count = strtol(argv[3],NULL,10);
}
} else {
count = 5000;
}
start_benchmark();
for (j = 0; j < count; j++) {
int retval = dictAdd(dict,sdsfromlonglong(j),(void*)j);
int retval = dictAdd(dict,stringFromLongLong(j),(void*)j);
assert(retval == DICT_OK);
}
end_benchmark("Inserting");
......@@ -1243,28 +1253,28 @@ int main(int argc, char **argv) {
start_benchmark();
for (j = 0; j < count; j++) {
sds key = sdsfromlonglong(j);
char *key = stringFromLongLong(j);
dictEntry *de = dictFind(dict,key);
assert(de != NULL);
sdsfree(key);
zfree(key);
}
end_benchmark("Linear access of existing elements");
start_benchmark();
for (j = 0; j < count; j++) {
sds key = sdsfromlonglong(j);
char *key = stringFromLongLong(j);
dictEntry *de = dictFind(dict,key);
assert(de != NULL);
sdsfree(key);
zfree(key);
}
end_benchmark("Linear access of existing elements (2nd round)");
start_benchmark();
for (j = 0; j < count; j++) {
sds key = sdsfromlonglong(rand() % count);
char *key = stringFromLongLong(rand() % count);
dictEntry *de = dictFind(dict,key);
assert(de != NULL);
sdsfree(key);
zfree(key);
}
end_benchmark("Random access of existing elements");
......@@ -1277,17 +1287,17 @@ int main(int argc, char **argv) {
start_benchmark();
for (j = 0; j < count; j++) {
sds key = sdsfromlonglong(rand() % count);
char *key = stringFromLongLong(rand() % count);
key[0] = 'X';
dictEntry *de = dictFind(dict,key);
assert(de == NULL);
sdsfree(key);
zfree(key);
}
end_benchmark("Accessing missing");
start_benchmark();
for (j = 0; j < count; j++) {
sds key = sdsfromlonglong(j);
char *key = stringFromLongLong(j);
int retval = dictDelete(dict,key);
assert(retval == DICT_OK);
key[0] += 17; /* Change first number to letter. */
......@@ -1295,5 +1305,7 @@ int main(int argc, char **argv) {
assert(retval == DICT_OK);
}
end_benchmark("Removing and adding");
dictRelease(dict);
return 0;
}
#endif
......@@ -201,4 +201,8 @@ extern dictType dictTypeHeapStringCopyKey;
extern dictType dictTypeHeapStrings;
extern dictType dictTypeHeapStringCopyKeyValue;
#ifdef REDIS_TEST
int dictTest(int argc, char *argv[], int accurate);
#endif
#endif /* __DICT_H */
......@@ -105,11 +105,12 @@ uint64_t intrev64(uint64_t v) {
#include <stdio.h>
#define UNUSED(x) (void)(x)
int endianconvTest(int argc, char *argv[]) {
int endianconvTest(int argc, char *argv[], int accurate) {
char buf[32];
UNUSED(argc);
UNUSED(argv);
UNUSED(accurate);
sprintf(buf,"ciaoroma");
memrev16(buf);
......
......@@ -72,7 +72,7 @@ uint64_t intrev64(uint64_t v);
#endif
#ifdef REDIS_TEST
int endianconvTest(int argc, char *argv[]);
int endianconvTest(int argc, char *argv[], int accurate);
#endif
#endif
......@@ -184,7 +184,7 @@ struct commandHelp {
8,
"6.2.0" },
{ "CLIENT KILL",
"[ip:port] [ID client-id] [TYPE normal|master|slave|pubsub] [USER username] [ADDR ip:port] [SKIPME yes/no]",
"[ip:port] [ID client-id] [TYPE normal|master|slave|pubsub] [USER username] [ADDR ip:port] [LADDR ip:port] [SKIPME yes/no]",
"Kill the connection of a client",
8,
"2.4.0" },
......
......@@ -284,7 +284,7 @@ size_t intsetBlobLen(intset *is) {
return sizeof(intset)+intrev32ifbe(is->length)*intrev32ifbe(is->encoding);
}
/* Validate the integrity of the data stracture.
/* Validate the integrity of the data structure.
* when `deep` is 0, only the integrity of the header is validated.
* when `deep` is 1, we make sure there are no duplicate or out of order records. */
int intsetValidateIntegrity(const unsigned char *p, size_t size, int deep) {
......@@ -392,7 +392,7 @@ static void checkConsistency(intset *is) {
}
#define UNUSED(x) (void)(x)
int intsetTest(int argc, char **argv) {
int intsetTest(int argc, char **argv, int accurate) {
uint8_t success;
int i;
intset *is;
......@@ -400,6 +400,7 @@ int intsetTest(int argc, char **argv) {
UNUSED(argc);
UNUSED(argv);
UNUSED(accurate);
printf("Value encodings: "); {
assert(_intsetValueEncoding(-32768) == INTSET_ENC_INT16);
......@@ -424,6 +425,7 @@ int intsetTest(int argc, char **argv) {
is = intsetAdd(is,4,&success); assert(success);
is = intsetAdd(is,4,&success); assert(!success);
ok();
zfree(is);
}
printf("Large number of random adds: "); {
......@@ -436,6 +438,7 @@ int intsetTest(int argc, char **argv) {
assert(intrev32ifbe(is->length) == inserts);
checkConsistency(is);
ok();
zfree(is);
}
printf("Upgrade from int16 to int32: "); {
......@@ -447,6 +450,7 @@ int intsetTest(int argc, char **argv) {
assert(intsetFind(is,32));
assert(intsetFind(is,65535));
checkConsistency(is);
zfree(is);
is = intsetNew();
is = intsetAdd(is,32,NULL);
......@@ -457,6 +461,7 @@ int intsetTest(int argc, char **argv) {
assert(intsetFind(is,-65535));
checkConsistency(is);
ok();
zfree(is);
}
printf("Upgrade from int16 to int64: "); {
......@@ -468,6 +473,7 @@ int intsetTest(int argc, char **argv) {
assert(intsetFind(is,32));
assert(intsetFind(is,4294967295));
checkConsistency(is);
zfree(is);
is = intsetNew();
is = intsetAdd(is,32,NULL);
......@@ -478,6 +484,7 @@ int intsetTest(int argc, char **argv) {
assert(intsetFind(is,-4294967295));
checkConsistency(is);
ok();
zfree(is);
}
printf("Upgrade from int32 to int64: "); {
......@@ -489,6 +496,7 @@ int intsetTest(int argc, char **argv) {
assert(intsetFind(is,65535));
assert(intsetFind(is,4294967295));
checkConsistency(is);
zfree(is);
is = intsetNew();
is = intsetAdd(is,65535,NULL);
......@@ -499,6 +507,7 @@ int intsetTest(int argc, char **argv) {
assert(intsetFind(is,-4294967295));
checkConsistency(is);
ok();
zfree(is);
}
printf("Stress lookups: "); {
......@@ -512,6 +521,7 @@ int intsetTest(int argc, char **argv) {
for (i = 0; i < num; i++) intsetSearch(is,rand() % ((1<<bits)-1),NULL);
printf("%ld lookups, %ld element set, %lldusec\n",
num,size,usec()-start);
zfree(is);
}
printf("Stress add+delete: "); {
......@@ -528,6 +538,7 @@ int intsetTest(int argc, char **argv) {
}
checkConsistency(is);
ok();
zfree(is);
}
return 0;
......
......@@ -49,7 +49,7 @@ size_t intsetBlobLen(intset *is);
int intsetValidateIntegrity(const unsigned char *is, size_t size, int deep);
#ifdef REDIS_TEST
int intsetTest(int argc, char *argv[]);
int intsetTest(int argc, char *argv[], int accurate);
#endif
#endif // __INTSET_H
......@@ -256,7 +256,7 @@ sds createLatencyReport(void) {
if (dictSize(server.latency_events) == 0 &&
server.latency_monitor_threshold == 0)
{
report = sdscat(report,"I'm sorry, Dave, I can't do that. Latency monitoring is disabled in this Redis instance. You may use \"CONFIG SET latency-monitor-threshold <milliseconds>.\" in order to enable it. If we weren't in a deep space mission I'd suggest to take a look at http://redis.io/topics/latency-monitor.\n");
report = sdscat(report,"I'm sorry, Dave, I can't do that. Latency monitoring is disabled in this Redis instance. You may use \"CONFIG SET latency-monitor-threshold <milliseconds>.\" in order to enable it. If we weren't in a deep space mission I'd suggest to take a look at https://redis.io/topics/latency-monitor.\n");
return report;
}
......@@ -426,7 +426,7 @@ sds createLatencyReport(void) {
}
if (advise_slowlog_inspect) {
report = sdscat(report,"- Check your Slow Log to understand what are the commands you are running which are too slow to execute. Please check http://redis.io/commands/slowlog for more information.\n");
report = sdscat(report,"- Check your Slow Log to understand what are the commands you are running which are too slow to execute. Please check https://redis.io/commands/slowlog for more information.\n");
}
/* Intrinsic latency. */
......
......@@ -908,7 +908,7 @@ int lpValidateNext(unsigned char *lp, unsigned char **pp, size_t lpbytes) {
#undef OUT_OF_RANGE
}
/* Validate the integrity of the data stracture.
/* Validate the integrity of the data structure.
* when `deep` is 0, only the integrity of the header is validated.
* when `deep` is 1, we scan all the entries one by one. */
int lpValidateIntegrity(unsigned char *lp, size_t size, int deep){
......
......@@ -27,6 +27,30 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
/* --------------------------------------------------------------------------
* Modules API documentation information
*
* The comments in this file are used to generate the API documentation on the
* Redis website.
*
* Each function starting with RM_ and preceded by a block comment is included
* in the API documentation. To hide an RM_ function, put a blank line between
* the comment and the function definition or put the comment inside the
* function body.
*
* The functions are divided into sections. Each section is preceded by a
* documentation block, which is comment block starting with a markdown level 2
* heading, i.e. a line starting with ##, on the first line of the comment block
* (with the exception of a ----- line which can appear first). Other comment
* blocks, which are not intended for the modules API user, such as this comment
* block, do NOT start with a markdown level 2 heading, so they are included in
* the generated a API documentation.
*
* The documentation comments may contain markdown formatting. Some automatic
* replacements are done, such as the replacement of RM with RedisModule in
* function names. For details, see the script src/modules/gendoc.rb.
* -------------------------------------------------------------------------- */
#include "server.h"
#include "cluster.h"
#include "slowlog.h"
......@@ -169,6 +193,7 @@ typedef struct RedisModuleCtx RedisModuleCtx;
#define REDISMODULE_CTX_THREAD_SAFE (1<<4)
#define REDISMODULE_CTX_BLOCKED_DISCONNECTED (1<<5)
#define REDISMODULE_CTX_MODULE_COMMAND_CALL (1<<6)
#define REDISMODULE_CTX_MULTI_EMITTED (1<<7)
/* This represents a Redis key opened with RM_OpenKey(). */
struct RedisModuleKey {
......@@ -396,7 +421,10 @@ void RM_FreeDict(RedisModuleCtx *ctx, RedisModuleDict *d);
void RM_FreeServerInfo(RedisModuleCtx *ctx, RedisModuleServerInfoData *data);
/* --------------------------------------------------------------------------
* Heap allocation raw functions
* ## Heap allocation raw functions
*
* Memory allocated with these functions are taken into account by Redis key
* eviction algorithms and are reported in Redis memory usage information.
* -------------------------------------------------------------------------- */
/* Use like malloc(). Memory allocated with this function is reported in
......@@ -579,13 +607,13 @@ int moduleDelKeyIfEmpty(RedisModuleKey *key) {
* defined in the main executable having the same names.
* -------------------------------------------------------------------------- */
/* Lookup the requested module API and store the function pointer into the
int RM_GetApi(const char *funcname, void **targetPtrPtr) {
/* Lookup the requested module API and store the function pointer into the
* target pointer. The function returns REDISMODULE_ERR if there is no such
* named API, otherwise REDISMODULE_OK.
*
* This function is not meant to be used by modules developer, it is only
* used implicitly by including redismodule.h. */
int RM_GetApi(const char *funcname, void **targetPtrPtr) {
dictEntry *he = dictFind(server.moduleapi, funcname);
if (!he) return REDISMODULE_ERR;
*targetPtrPtr = dictGetVal(he);
......@@ -599,17 +627,21 @@ void moduleHandlePropagationAfterCommandCallback(RedisModuleCtx *ctx) {
/* We don't need to do anything here if the context was never used
* in order to propagate commands. */
if (!(ctx->flags & REDISMODULE_CTX_MULTI_EMITTED)) return;
/* We don't need to do anything here if the server isn't inside
* a transaction. */
if (!server.propagate_in_transaction) return;
/* If this command is executed from with Lua or MULTI/EXEC we do noy
/* If this command is executed from with Lua or MULTI/EXEC we do not
* need to propagate EXEC */
if (server.in_eval || server.in_exec) return;
/* Handle the replication of the final EXEC, since whatever a command
* emits is always wrapped around MULTI/EXEC. */
beforePropagateMultiOrExec(0);
alsoPropagate(server.execCommand,c->db->id,&shared.exec,1,
PROPAGATE_AOF|PROPAGATE_REPL);
afterPropagateExec();
/* If this is not a module command context (but is instead a simple
* callback context), we have to handle directly the "also propagate"
......@@ -708,6 +740,14 @@ int moduleGetCommandKeysViaAPI(struct redisCommand *cmd, robj **argv, int argc,
return result->numkeys;
}
/* --------------------------------------------------------------------------
* ## Commands API
*
* These functions are used to implement custom Redis commands.
*
* For examples, see https://redis.io/topics/modules-intro.
* -------------------------------------------------------------------------- */
/* Return non-zero if a module command, that was declared with the
* flag "getkeys-api", is called in a special way to get the keys positions
* and not to get executed. Otherwise zero is returned. */
......@@ -882,11 +922,15 @@ int RM_CreateCommand(RedisModuleCtx *ctx, const char *name, RedisModuleCmdFunc c
return REDISMODULE_OK;
}
/* Called by RM_Init() to setup the `ctx->module` structure.
/* --------------------------------------------------------------------------
* ## Module information and time measurement
* -------------------------------------------------------------------------- */
void RM_SetModuleAttribs(RedisModuleCtx *ctx, const char *name, int ver, int apiver) {
/* Called by RM_Init() to setup the `ctx->module` structure.
*
* This is an internal function, Redis modules developers don't need
* to use it. */
void RM_SetModuleAttribs(RedisModuleCtx *ctx, const char *name, int ver, int apiver) {
RedisModule *module;
if (ctx->module != NULL) return;
......@@ -952,20 +996,29 @@ int RM_BlockedClientMeasureTimeEnd(RedisModuleBlockedClient *bc) {
* repl-diskless-load to work if enabled.
* The module should use RedisModule_IsIOError after reads, before using the
* data that was read, and in case of error, propagate it upwards, and also be
* able to release the partially populated value and all it's allocations. */
* able to release the partially populated value and all it's allocations.
*
* REDISMODULE_OPTION_NO_IMPLICIT_SIGNAL_MODIFIED:
* See RM_SignalModifiedKey().
*/
void RM_SetModuleOptions(RedisModuleCtx *ctx, int options) {
ctx->module->options = options;
}
/* Signals that the key is modified from user's perspective (i.e. invalidate WATCH
* and client side caching). */
* and client side caching).
*
* This is done automatically when a key opened for writing is closed, unless
* the option REDISMODULE_OPTION_NO_IMPLICIT_SIGNAL_MODIFIED has been set using
* RM_SetModuleOptions().
*/
int RM_SignalModifiedKey(RedisModuleCtx *ctx, RedisModuleString *keyname) {
signalModifiedKey(ctx->client,ctx->client->db,keyname);
return REDISMODULE_OK;
}
/* --------------------------------------------------------------------------
* Automatic memory management for modules
* ## Automatic memory management for modules
* -------------------------------------------------------------------------- */
/* Enable automatic memory management.
......@@ -1061,7 +1114,7 @@ void autoMemoryCollect(RedisModuleCtx *ctx) {
}
/* --------------------------------------------------------------------------
* String objects APIs
* ## String objects APIs
* -------------------------------------------------------------------------- */
/* Create a new module string object. The returned string must be freed
......@@ -1330,14 +1383,6 @@ int RM_StringToLongDouble(const RedisModuleString *str, long double *ld) {
* Returns REDISMODULE_OK on success and returns REDISMODULE_ERR if the string
* is not a valid string representation of a stream ID. The special IDs "+" and
* "-" are allowed.
*
* RedisModuleStreamID is a struct with two 64-bit fields, which is used in
* stream functions and defined as
*
* typedef struct RedisModuleStreamID {
* uint64_t ms;
* uint64_t seq;
* } RedisModuleStreamID;
*/
int RM_StringToStreamID(const RedisModuleString *str, RedisModuleStreamID *id) {
streamID streamid;
......@@ -1392,13 +1437,15 @@ int RM_StringAppendBuffer(RedisModuleCtx *ctx, RedisModuleString *str, const cha
}
/* --------------------------------------------------------------------------
* Reply APIs
* ## Reply APIs
*
* These functions are used for sending replies to the client.
*
* Most functions always return REDISMODULE_OK so you can use it with
* 'return' in order to return from the command implementation with:
*
* if (... some condition ...)
* return RM_ReplyWithLongLong(ctx,mycount);
* return RedisModule_ReplyWithLongLong(ctx,mycount);
* -------------------------------------------------------------------------- */
/* Send an error about the number of arguments given to the command,
......@@ -1687,7 +1734,7 @@ int RM_ReplyWithLongDouble(RedisModuleCtx *ctx, long double ld) {
}
/* --------------------------------------------------------------------------
* Commands replication API
* ## Commands replication API
* -------------------------------------------------------------------------- */
/* Helper function to replicate MULTI the first time we replicate something
......@@ -1696,7 +1743,7 @@ int RM_ReplyWithLongDouble(RedisModuleCtx *ctx, long double ld) {
void moduleReplicateMultiIfNeeded(RedisModuleCtx *ctx) {
/* Skip this if client explicitly wrap the command with MULTI, or if
* the module command was called by a script. */
if (server.lua_caller || server.in_exec) return;
if (server.in_eval || server.in_exec) return;
/* If we already emitted MULTI return ASAP. */
if (server.propagate_in_transaction) return;
/* If this is a thread safe context, we do not want to wrap commands
......@@ -1707,10 +1754,12 @@ void moduleReplicateMultiIfNeeded(RedisModuleCtx *ctx) {
* context, we have to setup the op array for the "also propagate" API
* so that RM_Replicate() will work. */
if (!(ctx->flags & REDISMODULE_CTX_MODULE_COMMAND_CALL)) {
serverAssert(ctx->saved_oparray.ops == NULL);
ctx->saved_oparray = server.also_propagate;
redisOpArrayInit(&server.also_propagate);
}
execCommandPropagateMulti(ctx->client->db->id);
ctx->flags |= REDISMODULE_CTX_MULTI_EMITTED;
}
/* Replicate the specified command and arguments to slaves and AOF, as effect
......@@ -1734,7 +1783,7 @@ void moduleReplicateMultiIfNeeded(RedisModuleCtx *ctx) {
* the AOF or the replicas from the propagation of the specified command.
* Otherwise, by default, the command will be propagated in both channels.
*
* ## Note about calling this function from a thread safe context:
* #### Note about calling this function from a thread safe context:
*
* Normally when you call this function from the callback implementing a
* module command, or any other callback provided by the Redis Module API,
......@@ -1746,7 +1795,7 @@ void moduleReplicateMultiIfNeeded(RedisModuleCtx *ctx) {
* and the command specified is inserted in the AOF and replication stream
* immediately.
*
* ## Return value
* #### Return value
*
* The command returns REDISMODULE_ERR if the format specifiers are invalid
* or the command name does not belong to a known command. */
......@@ -1810,7 +1859,7 @@ int RM_ReplicateVerbatim(RedisModuleCtx *ctx) {
}
/* --------------------------------------------------------------------------
* DB and Key APIs -- Generic API
* ## DB and Key APIs -- Generic API
* -------------------------------------------------------------------------- */
/* Return the ID of the current client calling the currently active module
......@@ -2077,7 +2126,7 @@ int RM_GetContextFlags(RedisModuleCtx *ctx) {
flags |= REDISMODULE_CTX_FLAGS_LOADING;
/* Maxmemory and eviction policy */
if (server.maxmemory > 0) {
if (server.maxmemory > 0 && (!server.masterhost || !server.repl_slave_ignore_maxmemory)) {
flags |= REDISMODULE_CTX_FLAGS_MAXMEMORY;
if (server.maxmemory_policy != MAXMEMORY_NO_EVICTION)
......@@ -2327,7 +2376,7 @@ mstime_t RM_GetExpire(RedisModuleKey *key) {
* The function returns REDISMODULE_OK on success or REDISMODULE_ERR if
* the key was not open for writing or is an empty key. */
int RM_SetExpire(RedisModuleKey *key, mstime_t expire) {
if (!(key->mode & REDISMODULE_WRITE) || key->value == NULL)
if (!(key->mode & REDISMODULE_WRITE) || key->value == NULL || (expire < 0 && expire != REDISMODULE_NO_EXPIRE))
return REDISMODULE_ERR;
if (expire != REDISMODULE_NO_EXPIRE) {
expire += mstime();
......@@ -2338,6 +2387,36 @@ int RM_SetExpire(RedisModuleKey *key, mstime_t expire) {
return REDISMODULE_OK;
}
/* Return the key expire value, as absolute Unix timestamp.
* If no TTL is associated with the key or if the key is empty,
* REDISMODULE_NO_EXPIRE is returned. */
mstime_t RM_GetAbsExpire(RedisModuleKey *key) {
mstime_t expire = getExpire(key->db,key->key);
if (expire == -1 || key->value == NULL)
return REDISMODULE_NO_EXPIRE;
return expire;
}
/* Set a new expire for the key. If the special expire
* REDISMODULE_NO_EXPIRE is set, the expire is cancelled if there was
* one (the same as the PERSIST command).
*
* Note that the expire must be provided as a positive integer representing
* the absolute Unix timestamp the key should have.
*
* The function returns REDISMODULE_OK on success or REDISMODULE_ERR if
* the key was not open for writing or is an empty key. */
int RM_SetAbsExpire(RedisModuleKey *key, mstime_t expire) {
if (!(key->mode & REDISMODULE_WRITE) || key->value == NULL || (expire < 0 && expire != REDISMODULE_NO_EXPIRE))
return REDISMODULE_ERR;
if (expire != REDISMODULE_NO_EXPIRE) {
setExpire(key->ctx->client,key->db,key->key,expire);
} else {
removeExpire(key->db,key->key);
}
return REDISMODULE_OK;
}
/* Performs similar operation to FLUSHALL, and optionally start a new AOF file (if enabled)
* If restart_aof is true, you must make sure the command that triggered this call is not
* propagated to the AOF file.
......@@ -2361,7 +2440,9 @@ RedisModuleString *RM_RandomKey(RedisModuleCtx *ctx) {
}
/* --------------------------------------------------------------------------
* Key API for String type
* ## Key API for String type
*
* See also RM_ValueLength(), which returns the length of a string.
* -------------------------------------------------------------------------- */
/* If the key is open for writing, set the specified string 'str' as the
......@@ -2471,7 +2552,9 @@ int RM_StringTruncate(RedisModuleKey *key, size_t newlen) {
}
/* --------------------------------------------------------------------------
* Key API for List type
* ## Key API for List type
*
* See also RM_ValueLength(), which returns the length of a list.
* -------------------------------------------------------------------------- */
/* Push an element into a list, on head or tail depending on 'where' argument.
......@@ -2509,26 +2592,28 @@ RedisModuleString *RM_ListPop(RedisModuleKey *key, int where) {
}
/* --------------------------------------------------------------------------
* Key API for Sorted Set type
* ## Key API for Sorted Set type
*
* See also RM_ValueLength(), which returns the length of a sorted set.
* -------------------------------------------------------------------------- */
/* Conversion from/to public flags of the Modules API and our private flags,
* so that we have everything decoupled. */
int moduleZsetAddFlagsToCoreFlags(int flags) {
int retflags = 0;
if (flags & REDISMODULE_ZADD_XX) retflags |= ZADD_XX;
if (flags & REDISMODULE_ZADD_NX) retflags |= ZADD_NX;
if (flags & REDISMODULE_ZADD_GT) retflags |= ZADD_GT;
if (flags & REDISMODULE_ZADD_LT) retflags |= ZADD_LT;
if (flags & REDISMODULE_ZADD_XX) retflags |= ZADD_IN_XX;
if (flags & REDISMODULE_ZADD_NX) retflags |= ZADD_IN_NX;
if (flags & REDISMODULE_ZADD_GT) retflags |= ZADD_IN_GT;
if (flags & REDISMODULE_ZADD_LT) retflags |= ZADD_IN_LT;
return retflags;
}
/* See previous function comment. */
int moduleZsetAddFlagsFromCoreFlags(int flags) {
int retflags = 0;
if (flags & ZADD_ADDED) retflags |= REDISMODULE_ZADD_ADDED;
if (flags & ZADD_UPDATED) retflags |= REDISMODULE_ZADD_UPDATED;
if (flags & ZADD_NOP) retflags |= REDISMODULE_ZADD_NOP;
if (flags & ZADD_OUT_ADDED) retflags |= REDISMODULE_ZADD_ADDED;
if (flags & ZADD_OUT_UPDATED) retflags |= REDISMODULE_ZADD_UPDATED;
if (flags & ZADD_OUT_NOP) retflags |= REDISMODULE_ZADD_NOP;
return retflags;
}
......@@ -2565,16 +2650,16 @@ int moduleZsetAddFlagsFromCoreFlags(int flags) {
* * 'score' double value is not a number (NaN).
*/
int RM_ZsetAdd(RedisModuleKey *key, double score, RedisModuleString *ele, int *flagsptr) {
int flags = 0;
int in_flags = 0, out_flags = 0;
if (!(key->mode & REDISMODULE_WRITE)) return REDISMODULE_ERR;
if (key->value && key->value->type != OBJ_ZSET) return REDISMODULE_ERR;
if (key->value == NULL) moduleCreateEmptyKey(key,REDISMODULE_KEYTYPE_ZSET);
if (flagsptr) flags = moduleZsetAddFlagsToCoreFlags(*flagsptr);
if (zsetAdd(key->value,score,ele->ptr,&flags,NULL) == 0) {
if (flagsptr) in_flags = moduleZsetAddFlagsToCoreFlags(*flagsptr);
if (zsetAdd(key->value,score,ele->ptr,in_flags,&out_flags,NULL) == 0) {
if (flagsptr) *flagsptr = 0;
return REDISMODULE_ERR;
}
if (flagsptr) *flagsptr = moduleZsetAddFlagsFromCoreFlags(flags);
if (flagsptr) *flagsptr = moduleZsetAddFlagsFromCoreFlags(out_flags);
return REDISMODULE_OK;
}
......@@ -2592,22 +2677,17 @@ int RM_ZsetAdd(RedisModuleKey *key, double score, RedisModuleString *ele, int *f
* with the new score of the element after the increment, if no error
* is returned. */
int RM_ZsetIncrby(RedisModuleKey *key, double score, RedisModuleString *ele, int *flagsptr, double *newscore) {
int flags = 0;
int in_flags = 0, out_flags = 0;
if (!(key->mode & REDISMODULE_WRITE)) return REDISMODULE_ERR;
if (key->value && key->value->type != OBJ_ZSET) return REDISMODULE_ERR;
if (key->value == NULL) moduleCreateEmptyKey(key,REDISMODULE_KEYTYPE_ZSET);
if (flagsptr) flags = moduleZsetAddFlagsToCoreFlags(*flagsptr);
flags |= ZADD_INCR;
if (zsetAdd(key->value,score,ele->ptr,&flags,newscore) == 0) {
if (flagsptr) in_flags = moduleZsetAddFlagsToCoreFlags(*flagsptr);
in_flags |= ZADD_IN_INCR;
if (zsetAdd(key->value,score,ele->ptr,in_flags,&out_flags,newscore) == 0) {
if (flagsptr) *flagsptr = 0;
return REDISMODULE_ERR;
}
/* zsetAdd() may signal back that the resulting score is not a number. */
if (flagsptr && (*flagsptr & ZADD_NAN)) {
*flagsptr = 0;
return REDISMODULE_ERR;
}
if (flagsptr) *flagsptr = moduleZsetAddFlagsFromCoreFlags(flags);
if (flagsptr) *flagsptr = moduleZsetAddFlagsFromCoreFlags(out_flags);
return REDISMODULE_OK;
}
......@@ -2657,7 +2737,7 @@ int RM_ZsetScore(RedisModuleKey *key, RedisModuleString *ele, double *score) {
}
/* --------------------------------------------------------------------------
* Key API for Sorted Set iterator
* ## Key API for Sorted Set iterator
* -------------------------------------------------------------------------- */
void zsetKeyReset(RedisModuleKey *key) {
......@@ -2964,7 +3044,9 @@ int RM_ZsetRangePrev(RedisModuleKey *key) {
}
/* --------------------------------------------------------------------------
* Key API for Hash type
* ## Key API for Hash type
*
* See also RM_ValueLength(), which returns the number of fields in a hash.
* -------------------------------------------------------------------------- */
/* Set the field of the specified hash field to the specified value.
......@@ -3199,7 +3281,20 @@ int RM_HashGet(RedisModuleKey *key, int flags, ...) {
}
/* --------------------------------------------------------------------------
* Key API for the stream type.
* ## Key API for Stream type
*
* For an introduction to streams, see https://redis.io/topics/streams-intro.
*
* The type RedisModuleStreamID, which is used in stream functions, is a struct
* with two 64-bit fields and is defined as
*
* typedef struct RedisModuleStreamID {
* uint64_t ms;
* uint64_t seq;
* } RedisModuleStreamID;
*
* See also RM_ValueLength(), which returns the length of a stream, and the
* conversion functions RM_StringToStreamID() and RM_CreateStringFromStreamID().
* -------------------------------------------------------------------------- */
/* Adds an entry to a stream. Like XADD without trimming.
......@@ -3366,8 +3461,8 @@ int RM_StreamDelete(RedisModuleKey *key, RedisModuleStreamID *id) {
* //
* // ... Do stuff ...
* //
* RedisModule_Free(field);
* RedisModule_Free(value);
* RedisModule_FreeString(ctx, field);
* RedisModule_FreeString(ctx, value);
* }
* }
* RedisModule_StreamIteratorStop(key);
......@@ -3648,7 +3743,9 @@ long long RM_StreamTrimByID(RedisModuleKey *key, int flags, RedisModuleStreamID
}
/* --------------------------------------------------------------------------
* Redis <-> Modules generic Call() API
* ## Calling Redis commands from modules
*
* RM_Call() sends a command to Redis. The remaining functions handle the reply.
* -------------------------------------------------------------------------- */
/* Create a new RedisModuleCallReply object. The processing of the reply
......@@ -4067,20 +4164,30 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch
}
}
/* If we are using single commands replication, we need to wrap what
* we propagate into a MULTI/EXEC block, so that it will be atomic like
* a Lua script in the context of AOF and slaves. */
if (replicate) moduleReplicateMultiIfNeeded(ctx);
/* We need to use a global replication_allowed flag in order to prevent
* replication of nested RM_Calls. Example:
* 1. module1.foo does RM_Call of module2.bar without replication (i.e. no '!')
* 2. module2.bar internally calls RM_Call of INCR with '!'
* 3. at the end of module1.foo we call RM_ReplicateVerbatim
* We want the replica/AOF to see only module1.foo and not the INCR from module2.bar */
int prev_replication_allowed = server.replication_allowed;
server.replication_allowed = replicate && server.replication_allowed;
/* Run the command */
int call_flags = CMD_CALL_SLOWLOG | CMD_CALL_STATS | CMD_CALL_NOWRAP;
if (replicate) {
/* If we are using single commands replication, we need to wrap what
* we propagate into a MULTI/EXEC block, so that it will be atomic like
* a Lua script in the context of AOF and slaves. */
moduleReplicateMultiIfNeeded(ctx);
if (!(flags & REDISMODULE_ARGV_NO_AOF))
call_flags |= CMD_CALL_PROPAGATE_AOF;
if (!(flags & REDISMODULE_ARGV_NO_REPLICAS))
call_flags |= CMD_CALL_PROPAGATE_REPL;
}
call(c,call_flags);
server.replication_allowed = prev_replication_allowed;
serverAssert((c->flags & CLIENT_BLOCKED) == 0);
......@@ -4121,7 +4228,7 @@ const char *RM_CallReplyProto(RedisModuleCallReply *reply, size_t *len) {
}
/* --------------------------------------------------------------------------
* Modules data types
* ## Modules data types
*
* When String DMA or using existing data structures is not enough, it is
* possible to create new data types from scratch and export them to
......@@ -4264,6 +4371,12 @@ void moduleTypeNameByID(char *name, uint64_t moduleid) {
}
}
/* Return the name of the module that owns the specified moduleType. */
const char *moduleTypeModuleName(moduleType *mt) {
if (!mt || !mt->module) return NULL;
return mt->module->name;
}
/* Create a copy of a module type value using the copy callback. If failed
* or not supported, produce an error reply and return NULL.
*/
......@@ -4479,7 +4592,7 @@ void *RM_ModuleTypeGetValue(RedisModuleKey *key) {
}
/* --------------------------------------------------------------------------
* RDB loading and saving functions
* ## RDB loading and saving functions
* -------------------------------------------------------------------------- */
/* Called when there is a load error in the context of a module. On some
......@@ -4791,7 +4904,7 @@ ssize_t rdbSaveModulesAux(rio *rdb, int when) {
}
/* --------------------------------------------------------------------------
* Key digest API (DEBUG DIGEST interface for modules types)
* ## Key digest API (DEBUG DIGEST interface for modules types)
* -------------------------------------------------------------------------- */
/* Add a new element to the digest. This function can be called multiple times
......@@ -4912,7 +5025,7 @@ RedisModuleString *RM_SaveDataTypeToString(RedisModuleCtx *ctx, void *data, cons
}
/* --------------------------------------------------------------------------
* AOF API for modules data types
* ## AOF API for modules data types
* -------------------------------------------------------------------------- */
/* Emits a command into the AOF during the AOF rewriting process. This function
......@@ -4967,7 +5080,7 @@ void RM_EmitAOF(RedisModuleIO *io, const char *cmdname, const char *fmt, ...) {
}
/* --------------------------------------------------------------------------
* IO context handling
* ## IO context handling
* -------------------------------------------------------------------------- */
RedisModuleCtx *RM_GetContextFromIO(RedisModuleIO *io) {
......@@ -4994,7 +5107,7 @@ const RedisModuleString *RM_GetKeyNameFromModuleKey(RedisModuleKey *key) {
}
/* --------------------------------------------------------------------------
* Logging
* ## Logging
* -------------------------------------------------------------------------- */
/* This is the low level function implementing both:
......@@ -5025,10 +5138,10 @@ void moduleLogRaw(RedisModule *module, const char *levelstr, const char *fmt, va
* printf-alike specifiers, while level is a string describing the log
* level to use when emitting the log, and must be one of the following:
*
* * "debug"
* * "verbose"
* * "notice"
* * "warning"
* * "debug" (`REDISMODULE_LOGLEVEL_DEBUG`)
* * "verbose" (`REDISMODULE_LOGLEVEL_VERBOSE`)
* * "notice" (`REDISMODULE_LOGLEVEL_NOTICE`)
* * "warning" (`REDISMODULE_LOGLEVEL_WARNING`)
*
* If the specified log level is invalid, verbose is used by default.
* There is a fixed limit to the length of the log line this function is able
......@@ -5079,7 +5192,10 @@ void RM_LatencyAddSample(const char *event, mstime_t latency) {
}
/* --------------------------------------------------------------------------
* Blocking clients from modules
* ## Blocking clients from modules
*
* For a guide about blocking commands in modules, see
* https://redis.io/topics/modules-blocking-ops.
* -------------------------------------------------------------------------- */
/* Readable handler for the awake pipe. We do nothing here, the awake bytes
......@@ -5140,11 +5256,6 @@ void unblockClientFromModule(client *c) {
moduleUnblockClient(c);
bc->client = NULL;
/* Reset the client for a new query since, for blocking commands implemented
* into modules, we do not it immediately after the command returns (and
* the client blocks) in order to be still able to access the argument
* vector from callbacks. */
resetClient(c);
}
/* Block a client in the context of a module: this function implements both
......@@ -5544,6 +5655,12 @@ void moduleHandleBlockedClients(void) {
* API to unblock the client and the memory will be released. */
void moduleBlockedClientTimedOut(client *c) {
RedisModuleBlockedClient *bc = c->bpop.module_blocked_handle;
/* Protect against re-processing: don't serve clients that are already
* in the unblocking list for any reason (including RM_UnblockClient()
* explicit call). See #6798. */
if (bc->unblocked) return;
RedisModuleCtx ctx = REDISMODULE_CTX_INIT;
ctx.flags |= REDISMODULE_CTX_BLOCKED_TIMEOUT;
ctx.module = bc->module;
......@@ -5600,7 +5717,7 @@ int RM_BlockedClientDisconnected(RedisModuleCtx *ctx) {
}
/* --------------------------------------------------------------------------
* Thread Safe Contexts
* ## Thread Safe Contexts
* -------------------------------------------------------------------------- */
/* Return a context which can be used inside threads to make Redis context
......@@ -5710,7 +5827,7 @@ void moduleReleaseGIL(void) {
/* --------------------------------------------------------------------------
* Module Keyspace Notifications API
* ## Module Keyspace Notifications API
* -------------------------------------------------------------------------- */
/* Subscribe to keyspace notifications. This is a low-level version of the
......@@ -5734,6 +5851,7 @@ void moduleReleaseGIL(void) {
* - REDISMODULE_NOTIFY_EXPIRED: Expiration events
* - REDISMODULE_NOTIFY_EVICTED: Eviction events
* - REDISMODULE_NOTIFY_STREAM: Stream events
* - REDISMODULE_NOTIFY_MODULE: Module types events
* - REDISMODULE_NOTIFY_KEYMISS: Key-miss events
* - REDISMODULE_NOTIFY_ALL: All events (Excluding REDISMODULE_NOTIFY_KEYMISS)
* - REDISMODULE_NOTIFY_LOADED: A special notification available only for modules,
......@@ -5843,7 +5961,7 @@ void moduleUnsubscribeNotifications(RedisModule *module) {
}
/* --------------------------------------------------------------------------
* Modules Cluster API
* ## Modules Cluster API
* -------------------------------------------------------------------------- */
/* The Cluster message callback function pointer type. */
......@@ -6098,7 +6216,7 @@ void RM_SetClusterFlags(RedisModuleCtx *ctx, uint64_t flags) {
}
/* --------------------------------------------------------------------------
* Modules Timers API
* ## Modules Timers API
*
* Module timers are an high precision "green timers" abstraction where
* every module can register even millions of timers without problems, even if
......@@ -6272,7 +6390,7 @@ int RM_GetTimerInfo(RedisModuleCtx *ctx, RedisModuleTimerID id, uint64_t *remain
}
/* --------------------------------------------------------------------------
* Modules ACL API
* ## Modules ACL API
*
* Implements a hook into the authentication and authorization within Redis.
* --------------------------------------------------------------------------*/
......@@ -6502,7 +6620,7 @@ RedisModuleString *RM_GetClientCertificate(RedisModuleCtx *ctx, uint64_t client_
}
/* --------------------------------------------------------------------------
* Modules Dictionary API
* ## Modules Dictionary API
*
* Implements a sorted dictionary (actually backed by a radix tree) with
* the usual get / set / del / num-items API, together with an iterator
......@@ -6756,7 +6874,7 @@ int RM_DictCompare(RedisModuleDictIter *di, const char *op, RedisModuleString *k
/* --------------------------------------------------------------------------
* Modules Info fields
* ## Modules Info fields
* -------------------------------------------------------------------------- */
int RM_InfoEndDictField(RedisModuleInfoCtx *ctx);
......@@ -7070,7 +7188,7 @@ double RM_ServerInfoGetFieldDouble(RedisModuleServerInfoData *data, const char*
}
/* --------------------------------------------------------------------------
* Modules utility APIs
* ## Modules utility APIs
* -------------------------------------------------------------------------- */
/* Return random bytes using SHA1 in counter mode with a /dev/urandom
......@@ -7089,7 +7207,7 @@ void RM_GetRandomHexChars(char *dst, size_t len) {
}
/* --------------------------------------------------------------------------
* Modules API exporting / importing
* ## Modules API exporting / importing
* -------------------------------------------------------------------------- */
/* This function is called by a module in order to export some API with a
......@@ -7226,7 +7344,7 @@ int moduleUnregisterFilters(RedisModule *module) {
}
/* --------------------------------------------------------------------------
* Module Command Filter API
* ## Module Command Filter API
* -------------------------------------------------------------------------- */
/* Register a new command filter function.
......@@ -7436,7 +7554,7 @@ float RM_GetUsedMemoryRatio(){
}
/* --------------------------------------------------------------------------
* Scanning keyspace and hashes
* ## Scanning keyspace and hashes
* -------------------------------------------------------------------------- */
typedef void (*RedisModuleScanCB)(RedisModuleCtx *ctx, RedisModuleString *keyname, RedisModuleKey *key, void *privdata);
......@@ -7707,7 +7825,7 @@ int RM_ScanKey(RedisModuleKey *key, RedisModuleScanCursor *cursor, RedisModuleSc
/* --------------------------------------------------------------------------
* Module fork API
* ## Module fork API
* -------------------------------------------------------------------------- */
/* Create a background child process with the current frozen snaphost of the
......@@ -7767,7 +7885,7 @@ int TerminateModuleForkChild(int child_pid, int wait) {
serverLog(LL_VERBOSE,"Killing running module fork child: %ld",
(long) server.child_pid);
if (kill(server.child_pid,SIGUSR1) != -1 && wait) {
while(wait4(server.child_pid,&statloc,0,NULL) !=
while(waitpid(server.child_pid, &statloc, 0) !=
server.child_pid);
}
/* Reset the buffer accumulating changes while the child saves. */
......@@ -7801,7 +7919,7 @@ void ModuleForkDoneHandler(int exitcode, int bysignal) {
}
/* --------------------------------------------------------------------------
* Server hooks implementation
* ## Server hooks implementation
* -------------------------------------------------------------------------- */
/* Register to be notified, via a callback, when the specified server event
......@@ -8682,6 +8800,10 @@ size_t moduleCount(void) {
return dictSize(modules);
}
/* --------------------------------------------------------------------------
* ## Key eviction API
* -------------------------------------------------------------------------- */
/* Set the key last access time for LRU based eviction. not relevant if the
* servers's maxmemory policy is LFU based. Value is idle time in milliseconds.
* returns REDISMODULE_OK if the LRU was updated, REDISMODULE_ERR otherwise. */
......@@ -8732,6 +8854,10 @@ int RM_GetLFU(RedisModuleKey *key, long long *lfu_freq) {
return REDISMODULE_OK;
}
/* --------------------------------------------------------------------------
* ## Miscellaneous APIs
* -------------------------------------------------------------------------- */
/**
* Returns the full ContextFlags mask, using the return value
* the module can check if a certain set of flags are supported
......@@ -8880,6 +9006,10 @@ int *RM_GetCommandKeys(RedisModuleCtx *ctx, RedisModuleString **argv, int argc,
return res;
}
/* --------------------------------------------------------------------------
* ## Defrag API
* -------------------------------------------------------------------------- */
/* The defrag context, used to manage state during calls to the data type
* defrag callback.
*/
......
# coding: utf-8
# gendoc.rb -- Converts the top-comments inside module.c to modules API
# reference documentation in markdown format.
......@@ -21,15 +22,48 @@ def markdown(s)
l = l.gsub(/(?<![`A-z])[a-z_]+\(\)/, '`\0`')
# Add backquotes around macro and var names containing underscores.
l = l.gsub(/(?<![`A-z\*])[A-Za-z]+_[A-Za-z0-9_]+/){|x| "`#{x}`"}
# Link URLs preceded by space (i.e. when not already linked)
l = l.gsub(/ (https?:\/\/[A-Za-z0-9_\/\.\-]+[A-Za-z0-9\/])/,
' [\1](\1)')
# Link URLs preceded by space or newline (not already linked)
l = l.gsub(/(^| )(https?:\/\/[A-Za-z0-9_\/\.\-]+[A-Za-z0-9\/])/,
'\1[\2](\2)')
# Replace double-dash with unicode ndash
l = l.gsub(/ -- /, ' – ')
end
# Link function names to their definition within the page
l = l.gsub(/`(RedisModule_[A-z0-9]+)[()]*`/) {|x|
$index[$1] ? "[#{x}](\##{$1})" : x
}
newlines << l
}
return newlines.join("\n")
end
# Linebreak a prototype longer than 80 characters on the commas, but only
# between balanced parentheses so that we don't linebreak args which are
# function pointers, and then aligning each arg under each other.
def linebreak_proto(proto, indent)
if proto.bytesize <= 80
return proto
end
parts = proto.split(/,\s*/);
if parts.length == 1
return proto;
end
align_pos = proto.index("(") + 1;
align = " " * align_pos
result = parts.shift;
bracket_balance = 0;
parts.each{|part|
if bracket_balance == 0
result += ",\n" + indent + align
else
result += ", "
end
result += part
bracket_balance += part.count("(") - part.count(")")
}
return result;
end
# Given the source code array and the index at which an exported symbol was
# detected, extracts and outputs the documentation.
def docufy(src,i)
......@@ -38,7 +72,11 @@ def docufy(src,i)
name = name.sub("RM_","RedisModule_")
proto = src[i].sub("{","").strip+";\n"
proto = proto.sub("RM_","RedisModule_")
puts "## `#{name}`\n\n"
proto = linebreak_proto(proto, " ");
# Add a link target with the function name. (We don't trust the exact id of
# the generated one, which depends on the Markdown implementation.)
puts "<span id=\"#{name}\"></span>\n\n"
puts "### `#{name}`\n\n"
puts " #{proto}\n"
comment = ""
while true
......@@ -50,13 +88,87 @@ def docufy(src,i)
puts comment+"\n\n"
end
# Print a comment from line until */ is found, as markdown.
def section_doc(src, i)
name = get_section_heading(src, i)
comment = "<span id=\"#{section_name_to_id(name)}\"></span>\n\n"
while true
# append line, except if it's a horizontal divider
comment = comment + src[i] if src[i] !~ /^[\/ ]?\*{1,2} ?-{50,}/
break if src[i] =~ /\*\//
i = i+1
end
comment = markdown(comment)
puts comment+"\n\n"
end
# generates an id suitable for links within the page
def section_name_to_id(name)
return "section-" +
name.strip.downcase.gsub(/[^a-z0-9]+/, '-').gsub(/^-+|-+$/, '')
end
# Returns the name of the first section heading in the comment block for which
# is_section_doc(src, i) is true
def get_section_heading(src, i)
if src[i] =~ /^\/\*\*? \#+ *(.*)/
heading = $1
elsif src[i+1] =~ /^ ?\* \#+ *(.*)/
heading = $1
end
return heading.gsub(' -- ', ' – ')
end
# Returns true if the line is the start of a generic documentation section. Such
# section must start with the # symbol, i.e. a markdown heading, on the first or
# the second line.
def is_section_doc(src, i)
return src[i] =~ /^\/\*\*? \#/ ||
(src[i] =~ /^\/\*/ && src[i+1] =~ /^ ?\* \#/)
end
def is_func_line(src, i)
line = src[i]
return line =~ /RM_/ &&
line[0] != ' ' && line[0] != '#' && line[0] != '/' &&
src[i-1] =~ /\*\//
end
puts "# Modules API reference\n\n"
puts "<!-- This file is generated from module.c using gendoc.rb -->\n\n"
src = File.open("../module.c").to_a
src.each_with_index{|line,i|
if line =~ /RM_/ && line[0] != ' ' && line[0] != '#' && line[0] != '/'
if src[i-1] =~ /\*\//
docufy(src,i)
src = File.open(File.dirname(__FILE__) ++ "/../module.c").to_a
# Build function index
$index = {}
src.each_with_index do |line,i|
if is_func_line(src, i)
line =~ /RM_([A-z0-9]+)/
name = "RedisModule_#{$1}"
$index[name] = true
end
end
# Print TOC
puts "## Sections\n\n"
src.each_with_index do |_line,i|
if is_section_doc(src, i)
name = get_section_heading(src, i)
puts "* [#{name}](\##{section_name_to_id(name)})\n"
end
}
end
puts "* [Function index](#section-function-index)\n\n"
# Docufy: Print function prototype and markdown docs
src.each_with_index do |_line,i|
if is_func_line(src, i)
docufy(src, i)
elsif is_section_doc(src, i)
section_doc(src, i)
end
end
# Print function index
puts "<span id=\"section-function-index\"></span>\n\n"
puts "## Function index\n\n"
$index.keys.sort.each{|x| puts "* [`#{x}`](\##{x})\n"}
puts "\n"
......@@ -113,34 +113,34 @@ void discardCommand(client *c) {
addReply(c,shared.ok);
}
void beforePropagateMultiOrExec(int multi) {
if (multi) {
void beforePropagateMulti() {
/* Propagating MULTI */
serverAssert(!server.propagate_in_transaction);
server.propagate_in_transaction = 1;
} else {
}
void afterPropagateExec() {
/* Propagating EXEC */
serverAssert(server.propagate_in_transaction == 1);
server.propagate_in_transaction = 0;
}
}
/* Send a MULTI command to all the slaves and AOF file. Check the execCommand
* implementation for more information. */
void execCommandPropagateMulti(int dbid) {
beforePropagateMultiOrExec(1);
beforePropagateMulti();
propagate(server.multiCommand,dbid,&shared.multi,1,
PROPAGATE_AOF|PROPAGATE_REPL);
}
void execCommandPropagateExec(int dbid) {
beforePropagateMultiOrExec(0);
propagate(server.execCommand,dbid,&shared.exec,1,
PROPAGATE_AOF|PROPAGATE_REPL);
afterPropagateExec();
}
/* Aborts a transaction, with a specific error message.
* The transaction is always aboarted with -EXECABORT so that the client knows
* The transaction is always aborted with -EXECABORT so that the client knows
* the server exited the multi state, but the actual reason for the abort is
* included too.
* Note: 'error' may or may not end with \r\n. see addReplyErrorFormat. */
......@@ -202,11 +202,9 @@ void execCommand(client *c) {
c->cmd = c->mstate.commands[j].cmd;
/* ACL permissions are also checked at the time of execution in case
* they were changed after the commands were ququed. */
* they were changed after the commands were queued. */
int acl_errpos;
int acl_retval = ACLCheckCommandPerm(c,&acl_errpos);
if (acl_retval == ACL_OK && c->cmd->proc == publishCommand)
acl_retval = ACLCheckPubsubPerm(c,1,1,0,&acl_errpos);
int acl_retval = ACLCheckAllPerm(c,&acl_errpos);
if (acl_retval != ACL_OK) {
char *reason;
switch (acl_retval) {
......@@ -217,7 +215,8 @@ void execCommand(client *c) {
reason = "no permission to touch the specified keys";
break;
case ACL_DENIED_CHANNEL:
reason = "no permission to publish to the specified channel";
reason = "no permission to access one of the channels used "
"as arguments";
break;
default:
reason = "no permission";
......@@ -254,7 +253,6 @@ void execCommand(client *c) {
if (server.propagate_in_transaction) {
int is_master = server.masterhost == NULL;
server.dirty++;
beforePropagateMultiOrExec(0);
/* If inside the MULTI/EXEC block this instance was suddenly
* switched from master to slave (using the SLAVEOF command), the
* initial MULTI was propagated into the replication backlog, but the
......@@ -264,6 +262,7 @@ void execCommand(client *c) {
char *execcmd = "*1\r\n$4\r\nEXEC\r\n";
feedReplicationBacklog(execcmd,strlen(execcmd));
}
afterPropagateExec();
}
server.in_exec = 0;
......
......@@ -154,6 +154,7 @@ client *createClient(connection *conn) {
c->read_reploff = 0;
c->repl_ack_off = 0;
c->repl_ack_time = 0;
c->repl_last_partial_write = 0;
c->slave_listening_port = 0;
c->slave_addr = NULL;
c->slave_capa = SLAVE_CAPA_NONE;
......@@ -331,8 +332,9 @@ void _addReplyProtoToList(client *c, const char *s, size_t len) {
memcpy(tail->buf, s, len);
listAddNodeTail(c->reply, tail);
c->reply_bytes += tail->size;
}
asyncCloseClientOnOutputBufferLimitReached(c);
}
}
/* -----------------------------------------------------------------------------
......@@ -562,7 +564,7 @@ void *addReplyDeferredLen(client *c) {
void setDeferredReply(client *c, void *node, const char *s, size_t length) {
listNode *ln = (listNode*)node;
clientReplyBlock *next;
clientReplyBlock *next, *prev;
/* Abort when *node is NULL: when the client should not accept writes
* we return NULL in addReplyDeferredLen() */
......@@ -571,14 +573,31 @@ void setDeferredReply(client *c, void *node, const char *s, size_t length) {
/* Normally we fill this dummy NULL node, added by addReplyDeferredLen(),
* with a new buffer structure containing the protocol needed to specify
* the length of the array following. However sometimes when there is
* little memory to move, we may instead remove this NULL node, and prefix
* our protocol in the node immediately after to it, in order to save a
* write(2) syscall later. Conditions needed to do it:
* the length of the array following. However sometimes there might be room
* in the previous/next node so we can instead remove this NULL node, and
* suffix/prefix our data in the node immediately before/after it, in order
* to save a write(2) syscall later. Conditions needed to do it:
*
* - The prev node is non-NULL and has space in it or
* - The next node is non-NULL,
* - It has enough room already allocated
* - And not too large (avoid large memmove) */
if (ln->prev != NULL && (prev = listNodeValue(ln->prev)) &&
prev->size - prev->used > 0)
{
size_t len_to_copy = prev->size - prev->used;
if (len_to_copy > length)
len_to_copy = length;
memcpy(prev->buf + prev->used, s, len_to_copy);
prev->used += len_to_copy;
length -= len_to_copy;
if (length == 0) {
listDelNode(c->reply, ln);
return;
}
s += len_to_copy;
}
if (ln->next != NULL && (next = listNodeValue(ln->next)) &&
next->size - next->used >= length &&
next->used < PROTO_REPLY_CHUNK_BYTES * 4)
......@@ -596,8 +615,9 @@ void setDeferredReply(client *c, void *node, const char *s, size_t length) {
memcpy(buf->buf, s, length);
listNodeValue(ln) = buf;
c->reply_bytes += buf->size;
}
asyncCloseClientOnOutputBufferLimitReached(c);
}
}
/* Populate the length object and try gluing it to the next chunk. */
......@@ -1531,9 +1551,7 @@ int writeToClient(client *c, int handler_installed) {
}
atomicIncr(server.stat_net_output_bytes, totwritten);
if (nwritten == -1) {
if (connGetState(c->conn) == CONN_STATE_CONNECTED) {
nwritten = 0;
} else {
if (connGetState(c->conn) != CONN_STATE_CONNECTED) {
serverLog(LL_VERBOSE,
"Error writing to client: %s", connGetLastError(c->conn));
freeClientAsync(c);
......@@ -1645,6 +1663,9 @@ void resetClient(client *c) {
c->flags |= CLIENT_REPLY_SKIP;
c->flags &= ~CLIENT_REPLY_SKIP_NEXT;
}
/* Always clear the prevent logging field. */
c->flags &= ~CLIENT_PREVENT_LOGGING;
}
/* This function is used when we want to re-enter the event loop but there
......@@ -1954,13 +1975,10 @@ void commandProcessed(client *c) {
c->reploff = c->read_reploff - sdslen(c->querybuf) + c->qb_pos;
}
/* Don't reset the client structure for clients blocked in a
* module blocking command, so that the reply callback will
* still be able to access the client argv and argc field.
* The client will be reset in unblockClientFromModule(). */
if (!(c->flags & CLIENT_BLOCKED) ||
(c->btype != BLOCKED_MODULE && c->btype != BLOCKED_PAUSE))
{
/* Don't reset the client structure for blocked clients, so that the reply
* callback will still be able to access the client argv and argc fields.
* The client will be reset in unblockClient(). */
if (!(c->flags & CLIENT_BLOCKED)) {
resetClient(c);
}
......@@ -1990,12 +2008,20 @@ void commandProcessed(client *c) {
* of processing the command, otherwise C_OK is returned. */
int processCommandAndResetClient(client *c) {
int deadclient = 0;
client *old_client = server.current_client;
server.current_client = c;
if (processCommand(c) == C_OK) {
commandProcessed(c);
}
if (server.current_client == NULL) deadclient = 1;
server.current_client = NULL;
/*
* Restore the old client, this is needed because when a script
* times out, we will get into this code from processEventsWhileBlocked.
* Which will cause to set the server.current_client. If not restored
* we will return 1 to our caller which will falsely indicate the client
* is dead and will stop reading from its buffer.
*/
server.current_client = old_client;
/* performEvictions may flush slave output buffers. This may
* result in a slave, that may be the active client, to be
* freed. */
......@@ -2941,6 +2967,7 @@ void helloCommand(client *c) {
int moreargs = (c->argc-1) - j;
const char *opt = c->argv[j]->ptr;
if (!strcasecmp(opt,"AUTH") && moreargs >= 2) {
preventCommandLogging(c);
if (ACLAuthenticateUser(c, c->argv[j+1], c->argv[j+2]) == C_ERR) {
addReplyError(c,"-WRONGPASS invalid username-password pair or user is disabled.");
return;
......@@ -3007,7 +3034,7 @@ void securityWarningCommand(client *c) {
static time_t logged_time;
time_t now = time(NULL);
if (labs(now-logged_time) > 60) {
if (llabs(now-logged_time) > 60) {
serverLog(LL_WARNING,"Possible SECURITY ATTACK detected. It looks like somebody is sending POST or Host: commands to Redis. This is likely due to an attacker attempting to use Cross Protocol Scripting to compromise your Redis instance. Connection aborted.");
logged_time = now;
}
......@@ -3304,6 +3331,8 @@ int areClientsPaused(void) {
* if it has. Also returns true if clients are now paused and false
* otherwise. */
int checkClientPauseTimeoutAndReturnIfPaused(void) {
if (!areClientsPaused())
return 0;
if (server.client_pause_end_time < server.mstime) {
unpauseClients();
}
......@@ -3332,7 +3361,7 @@ void processEventsWhileBlocked(void) {
/* Note: when we are processing events while blocked (for instance during
* busy Lua scripts), we set a global flag. When such flag is set, we
* avoid handling the read part of clients using threaded I/O.
* See https://github.com/antirez/redis/issues/6988 for more info. */
* See https://github.com/redis/redis/issues/6988 for more info. */
ProcessingEventsWhileBlocked = 1;
while (iterations--) {
long long startval = server.events_processed_while_blocked;
......
......@@ -56,6 +56,7 @@ int keyspaceEventsStringToFlags(char *classes) {
case 'E': flags |= NOTIFY_KEYEVENT; break;
case 't': flags |= NOTIFY_STREAM; break;
case 'm': flags |= NOTIFY_KEY_MISS; break;
case 'd': flags |= NOTIFY_MODULE; break;
default: return -1;
}
}
......@@ -82,6 +83,7 @@ sds keyspaceEventsFlagsToString(int flags) {
if (flags & NOTIFY_EXPIRED) res = sdscatlen(res,"x",1);
if (flags & NOTIFY_EVICTED) res = sdscatlen(res,"e",1);
if (flags & NOTIFY_STREAM) res = sdscatlen(res,"t",1);
if (flags & NOTIFY_MODULE) res = sdscatlen(res,"d",1);
}
if (flags & NOTIFY_KEYSPACE) res = sdscatlen(res,"K",1);
if (flags & NOTIFY_KEYEVENT) res = sdscatlen(res,"E",1);
......
......@@ -727,7 +727,11 @@ int getRangeLongFromObjectOrReply(client *c, robj *o, long min, long max, long *
}
int getPositiveLongFromObjectOrReply(client *c, robj *o, long *target, const char *msg) {
if (msg) {
return getRangeLongFromObjectOrReply(c, o, 0, LONG_MAX, target, msg);
} else {
return getRangeLongFromObjectOrReply(c, o, 0, LONG_MAX, target, "value is out of range, must be positive");
}
}
int getIntFromObjectOrReply(client *c, robj *o, int *target, const char *msg) {
......
......@@ -331,21 +331,6 @@ int pubsubPublishMessage(robj *channel, robj *message) {
return receivers;
}
/* This wraps handling ACL channel permissions for the given client. */
int pubsubCheckACLPermissionsOrReply(client *c, int idx, int count, int literal) {
/* Check if the user can run the command according to the current
* ACLs. */
int acl_chanpos;
int acl_retval = ACLCheckPubsubPerm(c,idx,count,literal,&acl_chanpos);
if (acl_retval == ACL_DENIED_CHANNEL) {
addACLLogEntry(c,acl_retval,acl_chanpos,NULL);
addReplyError(c,
"-NOPERM this user has no permissions to access "
"one of the channels used as arguments");
}
return acl_retval;
}
/*-----------------------------------------------------------------------------
* Pubsub commands implementation
*----------------------------------------------------------------------------*/
......@@ -353,7 +338,6 @@ int pubsubCheckACLPermissionsOrReply(client *c, int idx, int count, int literal)
/* SUBSCRIBE channel [channel ...] */
void subscribeCommand(client *c) {
int j;
if (pubsubCheckACLPermissionsOrReply(c,1,c->argc-1,0) != ACL_OK) return;
if ((c->flags & CLIENT_DENY_BLOCKING) && !(c->flags & CLIENT_MULTI)) {
/**
* A client that has CLIENT_DENY_BLOCKING flag on
......@@ -387,7 +371,6 @@ void unsubscribeCommand(client *c) {
/* PSUBSCRIBE pattern [pattern ...] */
void psubscribeCommand(client *c) {
int j;
if (pubsubCheckACLPermissionsOrReply(c,1,c->argc-1,1) != ACL_OK) return;
if ((c->flags & CLIENT_DENY_BLOCKING) && !(c->flags & CLIENT_MULTI)) {
/**
* A client that has CLIENT_DENY_BLOCKING flag on
......@@ -420,7 +403,6 @@ void punsubscribeCommand(client *c) {
/* PUBLISH <channel> <message> */
void publishCommand(client *c) {
if (pubsubCheckACLPermissionsOrReply(c,1,1,0) != ACL_OK) return;
int receivers = pubsubPublishMessage(c->argv[1],c->argv[2]);
if (server.cluster_enabled)
clusterPropagatePublish(c->argv[1],c->argv[2]);
......
......@@ -315,7 +315,9 @@ REDIS_STATIC void __quicklistCompress(const quicklist *quicklist,
if (forward == node || reverse == node)
in_depth = 1;
if (forward == reverse)
/* We passed into compress depth of opposite side of the quicklist
* so there's no need to compress anything and we can exit. */
if (forward == reverse || forward->next == reverse)
return;
forward = forward->next;
......@@ -325,11 +327,9 @@ REDIS_STATIC void __quicklistCompress(const quicklist *quicklist,
if (!in_depth)
quicklistCompressNode(node);
if (depth > 2) {
/* At this point, forward and reverse are one node beyond depth */
quicklistCompressNode(forward);
quicklistCompressNode(reverse);
}
}
#define quicklistCompress(_ql, _node) \
......@@ -380,10 +380,11 @@ REDIS_STATIC void __quicklistInsertNode(quicklist *quicklist,
quicklist->head = quicklist->tail = new_node;
}
/* Update len first, so in __quicklistCompress we know exactly len */
quicklist->len++;
if (old_node)
quicklistCompress(quicklist, old_node);
quicklist->len++;
}
/* Wrappers for node inserting around existing node. */
......@@ -602,15 +603,16 @@ REDIS_STATIC void __quicklistDelNode(quicklist *quicklist,
quicklist->head = node->next;
}
/* Update len first, so in __quicklistCompress we know exactly len */
quicklist->len--;
quicklist->count -= node->count;
/* If we deleted a node within our compress depth, we
* now have compressed nodes needing to be decompressed. */
__quicklistCompress(quicklist, NULL);
quicklist->count -= node->count;
zfree(node->zl);
zfree(node);
quicklist->len--;
}
/* Delete one entry from list given the node for the entry and a pointer
......@@ -1296,17 +1298,24 @@ void quicklistRotate(quicklist *quicklist) {
/* First, get the tail entry */
unsigned char *p = ziplistIndex(quicklist->tail->zl, -1);
unsigned char *value;
unsigned char *value, *tmp;
long long longval;
unsigned int sz;
char longstr[32] = {0};
ziplistGet(p, &value, &sz, &longval);
ziplistGet(p, &tmp, &sz, &longval);
/* If value found is NULL, then ziplistGet populated longval instead */
if (!value) {
if (!tmp) {
/* Write the longval as a string so we can re-add it */
sz = ll2string(longstr, sizeof(longstr), longval);
value = (unsigned char *)longstr;
} else if (quicklist->len == 1) {
/* Copy buffer since there could be a memory overlap when move
* entity from tail to head in the same ziplist. */
value = zmalloc(sz);
memcpy(value, tmp, sz);
} else {
value = tmp;
}
/* Add tail entry to head (must happen before tail is deleted). */
......@@ -1321,6 +1330,8 @@ void quicklistRotate(quicklist *quicklist) {
/* Remove tail entry. */
quicklistDelIndex(quicklist, quicklist->tail, &p);
if (value != (unsigned char*)longstr && value != tmp)
zfree(value);
}
/* pop from quicklist and return result in 'data' ptr. Value of 'data'
......@@ -1509,8 +1520,6 @@ void quicklistBookmarksClear(quicklist *ql) {
#define yell(str, ...) printf("ERROR! " str "\n\n", __VA_ARGS__)
#define OK printf("\tOK\n")
#define ERROR \
do { \
printf("\tERROR!\n"); \
......@@ -1630,7 +1639,6 @@ static int _ql_verify(quicklist *ql, uint32_t len, uint32_t count,
}
if (ql->len == 0 && !errors) {
OK;
return errors;
}
......@@ -1679,8 +1687,6 @@ static int _ql_verify(quicklist *ql, uint32_t len, uint32_t count,
}
}
if (!errors)
OK;
return errors;
}
......@@ -1692,9 +1698,10 @@ static char *genstr(char *prefix, int i) {
}
/* main test, but callable from other files */
int quicklistTest(int argc, char *argv[]) {
int quicklistTest(int argc, char *argv[], int accurate) {
UNUSED(argc);
UNUSED(argv);
UNUSED(accurate);
unsigned int err = 0;
int optimize_start =
......@@ -1703,11 +1710,14 @@ int quicklistTest(int argc, char *argv[]) {
printf("Starting optimization offset at: %d\n", optimize_start);
int options[] = {0, 1, 2, 3, 4, 5, 6, 10};
int fills[] = {-5, -4, -3, -2, -1, 0,
1, 2, 32, 66, 128, 999};
size_t option_count = sizeof(options) / sizeof(*options);
int fill_count = (int)(sizeof(fills) / sizeof(*fills));
long long runtime[option_count];
for (int _i = 0; _i < (int)option_count; _i++) {
printf("Testing Option %d\n", options[_i]);
printf("Testing Compression option %d\n", options[_i]);
long long start = mstime();
TEST("create list") {
......@@ -1732,57 +1742,53 @@ int quicklistTest(int argc, char *argv[]) {
quicklistRelease(ql);
}
for (int f = optimize_start; f < 32; f++) {
TEST_DESC("add to tail 5x at fill %d at compress %d", f,
options[_i]) {
quicklist *ql = quicklistNew(f, options[_i]);
TEST_DESC("add to tail 5x at compress %d", options[_i]) {
for (int f = 0; f < fill_count; f++) {
quicklist *ql = quicklistNew(fills[f], options[_i]);
for (int i = 0; i < 5; i++)
quicklistPushTail(ql, genstr("hello", i), 32);
if (ql->count != 5)
ERROR;
if (f == 32)
if (fills[f] == 32)
ql_verify(ql, 1, 5, 5, 5);
quicklistRelease(ql);
}
}
for (int f = optimize_start; f < 32; f++) {
TEST_DESC("add to head 5x at fill %d at compress %d", f,
options[_i]) {
quicklist *ql = quicklistNew(f, options[_i]);
TEST_DESC("add to head 5x at compress %d", options[_i]) {
for (int f = 0; f < fill_count; f++) {
quicklist *ql = quicklistNew(fills[f], options[_i]);
for (int i = 0; i < 5; i++)
quicklistPushHead(ql, genstr("hello", i), 32);
if (ql->count != 5)
ERROR;
if (f == 32)
if (fills[f] == 32)
ql_verify(ql, 1, 5, 5, 5);
quicklistRelease(ql);
}
}
for (int f = optimize_start; f < 512; f++) {
TEST_DESC("add to tail 500x at fill %d at compress %d", f,
options[_i]) {
quicklist *ql = quicklistNew(f, options[_i]);
TEST_DESC("add to tail 500x at compress %d", options[_i]) {
for (int f = 0; f < fill_count; f++) {
quicklist *ql = quicklistNew(fills[f], options[_i]);
for (int i = 0; i < 500; i++)
quicklistPushTail(ql, genstr("hello", i), 64);
if (ql->count != 500)
ERROR;
if (f == 32)
if (fills[f] == 32)
ql_verify(ql, 16, 500, 32, 20);
quicklistRelease(ql);
}
}
for (int f = optimize_start; f < 512; f++) {
TEST_DESC("add to head 500x at fill %d at compress %d", f,
options[_i]) {
quicklist *ql = quicklistNew(f, options[_i]);
TEST_DESC("add to head 500x at compress %d", options[_i]) {
for (int f = 0; f < fill_count; f++) {
quicklist *ql = quicklistNew(fills[f], options[_i]);
for (int i = 0; i < 500; i++)
quicklistPushHead(ql, genstr("hello", i), 32);
if (ql->count != 500)
ERROR;
if (f == 32)
if (fills[f] == 32)
ql_verify(ql, 16, 500, 20, 32);
quicklistRelease(ql);
}
......@@ -1795,9 +1801,9 @@ int quicklistTest(int argc, char *argv[]) {
quicklistRelease(ql);
}
for (int f = optimize_start; f < 32; f++) {
TEST("rotate one val once") {
quicklist *ql = quicklistNew(f, options[_i]);
for (int f = 0; f < fill_count; f++) {
quicklist *ql = quicklistNew(fills[f], options[_i]);
quicklistPushHead(ql, "hello", 6);
quicklistRotate(ql);
/* Ignore compression verify because ziplist is
......@@ -1807,10 +1813,9 @@ int quicklistTest(int argc, char *argv[]) {
}
}
for (int f = optimize_start; f < 3; f++) {
TEST_DESC("rotate 500 val 5000 times at fill %d at compress %d", f,
options[_i]) {
quicklist *ql = quicklistNew(f, options[_i]);
TEST_DESC("rotate 500 val 5000 times at compress %d", options[_i]) {
for (int f = 0; f < fill_count; f++) {
quicklist *ql = quicklistNew(fills[f], options[_i]);
quicklistPushHead(ql, "900", 3);
quicklistPushHead(ql, "7000", 4);
quicklistPushHead(ql, "-1200", 5);
......@@ -1822,11 +1827,11 @@ int quicklistTest(int argc, char *argv[]) {
ql_info(ql);
quicklistRotate(ql);
}
if (f == 1)
if (fills[f] == 1)
ql_verify(ql, 504, 504, 1, 1);
else if (f == 2)
else if (fills[f] == 2)
ql_verify(ql, 252, 504, 2, 2);
else if (f == 32)
else if (fills[f] == 32)
ql_verify(ql, 16, 504, 32, 24);
quicklistRelease(ql);
}
......@@ -2003,11 +2008,10 @@ int quicklistTest(int argc, char *argv[]) {
quicklistRelease(ql);
}
for (int f = optimize_start; f < 12; f++) {
TEST_DESC("insert once in elements while iterating at fill %d at "
"compress %d\n",
f, options[_i]) {
quicklist *ql = quicklistNew(f, options[_i]);
TEST_DESC("insert once in elements while iterating at compress %d",
options[_i]) {
for (int f = 0; f < fill_count; f++) {
quicklist *ql = quicklistNew(fills[f], options[_i]);
quicklistPushTail(ql, "abc", 3);
quicklistSetFill(ql, 1);
quicklistPushTail(ql, "def", 3); /* force to unique node */
......@@ -2059,12 +2063,10 @@ int quicklistTest(int argc, char *argv[]) {
}
}
for (int f = optimize_start; f < 1024; f++) {
TEST_DESC(
"insert [before] 250 new in middle of 500 elements at fill"
" %d at compress %d",
f, options[_i]) {
quicklist *ql = quicklistNew(f, options[_i]);
TEST_DESC("insert [before] 250 new in middle of 500 elements at compress %d",
options[_i]) {
for (int f = 0; f < fill_count; f++) {
quicklist *ql = quicklistNew(fills[f], options[_i]);
for (int i = 0; i < 500; i++)
quicklistPushTail(ql, genstr("hello", i), 32);
for (int i = 0; i < 250; i++) {
......@@ -2072,17 +2074,16 @@ int quicklistTest(int argc, char *argv[]) {
quicklistIndex(ql, 250, &entry);
quicklistInsertBefore(ql, &entry, genstr("abc", i), 32);
}
if (f == 32)
if (fills[f] == 32)
ql_verify(ql, 25, 750, 32, 20);
quicklistRelease(ql);
}
}
for (int f = optimize_start; f < 1024; f++) {
TEST_DESC("insert [after] 250 new in middle of 500 elements at "
"fill %d at compress %d",
f, options[_i]) {
quicklist *ql = quicklistNew(f, options[_i]);
TEST_DESC("insert [after] 250 new in middle of 500 elements at compress %d",
options[_i]) {
for (int f = 0; f < fill_count; f++) {
quicklist *ql = quicklistNew(fills[f], options[_i]);
for (int i = 0; i < 500; i++)
quicklistPushHead(ql, genstr("hello", i), 32);
for (int i = 0; i < 250; i++) {
......@@ -2094,7 +2095,7 @@ int quicklistTest(int argc, char *argv[]) {
if (ql->count != 750)
ERR("List size not 750, but rather %ld", ql->count);
if (f == 32)
if (fills[f] == 32)
ql_verify(ql, 26, 750, 20, 32);
quicklistRelease(ql);
}
......@@ -2132,70 +2133,58 @@ int quicklistTest(int argc, char *argv[]) {
quicklistRelease(copy);
}
for (int f = optimize_start; f < 512; f++) {
for (int f = 0; f < fill_count; f++) {
TEST_DESC("index 1,200 from 500 list at fill %d at compress %d", f,
options[_i]) {
quicklist *ql = quicklistNew(f, options[_i]);
quicklist *ql = quicklistNew(fills[f], options[_i]);
for (int i = 0; i < 500; i++)
quicklistPushTail(ql, genstr("hello", i + 1), 32);
quicklistEntry entry;
quicklistIndex(ql, 1, &entry);
if (!strcmp((char *)entry.value, "hello2"))
OK;
else
if (strcmp((char *)entry.value, "hello2") != 0)
ERR("Value: %s", entry.value);
quicklistIndex(ql, 200, &entry);
if (!strcmp((char *)entry.value, "hello201"))
OK;
else
if (strcmp((char *)entry.value, "hello201") != 0)
ERR("Value: %s", entry.value);
quicklistRelease(ql);
}
TEST_DESC("index -1,-2 from 500 list at fill %d at compress %d", f,
options[_i]) {
quicklist *ql = quicklistNew(f, options[_i]);
TEST_DESC("index -1,-2 from 500 list at fill %d at compress %d",
fills[f], options[_i]) {
quicklist *ql = quicklistNew(fills[f], options[_i]);
for (int i = 0; i < 500; i++)
quicklistPushTail(ql, genstr("hello", i + 1), 32);
quicklistEntry entry;
quicklistIndex(ql, -1, &entry);
if (!strcmp((char *)entry.value, "hello500"))
OK;
else
if (strcmp((char *)entry.value, "hello500") != 0)
ERR("Value: %s", entry.value);
quicklistIndex(ql, -2, &entry);
if (!strcmp((char *)entry.value, "hello499"))
OK;
else
if (strcmp((char *)entry.value, "hello499") != 0)
ERR("Value: %s", entry.value);
quicklistRelease(ql);
}
TEST_DESC("index -100 from 500 list at fill %d at compress %d", f,
options[_i]) {
quicklist *ql = quicklistNew(f, options[_i]);
TEST_DESC("index -100 from 500 list at fill %d at compress %d",
fills[f], options[_i]) {
quicklist *ql = quicklistNew(fills[f], options[_i]);
for (int i = 0; i < 500; i++)
quicklistPushTail(ql, genstr("hello", i + 1), 32);
quicklistEntry entry;
quicklistIndex(ql, -100, &entry);
if (!strcmp((char *)entry.value, "hello401"))
OK;
else
if (strcmp((char *)entry.value, "hello401") != 0)
ERR("Value: %s", entry.value);
quicklistRelease(ql);
}
TEST_DESC("index too big +1 from 50 list at fill %d at compress %d",
f, options[_i]) {
quicklist *ql = quicklistNew(f, options[_i]);
fills[f], options[_i]) {
quicklist *ql = quicklistNew(fills[f], options[_i]);
for (int i = 0; i < 50; i++)
quicklistPushTail(ql, genstr("hello", i + 1), 32);
quicklistEntry entry;
if (quicklistIndex(ql, 50, &entry))
ERR("Index found at 50 with 50 list: %.*s", entry.sz,
entry.value);
else
OK;
quicklistRelease(ql);
}
}
......@@ -2367,12 +2356,11 @@ int quicklistTest(int argc, char *argv[]) {
quicklistReplaceAtIndex(ql, 1, "foo", 3);
quicklistReplaceAtIndex(ql, -1, "bar", 3);
quicklistRelease(ql);
OK;
}
for (int f = optimize_start; f < 16; f++) {
TEST_DESC("lrem test at fill %d at compress %d", f, options[_i]) {
quicklist *ql = quicklistNew(f, options[_i]);
TEST_DESC("lrem test at compress %d", options[_i]) {
for (int f = 0; f < fill_count; f++) {
quicklist *ql = quicklistNew(fills[f], options[_i]);
char *words[] = {"abc", "foo", "bar", "foobar", "foobared",
"zap", "bar", "test", "foo"};
char *result[] = {"abc", "foo", "foobar", "foobared",
......@@ -2397,14 +2385,12 @@ int quicklistTest(int argc, char *argv[]) {
/* check result of lrem 0 bar */
iter = quicklistGetIterator(ql, AL_START_HEAD);
i = 0;
int ok = 1;
while (quicklistNext(iter, &entry)) {
/* Result must be: abc, foo, foobar, foobared, zap, test,
* foo */
if (strncmp((char *)entry.value, result[i], entry.sz)) {
ERR("No match at position %d, got %.*s instead of %s",
i, entry.sz, entry.value, result[i]);
ok = 0;
}
i++;
}
......@@ -2441,23 +2427,18 @@ int quicklistTest(int argc, char *argv[]) {
entry.sz)) {
ERR("No match at position %d, got %.*s instead of %s",
i, entry.sz, entry.value, resultB[resB - 1 - i]);
ok = 0;
}
i++;
}
quicklistReleaseIterator(iter);
/* final result of all tests */
if (ok)
OK;
quicklistRelease(ql);
}
}
for (int f = optimize_start; f < 16; f++) {
TEST_DESC("iterate reverse + delete at fill %d at compress %d", f,
options[_i]) {
quicklist *ql = quicklistNew(f, options[_i]);
TEST_DESC("iterate reverse + delete at compress %d", options[_i]) {
for (int f = 0; f < fill_count; f++) {
quicklist *ql = quicklistNew(fills[f], options[_i]);
quicklistPushTail(ql, "abc", 3);
quicklistPushTail(ql, "def", 3);
quicklistPushTail(ql, "hij", 3);
......@@ -2494,10 +2475,9 @@ int quicklistTest(int argc, char *argv[]) {
}
}
for (int f = optimize_start; f < 800; f++) {
TEST_DESC("iterator at index test at fill %d at compress %d", f,
options[_i]) {
quicklist *ql = quicklistNew(f, options[_i]);
TEST_DESC("iterator at index test at compress %d", options[_i]) {
for (int f = 0; f < fill_count; f++) {
quicklist *ql = quicklistNew(fills[f], options[_i]);
char num[32];
long long nums[5000];
for (int i = 0; i < 760; i++) {
......@@ -2521,10 +2501,9 @@ int quicklistTest(int argc, char *argv[]) {
}
}
for (int f = optimize_start; f < 40; f++) {
TEST_DESC("ltrim test A at fill %d at compress %d", f,
options[_i]) {
quicklist *ql = quicklistNew(f, options[_i]);
TEST_DESC("ltrim test A at compress %d", options[_i]) {
for (int f = 0; f < fill_count; f++) {
quicklist *ql = quicklistNew(fills[f], options[_i]);
char num[32];
long long nums[5000];
for (int i = 0; i < 32; i++) {
......@@ -2532,7 +2511,7 @@ int quicklistTest(int argc, char *argv[]) {
int sz = ll2string(num, sizeof(num), nums[i]);
quicklistPushTail(ql, num, sz);
}
if (f == 32)
if (fills[f] == 32)
ql_verify(ql, 1, 32, 32, 32);
/* ltrim 25 53 (keep [25,32] inclusive = 7 remaining) */
quicklistDelRange(ql, 0, 25);
......@@ -2545,18 +2524,17 @@ int quicklistTest(int argc, char *argv[]) {
"%lld",
entry.longval, nums[25 + i]);
}
if (f == 32)
if (fills[f] == 32)
ql_verify(ql, 1, 7, 7, 7);
quicklistRelease(ql);
}
}
for (int f = optimize_start; f < 40; f++) {
TEST_DESC("ltrim test B at fill %d at compress %d", f,
options[_i]) {
TEST_DESC("ltrim test B at compress %d", options[_i]) {
for (int f = 0; f < fill_count; f++) {
/* Force-disable compression because our 33 sequential
* integers don't compress and the check always fails. */
quicklist *ql = quicklistNew(f, QUICKLIST_NOCOMPRESS);
quicklist *ql = quicklistNew(fills[f], QUICKLIST_NOCOMPRESS);
char num[32];
long long nums[5000];
for (int i = 0; i < 33; i++) {
......@@ -2564,24 +2542,20 @@ int quicklistTest(int argc, char *argv[]) {
int sz = ll2string(num, sizeof(num), nums[i]);
quicklistPushTail(ql, num, sz);
}
if (f == 32)
if (fills[f] == 32)
ql_verify(ql, 2, 33, 32, 1);
/* ltrim 5 16 (keep [5,16] inclusive = 12 remaining) */
quicklistDelRange(ql, 0, 5);
quicklistDelRange(ql, -16, 16);
if (f == 32)
if (fills[f] == 32)
ql_verify(ql, 1, 12, 12, 12);
quicklistEntry entry;
quicklistIndex(ql, 0, &entry);
if (entry.longval != 5)
ERR("A: longval not 5, but %lld", entry.longval);
else
OK;
quicklistIndex(ql, -1, &entry);
if (entry.longval != 16)
ERR("B! got instead: %lld", entry.longval);
else
OK;
quicklistPushTail(ql, "bobobob", 7);
quicklistIndex(ql, -1, &entry);
if (strncmp((char *)entry.value, "bobobob", 7))
......@@ -2598,10 +2572,9 @@ int quicklistTest(int argc, char *argv[]) {
}
}
for (int f = optimize_start; f < 40; f++) {
TEST_DESC("ltrim test C at fill %d at compress %d", f,
options[_i]) {
quicklist *ql = quicklistNew(f, options[_i]);
TEST_DESC("ltrim test C at compress %d", options[_i]) {
for (int f = 0; f < fill_count; f++) {
quicklist *ql = quicklistNew(fills[f], options[_i]);
char num[32];
long long nums[5000];
for (int i = 0; i < 33; i++) {
......@@ -2609,28 +2582,25 @@ int quicklistTest(int argc, char *argv[]) {
int sz = ll2string(num, sizeof(num), nums[i]);
quicklistPushTail(ql, num, sz);
}
if (f == 32)
if (fills[f] == 32)
ql_verify(ql, 2, 33, 32, 1);
/* ltrim 3 3 (keep [3,3] inclusive = 1 remaining) */
quicklistDelRange(ql, 0, 3);
quicklistDelRange(ql, -29,
4000); /* make sure not loop forever */
if (f == 32)
if (fills[f] == 32)
ql_verify(ql, 1, 1, 1, 1);
quicklistEntry entry;
quicklistIndex(ql, 0, &entry);
if (entry.longval != -5157318210846258173)
ERROR;
else
OK;
quicklistRelease(ql);
}
}
for (int f = optimize_start; f < 40; f++) {
TEST_DESC("ltrim test D at fill %d at compress %d", f,
options[_i]) {
quicklist *ql = quicklistNew(f, options[_i]);
TEST_DESC("ltrim test D at compress %d", options[_i]) {
for (int f = 0; f < fill_count; f++) {
quicklist *ql = quicklistNew(fills[f], options[_i]);
char num[32];
long long nums[5000];
for (int i = 0; i < 33; i++) {
......@@ -2638,7 +2608,7 @@ int quicklistTest(int argc, char *argv[]) {
int sz = ll2string(num, sizeof(num), nums[i]);
quicklistPushTail(ql, num, sz);
}
if (f == 32)
if (fills[f] == 32)
ql_verify(ql, 2, 33, 32, 1);
quicklistDelRange(ql, -12, 3);
if (ql->count != 30)
......@@ -2648,9 +2618,8 @@ int quicklistTest(int argc, char *argv[]) {
}
}
for (int f = optimize_start; f < 72; f++) {
TEST_DESC("create quicklist from ziplist at fill %d at compress %d",
f, options[_i]) {
TEST_DESC("create quicklist from ziplist at compress %d", options[_i]) {
for (int f = 0; f < fill_count; f++) {
unsigned char *zl = ziplistNew();
long long nums[64];
char num[64];
......@@ -2664,12 +2633,12 @@ int quicklistTest(int argc, char *argv[]) {
zl = ziplistPush(zl, (unsigned char *)genstr("hello", i),
32, ZIPLIST_TAIL);
}
quicklist *ql = quicklistCreateFromZiplist(f, options[_i], zl);
if (f == 1)
quicklist *ql = quicklistCreateFromZiplist(fills[f], options[_i], zl);
if (fills[f] == 1)
ql_verify(ql, 66, 66, 1, 1);
else if (f == 32)
else if (fills[f] == 32)
ql_verify(ql, 3, 66, 32, 2);
else if (f == 66)
else if (fills[f] == 66)
ql_verify(ql, 1, 66, 66, 66);
quicklistRelease(ql);
}
......@@ -2682,21 +2651,30 @@ int quicklistTest(int argc, char *argv[]) {
/* Run a longer test of compression depth outside of primary test loop. */
int list_sizes[] = {250, 251, 500, 999, 1000};
long long start = mstime();
for (int list = 0; list < (int)(sizeof(list_sizes) / sizeof(*list_sizes));
list++) {
for (int f = optimize_start; f < 128; f++) {
int list_count = accurate ? (int)(sizeof(list_sizes) / sizeof(*list_sizes)) : 1;
for (int list = 0; list < list_count; list++) {
TEST_DESC("verify specific compression of interior nodes with %d list ",
list_sizes[list]) {
for (int f = 0; f < fill_count; f++) {
for (int depth = 1; depth < 40; depth++) {
/* skip over many redundant test cases */
TEST_DESC("verify specific compression of interior nodes with "
"%d list "
"at fill %d at compress %d",
list_sizes[list], f, depth) {
quicklist *ql = quicklistNew(f, depth);
quicklist *ql = quicklistNew(fills[f], depth);
for (int i = 0; i < list_sizes[list]; i++) {
quicklistPushTail(ql, genstr("hello TAIL", i + 1), 64);
quicklistPushHead(ql, genstr("hello HEAD", i + 1), 64);
}
for (int step = 0; step < 2; step++) {
/* test remove node */
if (step == 1) {
for (int i = 0; i < list_sizes[list] / 2; i++) {
unsigned char *data;
quicklistPop(ql, QUICKLIST_HEAD, &data, NULL, NULL);
zfree(data);
quicklistPop(ql, QUICKLIST_TAIL, &data, NULL, NULL);
zfree(data);
}
}
quicklistNode *node = ql->head;
unsigned int low_raw = ql->compress;
unsigned int high_raw = ql->len - ql->compress;
......@@ -2721,6 +2699,8 @@ int quicklistTest(int argc, char *argv[]) {
}
}
}
}
quicklistRelease(ql);
}
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment