Commit c3ff4708 authored by antirez's avatar antirez
Browse files

Merge remote-tracking branch 'origin/2.6' into 2.6

parents 0e25c0cc f4eb4b33
......@@ -14,6 +14,18 @@ HIGH: There is a critical bug that may affect a subset of users. Upgrade!
CRITICAL: There is a critical bug affecting MOST USERS. Upgrade ASAP.
--------------------------------------------------------------------------------
---[ Redis 2.5.14 (2.6 Release Candidate 8) ]
* [BUGFIX] Fixed compilation on FreeBSD.
* [IMPROVED] SRANDMEMBER <count> that returns multiple random elements.
* [IMPROVED] Sentinel backported to 2.6. It will be taken in sync with 2.8.
* [IMPROVED] Helper function for scripting to return errors and status replies.
* [IMPROVED] SORT by nosort [ASC|DESC] uses sorted set elements ordering.
* [BUGFIX] Better resistence to system clock skew.
* [IMPROVED] Warn the user when the configured maxmemory seems odd.
* [BUGFIX] Hashing function is now murmurhash2 for security purposes.
* [IMPROVED] Install script no longer uses a template but redis.conf itself.
---[ Redis 2.5.13 (2.6 Release Candidate 7) ]
UPGRADE URGENCY: HIGH
......
1. Enter irc.freenode.org #redis and start talking with 'antirez' and/or 'pietern' to check if there is interest for such a feature and to understand the probability of it being merged. We'll try hard to keep Redis simple... so you'll likely encounter high resistance.
# IMPORTANT: HOW TO USE REDIS GITHUB ISSUES
2. Drop a message to the Redis Google Group with a proposal of semantics/API.
* Github issues SHOULD ONLY BE USED to report bugs, and for DETAILED feature
requests. Everything else belongs to the Redis Google Group.
3. If steps 1 and 2 are ok, use the following procedure to submit a patch:
PLEASE DO NOT POST GENERAL QUESTIONS that are not about bugs or suspected
bugs in the Github issues system. We'll be very happy to help you and provide
all the support in the Redis Google Group.
Redis Google Group address:
https://groups.google.com/forum/?fromgroups#!forum/redis-db
# How to provide a patch for a new feature
1. Drop a message to the Redis Google Group with a proposal of semantics/API.
2. If in steps 1 you get an acknowledge from the project leaders, use the
following procedure to submit a patch:
a. Fork Redis on github ( http://help.github.com/fork-a-repo/ )
b. Create a topic branch (git checkout -b my_branch)
......
......@@ -37,6 +37,7 @@
#include <stdlib.h>
#include <poll.h>
#include <string.h>
#include <time.h>
#include "ae.h"
#include "zmalloc.h"
......@@ -67,6 +68,7 @@ aeEventLoop *aeCreateEventLoop(int setsize) {
eventLoop->fired = zmalloc(sizeof(aeFiredEvent)*setsize);
if (eventLoop->events == NULL || eventLoop->fired == NULL) goto err;
eventLoop->setsize = setsize;
eventLoop->lastTime = time(NULL);
eventLoop->timeEventHead = NULL;
eventLoop->timeEventNextId = 0;
eventLoop->stop = 0;
......@@ -236,6 +238,24 @@ static int processTimeEvents(aeEventLoop *eventLoop) {
int processed = 0;
aeTimeEvent *te;
long long maxId;
time_t now = time(NULL);
/* If the system clock is moved to the future, and then set back to the
* right value, time events may be delayed in a random way. Often this
* means that scheduled operations will not be performed soon enough.
*
* Here we try to detect system clock skews, and force all the time
* events to be processed ASAP when this happens: the idea is that
* processing events earlier is less dangerous than delaying them
* indefinitely, and practice suggests it is. */
if (now < eventLoop->lastTime) {
te = eventLoop->timeEventHead;
while(te) {
te->when_sec = 0;
te = te->next;
}
}
eventLoop->lastTime = now;
te = eventLoop->timeEventHead;
maxId = eventLoop->timeEventNextId-1;
......
......@@ -88,6 +88,7 @@ typedef struct aeEventLoop {
int maxfd; /* highest file descriptor currently registered */
int setsize; /* max number of file descriptors tracked */
long long timeEventNextId;
time_t lastTime; /* Used to detect system clock skew */
aeFileEvent *events; /* Registered events */
aeFiredEvent *fired; /* Fired events */
aeTimeEvent *timeEventHead;
......
......@@ -438,7 +438,12 @@ void configSetCommand(redisClient *c) {
if (getLongLongFromObject(o,&ll) == REDIS_ERR ||
ll < 0) goto badfmt;
server.maxmemory = ll;
if (server.maxmemory) freeMemoryIfNeeded();
if (server.maxmemory) {
if (server.maxmemory < zmalloc_used_memory()) {
redisLog(REDIS_WARNING,"WARNING: the new maxmemory value set via CONFIG SET is smaller than the current memory usage. This will result in keys eviction and/or inability to accept new write commands depending on the maxmemory-policy.");
}
freeMemoryIfNeeded();
}
} else if (!strcasecmp(c->argv[2]->ptr,"maxmemory-policy")) {
if (!strcasecmp(o->ptr,"volatile-lru")) {
server.maxmemory_policy = REDIS_MAXMEMORY_VOLATILE_LRU;
......
......@@ -85,29 +85,73 @@ unsigned int dictIdentityHashFunction(unsigned int key)
return key;
}
static int dict_hash_function_seed = 5381;
static uint32_t dict_hash_function_seed = 5381;
void dictSetHashFunctionSeed(unsigned int seed) {
void dictSetHashFunctionSeed(uint32_t seed) {
dict_hash_function_seed = seed;
}
unsigned int dictGetHashFunctionSeed(void) {
uint32_t dictGetHashFunctionSeed(void) {
return dict_hash_function_seed;
}
/* Generic hash function (a popular one from Bernstein).
* I tested a few and this was the best. */
unsigned int dictGenHashFunction(const unsigned char *buf, int len) {
unsigned int hash = dict_hash_function_seed;
/* MurmurHash2, by Austin Appleby
* Note - This code makes a few assumptions about how your machine behaves -
* 1. We can read a 4-byte value from any address without crashing
* 2. sizeof(int) == 4
*
* And it has a few limitations -
*
* 1. It will not work incrementally.
* 2. It will not produce the same results on little-endian and big-endian
* machines.
*/
unsigned int dictGenHashFunction(const void *key, int len) {
/* 'm' and 'r' are mixing constants generated offline.
They're not really 'magic', they just happen to work well. */
uint32_t seed = dict_hash_function_seed;
const uint32_t m = 0x5bd1e995;
const int r = 24;
while (len--)
hash = ((hash << 5) + hash) + (*buf++); /* hash * 33 + c */
return hash;
/* Initialize the hash to a 'random' value */
uint32_t h = seed ^ len;
/* Mix 4 bytes at a time into the hash */
const unsigned char *data = (const unsigned char *)key;
while(len >= 4) {
uint32_t k = *(uint32_t*)data;
k *= m;
k ^= k >> r;
k *= m;
h *= m;
h ^= k;
data += 4;
len -= 4;
}
/* Handle the last few bytes of the input array */
switch(len) {
case 3: h ^= data[2] << 16;
case 2: h ^= data[1] << 8;
case 1: h ^= data[0]; h *= m;
};
/* Do a few final mixes of the hash to ensure the last few
* bytes are well-incorporated. */
h ^= h >> 13;
h *= m;
h ^= h >> 15;
return (unsigned int)h;
}
/* And a case insensitive version */
/* And a case insensitive hash function (based on djb hash) */
unsigned int dictGenCaseHashFunction(const unsigned char *buf, int len) {
unsigned int hash = dict_hash_function_seed;
unsigned int hash = (unsigned int)dict_hash_function_seed;
while (len--)
hash = ((hash << 5) + hash) + (tolower(*buf++)); /* hash * 33 + c */
......
......@@ -155,7 +155,7 @@ dictEntry *dictNext(dictIterator *iter);
void dictReleaseIterator(dictIterator *iter);
dictEntry *dictGetRandomKey(dict *d);
void dictPrintStats(dict *d);
unsigned int dictGenHashFunction(const unsigned char *buf, int len);
unsigned int dictGenHashFunction(const void *key, int len);
unsigned int dictGenCaseHashFunction(const unsigned char *buf, int len);
void dictEmpty(dict *d);
void dictEnableResize(void);
......
......@@ -2579,6 +2579,11 @@ int main(int argc, char **argv) {
redisLog(REDIS_NOTICE,"The server is now ready to accept connections at %s", server.unixsocket);
}
/* Warning the user about suspicious maxmemory setting. */
if (server.maxmemory > 0 && server.maxmemory < 1024*1024) {
redisLog(REDIS_WARNING,"WARNING: You specified a maxmemory value that is less than 1MB (current value is %llu bytes). Are you sure this is what you really want?", server.maxmemory);
}
aeSetBeforeSleepProc(server.el,beforeSleep);
aeMain(server.el);
aeDeleteEventLoop(server.el);
......
......@@ -721,7 +721,7 @@ void replicationCron(void) {
if (server.masterhost && server.repl_state == REDIS_REPL_TRANSFER &&
(time(NULL)-server.repl_transfer_lastio) > server.repl_timeout)
{
redisLog(REDIS_WARNING,"Timeout receiving bulk data from MASTER...");
redisLog(REDIS_WARNING,"Timeout receiving bulk data from MASTER... If the problem persists try to set the 'repl-timeout' parameter in redis.conf to a larger value.");
replicationAbortSyncTransfer();
}
......
......@@ -2,6 +2,8 @@
#include "pqsort.h" /* Partial qsort for SORT+LIMIT */
#include <math.h> /* isnan() */
zskiplistNode* zslGetElementByRank(zskiplist *zsl, unsigned long rank);
redisSortOperation *createSortOperation(int type, robj *pattern) {
redisSortOperation *so = zmalloc(sizeof(*so));
so->type = type;
......@@ -156,8 +158,9 @@ void sortCommand(redisClient *c) {
/* Lookup the key to sort. It must be of the right types */
sortval = lookupKeyRead(c->db,c->argv[1]);
if (sortval && sortval->type != REDIS_SET && sortval->type != REDIS_LIST &&
sortval->type != REDIS_ZSET)
if (sortval && sortval->type != REDIS_SET &&
sortval->type != REDIS_LIST &&
sortval->type != REDIS_ZSET)
{
addReply(c,shared.wrongtypeerr);
return;
......@@ -167,7 +170,7 @@ void sortCommand(redisClient *c) {
* Operations can be GET/DEL/INCR/DECR */
operations = listCreate();
listSetFreeMethod(operations,zfree);
j = 2;
j = 2; /* options start at argv[2] */
/* Now we need to protect sortval incrementing its count, in the future
* SORT may have options able to overwrite/delete keys during the sorting
......@@ -213,13 +216,18 @@ void sortCommand(redisClient *c) {
j++;
}
/* If we have STORE we need to force sorting for deterministic output
* and replication. We use alpha sorting since this is guaranteed to
* work with any input.
/* For the STORE option, or when SORT is called from a Lua script,
* we want to force a specific ordering even when no explicit ordering
* was asked (SORT BY nosort). This guarantees that replication / AOF
* is deterministic.
*
* We also want determinism when SORT is called from Lua scripts, so
* in this case we also force alpha sorting. */
if ((storekey || c->flags & REDIS_LUA_CLIENT) && dontsort) {
* However in the case 'dontsort' is true, but the type to sort is a
* sorted set, we don't need to do anything as ordering is guaranteed
* in this special case. */
if ((storekey || c->flags & REDIS_LUA_CLIENT) &&
(dontsort && sortval->type != REDIS_ZSET))
{
/* Force ALPHA sorting */
dontsort = 0;
alpha = 1;
sortby = NULL;
......@@ -229,13 +237,41 @@ void sortCommand(redisClient *c) {
if (sortval->type == REDIS_ZSET)
zsetConvert(sortval, REDIS_ENCODING_SKIPLIST);
/* Load the sorting vector with all the objects to sort */
/* Objtain the length of the object to sort. */
switch(sortval->type) {
case REDIS_LIST: vectorlen = listTypeLength(sortval); break;
case REDIS_SET: vectorlen = setTypeSize(sortval); break;
case REDIS_ZSET: vectorlen = dictSize(((zset*)sortval->ptr)->dict); break;
default: vectorlen = 0; redisPanic("Bad SORT type"); /* Avoid GCC warning */
}
/* Perform LIMIT start,count sanity checking. */
start = (limit_start < 0) ? 0 : limit_start;
end = (limit_count < 0) ? vectorlen-1 : start+limit_count-1;
if (start >= vectorlen) {
start = vectorlen-1;
end = vectorlen-2;
}
if (end >= vectorlen) end = vectorlen-1;
/* Optimization:
*
* 1) if the object to sort is a sorted set.
* 2) There is nothing to sort as dontsort is true (BY <constant string>).
* 3) We have a LIMIT option that actually reduces the number of elements
* to fetch.
*
* In this case to load all the objects in the vector is a huge waste of
* resources. We just allocate a vector that is big enough for the selected
* range length, and make sure to load just this part in the vector. */
if (sortval->type == REDIS_ZSET &&
dontsort &&
(start != 0 || end != vectorlen-1))
{
vectorlen = end-start+1;
}
/* Load the sorting vector with all the objects to sort */
vector = zmalloc(sizeof(redisSortObject)*vectorlen);
j = 0;
......@@ -259,6 +295,48 @@ void sortCommand(redisClient *c) {
j++;
}
setTypeReleaseIterator(si);
} else if (sortval->type == REDIS_ZSET && dontsort) {
/* Special handling for a sorted set, if 'dontsort' is true.
* This makes sure we return elements in the sorted set original
* ordering, accordingly to DESC / ASC options.
*
* Note that in this case we also handle LIMIT here in a direct
* way, just getting the required range, as an optimization. */
zset *zs = sortval->ptr;
zskiplist *zsl = zs->zsl;
zskiplistNode *ln;
robj *ele;
int rangelen = vectorlen;
/* Check if starting point is trivial, before doing log(N) lookup. */
if (desc) {
long zsetlen = dictSize(((zset*)sortval->ptr)->dict);
ln = zsl->tail;
if (start > 0)
ln = zslGetElementByRank(zsl,zsetlen-start);
} else {
ln = zsl->header->level[0].forward;
if (start > 0)
ln = zslGetElementByRank(zsl,start+1);
}
while(rangelen--) {
redisAssertWithInfo(c,sortval,ln != NULL);
ele = ln->obj;
vector[j].obj = ele;
vector[j].u.score = 0;
vector[j].u.cmpobj = NULL;
j++;
ln = desc ? ln->backward : ln->level[0].forward;
}
/* The code producing the output does not know that in the case of
* sorted set, 'dontsort', and LIMIT, we are able to get just the
* range, already sorted, so we need to adjust "start" and "end"
* to make sure start is set to 0. */
end -= start;
start = 0;
} else if (sortval->type == REDIS_ZSET) {
dict *set = ((zset*)sortval->ptr)->dict;
dictIterator *di;
......@@ -319,16 +397,6 @@ void sortCommand(redisClient *c) {
}
}
/* We are ready to sort the vector... perform a bit of sanity check
* on the LIMIT option too. We'll use a partial version of quicksort. */
start = (limit_start < 0) ? 0 : limit_start;
end = (limit_count < 0) ? vectorlen-1 : start+limit_count-1;
if (start >= vectorlen) {
start = vectorlen-1;
end = vectorlen-2;
}
if (end >= vectorlen) end = vectorlen-1;
if (dontsort == 0) {
server.sort_desc = desc;
server.sort_alpha = alpha;
......
#define REDIS_VERSION "2.5.13"
#define REDIS_VERSION "2.5.14"
......@@ -118,6 +118,47 @@ start_server {
r sort zset alpha desc
} {e d c b a}
test "SORT sorted set BY nosort should retain ordering" {
r del zset
r zadd zset 1 a
r zadd zset 5 b
r zadd zset 2 c
r zadd zset 10 d
r zadd zset 3 e
r multi
r sort zset by nosort asc
r sort zset by nosort desc
r exec
} {{a c e b d} {d b e c a}}
test "SORT sorted set BY nosort + LIMIT" {
r del zset
r zadd zset 1 a
r zadd zset 5 b
r zadd zset 2 c
r zadd zset 10 d
r zadd zset 3 e
assert_equal [r sort zset by nosort asc limit 0 1] {a}
assert_equal [r sort zset by nosort desc limit 0 1] {d}
assert_equal [r sort zset by nosort asc limit 0 2] {a c}
assert_equal [r sort zset by nosort desc limit 0 2] {d b}
assert_equal [r sort zset by nosort limit 5 10] {}
assert_equal [r sort zset by nosort limit -10 100] {a c e b d}
}
test "SORT sorted set BY nosort works as expected from scripts" {
r del zset
r zadd zset 1 a
r zadd zset 5 b
r zadd zset 2 c
r zadd zset 10 d
r zadd zset 3 e
r eval {
return {redis.call('sort','zset','by','nosort','asc'),
redis.call('sort','zset','by','nosort','desc')}
} 0
} {{a c e b d} {d b e c a}}
test "SORT sorted set: +inf and -inf handling" {
r del zset
r zadd zset -100 a
......
#!/bin/bash
#!/bin/sh
# Copyright 2011 Dvir Volk <dvirsk at gmail dot com>. All rights reserved.
#
......@@ -48,6 +48,7 @@ if [ `whoami` != "root" ] ; then
exit 1
fi
#Read the redis port
read -p "Please select the redis port for this instance: [$_REDIS_PORT] " REDIS_PORT
if [ ! `echo $REDIS_PORT | egrep "^[0-9]+\$"` ] ; then
......@@ -99,7 +100,7 @@ fi
#render the tmplates
TMP_FILE="/tmp/$REDIS_PORT.conf"
TPL_FILE="./redis.conf.tpl"
DEFAULT_CONFIG="../redis.conf"
INIT_TPL_FILE="./redis_init_script.tpl"
INIT_SCRIPT_DEST="/etc/init.d/redis_$REDIS_PORT"
PIDFILE="/var/run/redis_$REDIS_PORT.pid"
......@@ -112,9 +113,19 @@ if [ ! "$CLI_EXEC" ] ; then
CLI_EXEC=`dirname $REDIS_EXECUTABLE`"/redis-cli"
fi
#Generate config file from template
#Generate config file from the default config file as template
#changing only the stuff we're controlling from this script
echo "## Generated by install_server.sh ##" > $TMP_FILE
cat $TPL_FILE | while read line; do eval "echo \"$line\"" >> $TMP_FILE; done
SED_EXPR="s#^port [0-9]{4}\$#port ${REDIS_PORT}#;\
s#^logfile .+\$#logfile ${REDIS_LOG_FILE}#;\
s#^dir .+\$#dir ${REDIS_DATA_DIR}#;\
s#^pidfile .+\$#pidfile ${PIDFILE}#;\
s#^daemonize no\$#daemonize yes#;"
echo $SED_EXPR
sed -r "$SED_EXPR" $DEFAULT_CONFIG >> $TMP_FILE
#cat $TPL_FILE | while read line; do eval "echo \"$line\"" >> $TMP_FILE; done
cp -f $TMP_FILE $REDIS_CONFIG_FILE || exit 1
#Generate sample script from template file
......@@ -146,9 +157,9 @@ REDIS_CHKCONFIG_INFO=\
# Description: Redis daemon\n
### END INIT INFO\n\n"
if [[ ! `which chkconfig` ]] ; then
if [ !`which chkconfig` ] ; then
#combine the header and the template (which is actually a static footer)
echo -e $REDIS_INIT_HEADER > $TMP_FILE && cat $INIT_TPL_FILE >> $TMP_FILE || die "Could not write init script to $TMP_FILE"
echo $REDIS_INIT_HEADER > $TMP_FILE && cat $INIT_TPL_FILE >> $TMP_FILE || die "Could not write init script to $TMP_FILE"
else
#if we're a box with chkconfig on it we want to include info for chkconfig
echo -e $REDIS_INIT_HEADER $REDIS_CHKCONFIG_INFO > $TMP_FILE && cat $INIT_TPL_FILE >> $TMP_FILE || die "Could not write init script to $TMP_FILE"
......@@ -160,7 +171,7 @@ echo "Copied $TMP_FILE => $INIT_SCRIPT_DEST"
#Install the service
echo "Installing service..."
if [[ ! `which chkconfig` ]] ; then
if [ !`which chkconfig` ] ; then
#if we're not a chkconfig box assume we're able to use update-rc.d
update-rc.d redis_$REDIS_PORT defaults && echo "Success!"
else
......
This diff is collapsed.
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment