Commit 05ba119f authored by Salvatore Sanfilippo's avatar Salvatore Sanfilippo
Browse files

Merge pull request #2143 from mattsta/quicklist

Quicklist (linked list + ziplist)
parents 9e718a1f 5870e224
...@@ -209,43 +209,49 @@ int rdbTryIntegerEncoding(char *s, size_t len, unsigned char *enc) { ...@@ -209,43 +209,49 @@ int rdbTryIntegerEncoding(char *s, size_t len, unsigned char *enc) {
return rdbEncodeInteger(value,enc); return rdbEncodeInteger(value,enc);
} }
int rdbSaveLzfStringObject(rio *rdb, unsigned char *s, size_t len) { int rdbSaveLzfBlob(rio *rdb, void *data, size_t compress_len,
size_t comprlen, outlen; size_t original_len) {
unsigned char byte; unsigned char byte;
int n, nwritten = 0; int n, nwritten = 0;
void *out;
/* We require at least four bytes compression for this to be worth it */
if (len <= 4) return 0;
outlen = len-4;
if ((out = zmalloc(outlen+1)) == NULL) return 0;
comprlen = lzf_compress(s, len, out, outlen);
if (comprlen == 0) {
zfree(out);
return 0;
}
/* Data compressed! Let's save it on disk */ /* Data compressed! Let's save it on disk */
byte = (REDIS_RDB_ENCVAL<<6)|REDIS_RDB_ENC_LZF; byte = (REDIS_RDB_ENCVAL<<6)|REDIS_RDB_ENC_LZF;
if ((n = rdbWriteRaw(rdb,&byte,1)) == -1) goto writeerr; if ((n = rdbWriteRaw(rdb,&byte,1)) == -1) goto writeerr;
nwritten += n; nwritten += n;
if ((n = rdbSaveLen(rdb,comprlen)) == -1) goto writeerr; if ((n = rdbSaveLen(rdb,compress_len)) == -1) goto writeerr;
nwritten += n; nwritten += n;
if ((n = rdbSaveLen(rdb,len)) == -1) goto writeerr; if ((n = rdbSaveLen(rdb,original_len)) == -1) goto writeerr;
nwritten += n; nwritten += n;
if ((n = rdbWriteRaw(rdb,out,comprlen)) == -1) goto writeerr; if ((n = rdbWriteRaw(rdb,data,compress_len)) == -1) goto writeerr;
nwritten += n; nwritten += n;
zfree(out);
return nwritten; return nwritten;
writeerr: writeerr:
zfree(out);
return -1; return -1;
} }
int rdbSaveLzfStringObject(rio *rdb, unsigned char *s, size_t len) {
size_t comprlen, outlen;
void *out;
/* We require at least four bytes compression for this to be worth it */
if (len <= 4) return 0;
outlen = len-4;
if ((out = zmalloc(outlen+1)) == NULL) return 0;
comprlen = lzf_compress(s, len, out, outlen);
if (comprlen == 0) {
zfree(out);
return 0;
}
size_t nwritten = rdbSaveLzfBlob(rdb, out, comprlen, len);
zfree(out);
return nwritten;
}
robj *rdbLoadLzfStringObject(rio *rdb) { robj *rdbLoadLzfStringObject(rio *rdb) {
unsigned int len, clen; unsigned int len, clen;
unsigned char *c = NULL; unsigned char *c = NULL;
...@@ -433,10 +439,8 @@ int rdbSaveObjectType(rio *rdb, robj *o) { ...@@ -433,10 +439,8 @@ int rdbSaveObjectType(rio *rdb, robj *o) {
case REDIS_STRING: case REDIS_STRING:
return rdbSaveType(rdb,REDIS_RDB_TYPE_STRING); return rdbSaveType(rdb,REDIS_RDB_TYPE_STRING);
case REDIS_LIST: case REDIS_LIST:
if (o->encoding == REDIS_ENCODING_ZIPLIST) if (o->encoding == REDIS_ENCODING_QUICKLIST)
return rdbSaveType(rdb,REDIS_RDB_TYPE_LIST_ZIPLIST); return rdbSaveType(rdb,REDIS_RDB_TYPE_LIST_QUICKLIST);
else if (o->encoding == REDIS_ENCODING_LINKEDLIST)
return rdbSaveType(rdb,REDIS_RDB_TYPE_LIST);
else else
redisPanic("Unknown list encoding"); redisPanic("Unknown list encoding");
case REDIS_SET: case REDIS_SET:
...@@ -477,7 +481,7 @@ int rdbLoadObjectType(rio *rdb) { ...@@ -477,7 +481,7 @@ int rdbLoadObjectType(rio *rdb) {
/* Save a Redis object. Returns -1 on error, number of bytes written on success. */ /* Save a Redis object. Returns -1 on error, number of bytes written on success. */
int rdbSaveObject(rio *rdb, robj *o) { int rdbSaveObject(rio *rdb, robj *o) {
int n, nwritten = 0; int n = 0, nwritten = 0;
if (o->type == REDIS_STRING) { if (o->type == REDIS_STRING) {
/* Save a string value */ /* Save a string value */
...@@ -485,25 +489,24 @@ int rdbSaveObject(rio *rdb, robj *o) { ...@@ -485,25 +489,24 @@ int rdbSaveObject(rio *rdb, robj *o) {
nwritten += n; nwritten += n;
} else if (o->type == REDIS_LIST) { } else if (o->type == REDIS_LIST) {
/* Save a list value */ /* Save a list value */
if (o->encoding == REDIS_ENCODING_ZIPLIST) { if (o->encoding == REDIS_ENCODING_QUICKLIST) {
size_t l = ziplistBlobLen((unsigned char*)o->ptr); quicklist *ql = o->ptr;
quicklistNode *node = ql->head;
if ((n = rdbSaveRawString(rdb,o->ptr,l)) == -1) return -1;
nwritten += n;
} else if (o->encoding == REDIS_ENCODING_LINKEDLIST) {
list *list = o->ptr;
listIter li;
listNode *ln;
if ((n = rdbSaveLen(rdb,listLength(list))) == -1) return -1; if ((n = rdbSaveLen(rdb,ql->len)) == -1) return -1;
nwritten += n; nwritten += n;
listRewind(list,&li); do {
while((ln = listNext(&li))) { if (quicklistNodeIsCompressed(node)) {
robj *eleobj = listNodeValue(ln); void *data;
if ((n = rdbSaveStringObject(rdb,eleobj)) == -1) return -1; size_t compress_len = quicklistGetLzf(node, &data);
nwritten += n; if ((n = rdbSaveLzfBlob(rdb,data,compress_len,node->sz)) == -1) return -1;
} nwritten += n;
} else {
if ((n = rdbSaveRawString(rdb,node->zl,node->sz)) == -1) return -1;
nwritten += n;
}
} while ((node = node->next));
} else { } else {
redisPanic("Unknown list encoding"); redisPanic("Unknown list encoding");
} }
...@@ -720,7 +723,7 @@ int rdbSave(char *filename) { ...@@ -720,7 +723,7 @@ int rdbSave(char *filename) {
char tmpfile[256]; char tmpfile[256];
FILE *fp; FILE *fp;
rio rdb; rio rdb;
int error; int error = 0;
snprintf(tmpfile,256,"temp-%d.rdb", (int) getpid()); snprintf(tmpfile,256,"temp-%d.rdb", (int) getpid());
fp = fopen(tmpfile,"w"); fp = fopen(tmpfile,"w");
...@@ -831,33 +834,18 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) { ...@@ -831,33 +834,18 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) {
/* Read list value */ /* Read list value */
if ((len = rdbLoadLen(rdb,NULL)) == REDIS_RDB_LENERR) return NULL; if ((len = rdbLoadLen(rdb,NULL)) == REDIS_RDB_LENERR) return NULL;
/* Use a real list when there are too many entries */ o = createQuicklistObject();
if (len > server.list_max_ziplist_entries) { quicklistSetOptions(o->ptr, server.list_max_ziplist_size,
o = createListObject(); server.list_compress_depth);
} else {
o = createZiplistObject();
}
/* Load every single element of the list */ /* Load every single element of the list */
while(len--) { while(len--) {
if ((ele = rdbLoadEncodedStringObject(rdb)) == NULL) return NULL; if ((ele = rdbLoadEncodedStringObject(rdb)) == NULL) return NULL;
dec = getDecodedObject(ele);
/* If we are using a ziplist and the value is too big, convert size_t len = sdslen(dec->ptr);
* the object to a real list. */ quicklistPushTail(o->ptr, dec->ptr, len);
if (o->encoding == REDIS_ENCODING_ZIPLIST && decrRefCount(dec);
sdsEncodedObject(ele) && decrRefCount(ele);
sdslen(ele->ptr) > server.list_max_ziplist_value)
listTypeConvert(o,REDIS_ENCODING_LINKEDLIST);
if (o->encoding == REDIS_ENCODING_ZIPLIST) {
dec = getDecodedObject(ele);
o->ptr = ziplistPush(o->ptr,dec->ptr,sdslen(dec->ptr),REDIS_TAIL);
decrRefCount(dec);
decrRefCount(ele);
} else {
ele = tryObjectEncoding(ele);
listAddNodeTail(o->ptr,ele);
}
} }
} else if (rdbtype == REDIS_RDB_TYPE_SET) { } else if (rdbtype == REDIS_RDB_TYPE_SET) {
/* Read list/set value */ /* Read list/set value */
...@@ -994,20 +982,30 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) { ...@@ -994,20 +982,30 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) {
/* All pairs should be read by now */ /* All pairs should be read by now */
redisAssert(len == 0); redisAssert(len == 0);
} else if (rdbtype == REDIS_RDB_TYPE_LIST_QUICKLIST) {
if ((len = rdbLoadLen(rdb,NULL)) == REDIS_RDB_LENERR) return NULL;
o = createQuicklistObject();
quicklistSetOptions(o->ptr, server.list_max_ziplist_size,
server.list_compress_depth);
while (len--) {
if ((ele = rdbLoadStringObject(rdb)) == NULL) return NULL;
/* 'ele' contains a sds of the ziplist, but we need to extract
* the actual ziplist for future usage. We must copy the
* sds contents to a new buffer. */
unsigned char *zl = (unsigned char *)sdsnative(ele->ptr);
zfree(ele); /* free robj container since we keep the ziplist */
quicklistAppendZiplist(o->ptr, zl);
}
} else if (rdbtype == REDIS_RDB_TYPE_HASH_ZIPMAP || } else if (rdbtype == REDIS_RDB_TYPE_HASH_ZIPMAP ||
rdbtype == REDIS_RDB_TYPE_LIST_ZIPLIST || rdbtype == REDIS_RDB_TYPE_LIST_ZIPLIST ||
rdbtype == REDIS_RDB_TYPE_SET_INTSET || rdbtype == REDIS_RDB_TYPE_SET_INTSET ||
rdbtype == REDIS_RDB_TYPE_ZSET_ZIPLIST || rdbtype == REDIS_RDB_TYPE_ZSET_ZIPLIST ||
rdbtype == REDIS_RDB_TYPE_HASH_ZIPLIST) rdbtype == REDIS_RDB_TYPE_HASH_ZIPLIST)
{ {
robj *aux = rdbLoadStringObject(rdb); o = rdbLoadStringObject(rdb);
if (o == NULL) return NULL;
if (aux == NULL) return NULL; o->ptr = sdsnative(o->ptr);
o = createObject(REDIS_STRING,NULL); /* string is just placeholder */
o->ptr = zmalloc(sdslen(aux->ptr));
memcpy(o->ptr,aux->ptr,sdslen(aux->ptr));
decrRefCount(aux);
/* Fix the object encoding, and make sure to convert the encoded /* Fix the object encoding, and make sure to convert the encoded
* data type into the base type if accordingly to the current * data type into the base type if accordingly to the current
...@@ -1048,8 +1046,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) { ...@@ -1048,8 +1046,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) {
case REDIS_RDB_TYPE_LIST_ZIPLIST: case REDIS_RDB_TYPE_LIST_ZIPLIST:
o->type = REDIS_LIST; o->type = REDIS_LIST;
o->encoding = REDIS_ENCODING_ZIPLIST; o->encoding = REDIS_ENCODING_ZIPLIST;
if (ziplistLen(o->ptr) > server.list_max_ziplist_entries) listTypeConvert(o,REDIS_ENCODING_QUICKLIST);
listTypeConvert(o,REDIS_ENCODING_LINKEDLIST);
break; break;
case REDIS_RDB_TYPE_SET_INTSET: case REDIS_RDB_TYPE_SET_INTSET:
o->type = REDIS_SET; o->type = REDIS_SET;
......
...@@ -38,7 +38,7 @@ ...@@ -38,7 +38,7 @@
/* The current RDB version. When the format changes in a way that is no longer /* The current RDB version. When the format changes in a way that is no longer
* backward compatible this number gets incremented. */ * backward compatible this number gets incremented. */
#define REDIS_RDB_VERSION 6 #define REDIS_RDB_VERSION 7
/* Defines related to the dump file format. To store 32 bits lengths for short /* Defines related to the dump file format. To store 32 bits lengths for short
* keys requires a lot of space, so we check the most significant 2 bits of * keys requires a lot of space, so we check the most significant 2 bits of
...@@ -74,6 +74,7 @@ ...@@ -74,6 +74,7 @@
#define REDIS_RDB_TYPE_SET 2 #define REDIS_RDB_TYPE_SET 2
#define REDIS_RDB_TYPE_ZSET 3 #define REDIS_RDB_TYPE_ZSET 3
#define REDIS_RDB_TYPE_HASH 4 #define REDIS_RDB_TYPE_HASH 4
/* NOTE: WHEN ADDING NEW RDB TYPE, UPDATE rdbIsObjectType() BELOW */
/* Object types for encoded objects. */ /* Object types for encoded objects. */
#define REDIS_RDB_TYPE_HASH_ZIPMAP 9 #define REDIS_RDB_TYPE_HASH_ZIPMAP 9
...@@ -81,9 +82,11 @@ ...@@ -81,9 +82,11 @@
#define REDIS_RDB_TYPE_SET_INTSET 11 #define REDIS_RDB_TYPE_SET_INTSET 11
#define REDIS_RDB_TYPE_ZSET_ZIPLIST 12 #define REDIS_RDB_TYPE_ZSET_ZIPLIST 12
#define REDIS_RDB_TYPE_HASH_ZIPLIST 13 #define REDIS_RDB_TYPE_HASH_ZIPLIST 13
#define REDIS_RDB_TYPE_LIST_QUICKLIST 14
/* NOTE: WHEN ADDING NEW RDB TYPE, UPDATE rdbIsObjectType() BELOW */
/* Test if a type is an object type. */ /* Test if a type is an object type. */
#define rdbIsObjectType(t) ((t >= 0 && t <= 4) || (t >= 9 && t <= 13)) #define rdbIsObjectType(t) ((t >= 0 && t <= 4) || (t >= 9 && t <= 14))
/* Special RDB opcodes (saved/loaded with rdbSaveType/rdbLoadType). */ /* Special RDB opcodes (saved/loaded with rdbSaveType/rdbLoadType). */
#define REDIS_RDB_OPCODE_EXPIRETIME_MS 252 #define REDIS_RDB_OPCODE_EXPIRETIME_MS 252
......
...@@ -738,12 +738,24 @@ int main(int argc, const char **argv) { ...@@ -738,12 +738,24 @@ int main(int argc, const char **argv) {
free(cmd); free(cmd);
} }
if (test_is_selected("rpush")) {
len = redisFormatCommand(&cmd,"RPUSH mylist %s",data);
benchmark("RPUSH",cmd,len);
free(cmd);
}
if (test_is_selected("lpop")) { if (test_is_selected("lpop")) {
len = redisFormatCommand(&cmd,"LPOP mylist"); len = redisFormatCommand(&cmd,"LPOP mylist");
benchmark("LPOP",cmd,len); benchmark("LPOP",cmd,len);
free(cmd); free(cmd);
} }
if (test_is_selected("rpop")) {
len = redisFormatCommand(&cmd,"RPOP mylist");
benchmark("RPOP",cmd,len);
free(cmd);
}
if (test_is_selected("sadd")) { if (test_is_selected("sadd")) {
len = redisFormatCommand(&cmd, len = redisFormatCommand(&cmd,
"SADD myset element:__rand_int__"); "SADD myset element:__rand_int__");
......
...@@ -629,6 +629,9 @@ static int cliSendCommand(int argc, char **argv, int repeat) { ...@@ -629,6 +629,9 @@ static int cliSendCommand(int argc, char **argv, int repeat) {
output_raw = 0; output_raw = 0;
if (!strcasecmp(command,"info") || if (!strcasecmp(command,"info") ||
(argc == 3 && !strcasecmp(command,"debug") &&
(!strcasecmp(argv[1],"jemalloc") &&
!strcasecmp(argv[2],"info"))) ||
(argc == 2 && !strcasecmp(command,"cluster") && (argc == 2 && !strcasecmp(command,"cluster") &&
(!strcasecmp(argv[1],"nodes") || (!strcasecmp(argv[1],"nodes") ||
!strcasecmp(argv[1],"info"))) || !strcasecmp(argv[1],"info"))) ||
......
...@@ -1449,8 +1449,8 @@ void initServerConfig(void) { ...@@ -1449,8 +1449,8 @@ void initServerConfig(void) {
server.maxmemory_samples = REDIS_DEFAULT_MAXMEMORY_SAMPLES; server.maxmemory_samples = REDIS_DEFAULT_MAXMEMORY_SAMPLES;
server.hash_max_ziplist_entries = REDIS_HASH_MAX_ZIPLIST_ENTRIES; server.hash_max_ziplist_entries = REDIS_HASH_MAX_ZIPLIST_ENTRIES;
server.hash_max_ziplist_value = REDIS_HASH_MAX_ZIPLIST_VALUE; server.hash_max_ziplist_value = REDIS_HASH_MAX_ZIPLIST_VALUE;
server.list_max_ziplist_entries = REDIS_LIST_MAX_ZIPLIST_ENTRIES; server.list_max_ziplist_size = REDIS_LIST_MAX_ZIPLIST_SIZE;
server.list_max_ziplist_value = REDIS_LIST_MAX_ZIPLIST_VALUE; server.list_compress_depth = REDIS_LIST_COMPRESS_DEPTH;
server.set_max_intset_entries = REDIS_SET_MAX_INTSET_ENTRIES; server.set_max_intset_entries = REDIS_SET_MAX_INTSET_ENTRIES;
server.zset_max_ziplist_entries = REDIS_ZSET_MAX_ZIPLIST_ENTRIES; server.zset_max_ziplist_entries = REDIS_ZSET_MAX_ZIPLIST_ENTRIES;
server.zset_max_ziplist_value = REDIS_ZSET_MAX_ZIPLIST_VALUE; server.zset_max_ziplist_value = REDIS_ZSET_MAX_ZIPLIST_VALUE;
...@@ -3058,11 +3058,7 @@ void infoCommand(redisClient *c) { ...@@ -3058,11 +3058,7 @@ void infoCommand(redisClient *c) {
addReply(c,shared.syntaxerr); addReply(c,shared.syntaxerr);
return; return;
} }
sds info = genRedisInfoString(section); addReplyBulkSds(c, genRedisInfoString(section));
addReplySds(c,sdscatprintf(sdsempty(),"$%lu\r\n",
(unsigned long)sdslen(info)));
addReplySds(c,info);
addReply(c,shared.crlf);
} }
void monitorCommand(redisClient *c) { void monitorCommand(redisClient *c) {
...@@ -3659,6 +3655,32 @@ int redisIsSupervised(void) { ...@@ -3659,6 +3655,32 @@ int redisIsSupervised(void) {
int main(int argc, char **argv) { int main(int argc, char **argv) {
struct timeval tv; struct timeval tv;
#ifdef REDIS_TEST
if (argc == 3 && !strcasecmp(argv[1], "test")) {
if (!strcasecmp(argv[2], "ziplist")) {
return ziplistTest(argc, argv);
} else if (!strcasecmp(argv[2], "quicklist")) {
quicklistTest(argc, argv);
} else if (!strcasecmp(argv[2], "intset")) {
return intsetTest(argc, argv);
} else if (!strcasecmp(argv[2], "zipmap")) {
return zipmapTest(argc, argv);
} else if (!strcasecmp(argv[2], "sha1test")) {
return sha1Test(argc, argv);
} else if (!strcasecmp(argv[2], "util")) {
return utilTest(argc, argv);
} else if (!strcasecmp(argv[2], "sds")) {
return sdsTest(argc, argv);
} else if (!strcasecmp(argv[2], "endianconv")) {
return endianconvTest(argc, argv);
} else if (!strcasecmp(argv[2], "crc64")) {
return crc64Test(argc, argv);
}
return -1; /* test not found */
}
#endif
/* We need to initialize our libraries, and the server configuration. */ /* We need to initialize our libraries, and the server configuration. */
#ifdef INIT_SETPROCTITLE_REPLACEMENT #ifdef INIT_SETPROCTITLE_REPLACEMENT
spt_init(argc, argv); spt_init(argc, argv);
......
...@@ -65,6 +65,13 @@ typedef long long mstime_t; /* millisecond time type. */ ...@@ -65,6 +65,13 @@ typedef long long mstime_t; /* millisecond time type. */
#include "util.h" /* Misc functions useful in many places */ #include "util.h" /* Misc functions useful in many places */
#include "latency.h" /* Latency monitor API */ #include "latency.h" /* Latency monitor API */
#include "sparkline.h" /* ASII graphs API */ #include "sparkline.h" /* ASII graphs API */
#include "quicklist.h"
/* Following includes allow test functions to be called from Redis main() */
#include "zipmap.h"
#include "sha1.h"
#include "endianconv.h"
#include "crc64.h"
/* Error codes */ /* Error codes */
#define REDIS_OK 0 #define REDIS_OK 0
...@@ -198,6 +205,7 @@ typedef long long mstime_t; /* millisecond time type. */ ...@@ -198,6 +205,7 @@ typedef long long mstime_t; /* millisecond time type. */
#define REDIS_ENCODING_INTSET 6 /* Encoded as intset */ #define REDIS_ENCODING_INTSET 6 /* Encoded as intset */
#define REDIS_ENCODING_SKIPLIST 7 /* Encoded as skiplist */ #define REDIS_ENCODING_SKIPLIST 7 /* Encoded as skiplist */
#define REDIS_ENCODING_EMBSTR 8 /* Embedded sds string encoding */ #define REDIS_ENCODING_EMBSTR 8 /* Embedded sds string encoding */
#define REDIS_ENCODING_QUICKLIST 9 /* Encoded as linked list of ziplists */
/* Defines related to the dump file format. To store 32 bits lengths for short /* Defines related to the dump file format. To store 32 bits lengths for short
* keys requires a lot of space, so we check the most significant 2 bits of * keys requires a lot of space, so we check the most significant 2 bits of
...@@ -323,12 +331,14 @@ typedef long long mstime_t; /* millisecond time type. */ ...@@ -323,12 +331,14 @@ typedef long long mstime_t; /* millisecond time type. */
/* Zip structure related defaults */ /* Zip structure related defaults */
#define REDIS_HASH_MAX_ZIPLIST_ENTRIES 512 #define REDIS_HASH_MAX_ZIPLIST_ENTRIES 512
#define REDIS_HASH_MAX_ZIPLIST_VALUE 64 #define REDIS_HASH_MAX_ZIPLIST_VALUE 64
#define REDIS_LIST_MAX_ZIPLIST_ENTRIES 512
#define REDIS_LIST_MAX_ZIPLIST_VALUE 64
#define REDIS_SET_MAX_INTSET_ENTRIES 512 #define REDIS_SET_MAX_INTSET_ENTRIES 512
#define REDIS_ZSET_MAX_ZIPLIST_ENTRIES 128 #define REDIS_ZSET_MAX_ZIPLIST_ENTRIES 128
#define REDIS_ZSET_MAX_ZIPLIST_VALUE 64 #define REDIS_ZSET_MAX_ZIPLIST_VALUE 64
/* List defaults */
#define REDIS_LIST_MAX_ZIPLIST_SIZE -2
#define REDIS_LIST_COMPRESS_DEPTH 0
/* HyperLogLog defines */ /* HyperLogLog defines */
#define REDIS_DEFAULT_HLL_SPARSE_MAX_BYTES 3000 #define REDIS_DEFAULT_HLL_SPARSE_MAX_BYTES 3000
...@@ -863,12 +873,14 @@ struct redisServer { ...@@ -863,12 +873,14 @@ struct redisServer {
/* Zip structure config, see redis.conf for more information */ /* Zip structure config, see redis.conf for more information */
size_t hash_max_ziplist_entries; size_t hash_max_ziplist_entries;
size_t hash_max_ziplist_value; size_t hash_max_ziplist_value;
size_t list_max_ziplist_entries;
size_t list_max_ziplist_value;
size_t set_max_intset_entries; size_t set_max_intset_entries;
size_t zset_max_ziplist_entries; size_t zset_max_ziplist_entries;
size_t zset_max_ziplist_value; size_t zset_max_ziplist_value;
size_t hll_sparse_max_bytes; size_t hll_sparse_max_bytes;
/* List parameters */
int list_max_ziplist_size;
int list_compress_depth;
/* time cache */
time_t unixtime; /* Unix time sampled every cron cycle. */ time_t unixtime; /* Unix time sampled every cron cycle. */
long long mstime; /* Like 'unixtime' but with milliseconds resolution. */ long long mstime; /* Like 'unixtime' but with milliseconds resolution. */
/* Pubsub */ /* Pubsub */
...@@ -958,15 +970,13 @@ typedef struct { ...@@ -958,15 +970,13 @@ typedef struct {
robj *subject; robj *subject;
unsigned char encoding; unsigned char encoding;
unsigned char direction; /* Iteration direction */ unsigned char direction; /* Iteration direction */
unsigned char *zi; quicklistIter *iter;
listNode *ln;
} listTypeIterator; } listTypeIterator;
/* Structure for an entry while iterating over a list. */ /* Structure for an entry while iterating over a list. */
typedef struct { typedef struct {
listTypeIterator *li; listTypeIterator *li;
unsigned char *zi; /* Entry in ziplist */ quicklistEntry entry; /* Entry in quicklist */
listNode *ln; /* Entry in linked list */
} listTypeEntry; } listTypeEntry;
/* Structure to hold set iteration abstraction. */ /* Structure to hold set iteration abstraction. */
...@@ -1043,6 +1053,7 @@ void addReplyBulkCBuffer(redisClient *c, void *p, size_t len); ...@@ -1043,6 +1053,7 @@ void addReplyBulkCBuffer(redisClient *c, void *p, size_t len);
void addReplyBulkLongLong(redisClient *c, long long ll); void addReplyBulkLongLong(redisClient *c, long long ll);
void addReply(redisClient *c, robj *obj); void addReply(redisClient *c, robj *obj);
void addReplySds(redisClient *c, sds s); void addReplySds(redisClient *c, sds s);
void addReplyBulkSds(redisClient *c, sds s);
void addReplyError(redisClient *c, char *err); void addReplyError(redisClient *c, char *err);
void addReplyStatus(redisClient *c, char *status); void addReplyStatus(redisClient *c, char *status);
void addReplyDouble(redisClient *c, double d); void addReplyDouble(redisClient *c, double d);
...@@ -1092,7 +1103,7 @@ int listTypeNext(listTypeIterator *li, listTypeEntry *entry); ...@@ -1092,7 +1103,7 @@ int listTypeNext(listTypeIterator *li, listTypeEntry *entry);
robj *listTypeGet(listTypeEntry *entry); robj *listTypeGet(listTypeEntry *entry);
void listTypeInsert(listTypeEntry *entry, robj *value, int where); void listTypeInsert(listTypeEntry *entry, robj *value, int where);
int listTypeEqual(listTypeEntry *entry, robj *o); int listTypeEqual(listTypeEntry *entry, robj *o);
void listTypeDelete(listTypeEntry *entry); void listTypeDelete(listTypeIterator *iter, listTypeEntry *entry);
void listTypeConvert(robj *subject, int enc); void listTypeConvert(robj *subject, int enc);
void unblockClientWaitingData(redisClient *c); void unblockClientWaitingData(redisClient *c);
void handleClientsBlockedOnLists(void); void handleClientsBlockedOnLists(void);
...@@ -1130,7 +1141,7 @@ robj *getDecodedObject(robj *o); ...@@ -1130,7 +1141,7 @@ robj *getDecodedObject(robj *o);
size_t stringObjectLen(robj *o); size_t stringObjectLen(robj *o);
robj *createStringObjectFromLongLong(long long value); robj *createStringObjectFromLongLong(long long value);
robj *createStringObjectFromLongDouble(long double value, int humanfriendly); robj *createStringObjectFromLongDouble(long double value, int humanfriendly);
robj *createListObject(void); robj *createQuicklistObject(void);
robj *createZiplistObject(void); robj *createZiplistObject(void);
robj *createSetObject(void); robj *createSetObject(void);
robj *createIntsetObject(void); robj *createIntsetObject(void);
......
...@@ -88,6 +88,17 @@ void sdsfree(sds s) { ...@@ -88,6 +88,17 @@ void sdsfree(sds s) {
zfree(s-sizeof(struct sdshdr)); zfree(s-sizeof(struct sdshdr));
} }
/* Remove sds header so we can use buffer as malloc'd byte array.
* Returns the contents of 's' usable as a full malloc'd C string. */
char *sdsnative(sds s) {
if (!s) return NULL;
size_t len = sdslen(s);
char *base = s-sizeof(struct sdshdr);
memmove(base, s, len);
return zrealloc(base, len);
}
/* Set the sds string length to the length as obtained with strlen(), so /* Set the sds string length to the length as obtained with strlen(), so
* considering as content only up to the first null term character. * considering as content only up to the first null term character.
* *
...@@ -962,12 +973,15 @@ sds sdsjoin(char **argv, int argc, char *sep) { ...@@ -962,12 +973,15 @@ sds sdsjoin(char **argv, int argc, char *sep) {
return join; return join;
} }
#ifdef SDS_TEST_MAIN #if defined(REDIS_TEST) || defined(SDS_TEST_MAIN)
#include <stdio.h> #include <stdio.h>
#include "testhelp.h" #include "testhelp.h"
#include "limits.h" #include "limits.h"
int main(void) { #define UNUSED(x) (void)(x)
int sdsTest(int argc, char *argv[]) {
UNUSED(argc);
UNUSED(argv);
{ {
struct sdshdr *sh; struct sdshdr *sh;
sds x = sdsnew("foo"), y; sds x = sdsnew("foo"), y;
...@@ -1092,7 +1106,7 @@ int main(void) { ...@@ -1092,7 +1106,7 @@ int main(void) {
memcmp(y,"\"\\a\\n\\x00foo\\r\"",15) == 0) memcmp(y,"\"\\a\\n\\x00foo\\r\"",15) == 0)
{ {
int oldfree; unsigned int oldfree;
sdsfree(x); sdsfree(x);
x = sdsnew("0"); x = sdsnew("0");
...@@ -1113,3 +1127,9 @@ int main(void) { ...@@ -1113,3 +1127,9 @@ int main(void) {
return 0; return 0;
} }
#endif #endif
#ifdef SDS_TEST_MAIN
int main(void) {
return sdsTest();
}
#endif
...@@ -60,6 +60,7 @@ sds sdsempty(void); ...@@ -60,6 +60,7 @@ sds sdsempty(void);
size_t sdslen(const sds s); size_t sdslen(const sds s);
sds sdsdup(const sds s); sds sdsdup(const sds s);
void sdsfree(sds s); void sdsfree(sds s);
char *sdsnative(sds s);
size_t sdsavail(const sds s); size_t sdsavail(const sds s);
sds sdsgrowzero(sds s, size_t len); sds sdsgrowzero(sds s, size_t len);
sds sdscatlen(sds s, const void *t, size_t len); sds sdscatlen(sds s, const void *t, size_t len);
...@@ -98,4 +99,8 @@ void sdsIncrLen(sds s, int incr); ...@@ -98,4 +99,8 @@ void sdsIncrLen(sds s, int incr);
sds sdsRemoveFreeSpace(sds s); sds sdsRemoveFreeSpace(sds s);
size_t sdsAllocSize(sds s); size_t sdsAllocSize(sds s);
#ifdef REDIS_TEST
int sdsTest(int argc, char *argv[]);
#endif
#endif #endif
...@@ -577,7 +577,7 @@ void sentinelEvent(int level, char *type, sentinelRedisInstance *ri, ...@@ -577,7 +577,7 @@ void sentinelEvent(int level, char *type, sentinelRedisInstance *ri,
if (level == REDIS_WARNING && ri != NULL) { if (level == REDIS_WARNING && ri != NULL) {
sentinelRedisInstance *master = (ri->flags & SRI_MASTER) ? sentinelRedisInstance *master = (ri->flags & SRI_MASTER) ?
ri : ri->master; ri : ri->master;
if (master->notification_script) { if (master && master->notification_script) {
sentinelScheduleScriptExecution(master->notification_script, sentinelScheduleScriptExecution(master->notification_script,
type,msg,NULL); type,msg,NULL);
} }
...@@ -2908,10 +2908,7 @@ void sentinelInfoCommand(redisClient *c) { ...@@ -2908,10 +2908,7 @@ void sentinelInfoCommand(redisClient *c) {
dictReleaseIterator(di); dictReleaseIterator(di);
} }
addReplySds(c,sdscatprintf(sdsempty(),"$%lu\r\n", addReplyBulkSds(c, info);
(unsigned long)sdslen(info)));
addReplySds(c,info);
addReply(c,shared.crlf);
} }
/* Implements Sentinel verison of the ROLE command. The output is /* Implements Sentinel verison of the ROLE command. The output is
......
...@@ -199,16 +199,19 @@ void SHA1Final(unsigned char digest[20], SHA1_CTX* context) ...@@ -199,16 +199,19 @@ void SHA1Final(unsigned char digest[20], SHA1_CTX* context)
} }
/* ================ end of sha1.c ================ */ /* ================ end of sha1.c ================ */
#if 0 #ifdef REDIS_TEST
#define BUFSIZE 4096 #define BUFSIZE 4096
int #define UNUSED(x) (void)(x)
main(int argc, char **argv) int sha1Test(int argc, char **argv)
{ {
SHA1_CTX ctx; SHA1_CTX ctx;
unsigned char hash[20], buf[BUFSIZE]; unsigned char hash[20], buf[BUFSIZE];
int i; int i;
UNUSED(argc);
UNUSED(argv);
for(i=0;i<BUFSIZE;i++) for(i=0;i<BUFSIZE;i++)
buf[i] = i; buf[i] = i;
...@@ -223,6 +226,4 @@ main(int argc, char **argv) ...@@ -223,6 +226,4 @@ main(int argc, char **argv)
printf("\n"); printf("\n");
return 0; return 0;
} }
#endif #endif
#ifndef SHA1_H
#define SHA1_H
/* ================ sha1.h ================ */ /* ================ sha1.h ================ */
/* /*
SHA-1 in C SHA-1 in C
...@@ -15,3 +17,8 @@ void SHA1Transform(u_int32_t state[5], const unsigned char buffer[64]); ...@@ -15,3 +17,8 @@ void SHA1Transform(u_int32_t state[5], const unsigned char buffer[64]);
void SHA1Init(SHA1_CTX* context); void SHA1Init(SHA1_CTX* context);
void SHA1Update(SHA1_CTX* context, const unsigned char* data, u_int32_t len); void SHA1Update(SHA1_CTX* context, const unsigned char* data, u_int32_t len);
void SHA1Final(unsigned char digest[20], SHA1_CTX* context); void SHA1Final(unsigned char digest[20], SHA1_CTX* context);
#ifdef REDIS_TEST
int sha1Test(int argc, char **argv);
#endif
#endif
...@@ -220,7 +220,7 @@ void sortCommand(redisClient *c) { ...@@ -220,7 +220,7 @@ void sortCommand(redisClient *c) {
if (sortval) if (sortval)
incrRefCount(sortval); incrRefCount(sortval);
else else
sortval = createListObject(); sortval = createQuicklistObject();
/* The SORT command has an SQL-alike syntax, parse it */ /* The SORT command has an SQL-alike syntax, parse it */
while(j < c->argc) { while(j < c->argc) {
...@@ -420,6 +420,7 @@ void sortCommand(redisClient *c) { ...@@ -420,6 +420,7 @@ void sortCommand(redisClient *c) {
} else { } else {
redisPanic("Unknown type"); redisPanic("Unknown type");
} }
printf("j: %d; vectorlen: %d\n", j, vectorlen);
redisAssertWithInfo(c,sortval,j == vectorlen); redisAssertWithInfo(c,sortval,j == vectorlen);
/* Now it's time to load the right scores in the sorting vector */ /* Now it's time to load the right scores in the sorting vector */
...@@ -509,7 +510,7 @@ void sortCommand(redisClient *c) { ...@@ -509,7 +510,7 @@ void sortCommand(redisClient *c) {
} }
} }
} else { } else {
robj *sobj = createZiplistObject(); robj *sobj = createQuicklistObject();
/* STORE option specified, set the sorting result as a List object */ /* STORE option specified, set the sorting result as a List object */
for (j = start; j <= end; j++) { for (j = start; j <= end; j++) {
......
...@@ -33,75 +33,37 @@ ...@@ -33,75 +33,37 @@
* List API * List API
*----------------------------------------------------------------------------*/ *----------------------------------------------------------------------------*/
/* Check the argument length to see if it requires us to convert the ziplist
* to a real list. Only check raw-encoded objects because integer encoded
* objects are never too long. */
void listTypeTryConversion(robj *subject, robj *value) {
if (subject->encoding != REDIS_ENCODING_ZIPLIST) return;
if (sdsEncodedObject(value) &&
sdslen(value->ptr) > server.list_max_ziplist_value)
listTypeConvert(subject,REDIS_ENCODING_LINKEDLIST);
}
/* The function pushes an element to the specified list object 'subject', /* The function pushes an element to the specified list object 'subject',
* at head or tail position as specified by 'where'. * at head or tail position as specified by 'where'.
* *
* There is no need for the caller to increment the refcount of 'value' as * There is no need for the caller to increment the refcount of 'value' as
* the function takes care of it if needed. */ * the function takes care of it if needed. */
void listTypePush(robj *subject, robj *value, int where) { void listTypePush(robj *subject, robj *value, int where) {
/* Check if we need to convert the ziplist */ if (subject->encoding == REDIS_ENCODING_QUICKLIST) {
listTypeTryConversion(subject,value); int pos = (where == REDIS_HEAD) ? QUICKLIST_HEAD : QUICKLIST_TAIL;
if (subject->encoding == REDIS_ENCODING_ZIPLIST &&
ziplistLen(subject->ptr) >= server.list_max_ziplist_entries)
listTypeConvert(subject,REDIS_ENCODING_LINKEDLIST);
if (subject->encoding == REDIS_ENCODING_ZIPLIST) {
int pos = (where == REDIS_HEAD) ? ZIPLIST_HEAD : ZIPLIST_TAIL;
value = getDecodedObject(value); value = getDecodedObject(value);
subject->ptr = ziplistPush(subject->ptr,value->ptr,sdslen(value->ptr),pos); size_t len = sdslen(value->ptr);
quicklistPush(subject->ptr, value->ptr, len, pos);
decrRefCount(value); decrRefCount(value);
} else if (subject->encoding == REDIS_ENCODING_LINKEDLIST) {
if (where == REDIS_HEAD) {
listAddNodeHead(subject->ptr,value);
} else {
listAddNodeTail(subject->ptr,value);
}
incrRefCount(value);
} else { } else {
redisPanic("Unknown list encoding"); redisPanic("Unknown list encoding");
} }
} }
void *listPopSaver(unsigned char *data, unsigned int sz) {
return createStringObject((char*)data,sz);
}
robj *listTypePop(robj *subject, int where) { robj *listTypePop(robj *subject, int where) {
long long vlong;
robj *value = NULL; robj *value = NULL;
if (subject->encoding == REDIS_ENCODING_ZIPLIST) {
unsigned char *p; int ql_where = where == REDIS_HEAD ? QUICKLIST_HEAD : QUICKLIST_TAIL;
unsigned char *vstr; if (subject->encoding == REDIS_ENCODING_QUICKLIST) {
unsigned int vlen; if (quicklistPopCustom(subject->ptr, ql_where, (unsigned char **)&value,
long long vlong; NULL, &vlong, listPopSaver)) {
int pos = (where == REDIS_HEAD) ? 0 : -1; if (!value)
p = ziplistIndex(subject->ptr,pos);
if (ziplistGet(p,&vstr,&vlen,&vlong)) {
if (vstr) {
value = createStringObject((char*)vstr,vlen);
} else {
value = createStringObjectFromLongLong(vlong); value = createStringObjectFromLongLong(vlong);
}
/* We only need to delete an element when it exists */
subject->ptr = ziplistDelete(subject->ptr,&p);
}
} else if (subject->encoding == REDIS_ENCODING_LINKEDLIST) {
list *list = subject->ptr;
listNode *ln;
if (where == REDIS_HEAD) {
ln = listFirst(list);
} else {
ln = listLast(list);
}
if (ln != NULL) {
value = listNodeValue(ln);
incrRefCount(value);
listDelNode(list,ln);
} }
} else { } else {
redisPanic("Unknown list encoding"); redisPanic("Unknown list encoding");
...@@ -110,25 +72,28 @@ robj *listTypePop(robj *subject, int where) { ...@@ -110,25 +72,28 @@ robj *listTypePop(robj *subject, int where) {
} }
unsigned long listTypeLength(robj *subject) { unsigned long listTypeLength(robj *subject) {
if (subject->encoding == REDIS_ENCODING_ZIPLIST) { if (subject->encoding == REDIS_ENCODING_QUICKLIST) {
return ziplistLen(subject->ptr); return quicklistCount(subject->ptr);
} else if (subject->encoding == REDIS_ENCODING_LINKEDLIST) {
return listLength((list*)subject->ptr);
} else { } else {
redisPanic("Unknown list encoding"); redisPanic("Unknown list encoding");
} }
} }
/* Initialize an iterator at the specified index. */ /* Initialize an iterator at the specified index. */
listTypeIterator *listTypeInitIterator(robj *subject, long index, unsigned char direction) { listTypeIterator *listTypeInitIterator(robj *subject, long index,
unsigned char direction) {
listTypeIterator *li = zmalloc(sizeof(listTypeIterator)); listTypeIterator *li = zmalloc(sizeof(listTypeIterator));
li->subject = subject; li->subject = subject;
li->encoding = subject->encoding; li->encoding = subject->encoding;
li->direction = direction; li->direction = direction;
if (li->encoding == REDIS_ENCODING_ZIPLIST) { li->iter = NULL;
li->zi = ziplistIndex(subject->ptr,index); /* REDIS_HEAD means start at TAIL and move *towards* head.
} else if (li->encoding == REDIS_ENCODING_LINKEDLIST) { * REDIS_TAIL means start at HEAD and move *towards tail. */
li->ln = listIndex(subject->ptr,index); int iter_direction =
direction == REDIS_HEAD ? AL_START_TAIL : AL_START_HEAD;
if (li->encoding == REDIS_ENCODING_QUICKLIST) {
li->iter = quicklistGetIteratorAtIdx(li->subject->ptr,
iter_direction, index);
} else { } else {
redisPanic("Unknown list encoding"); redisPanic("Unknown list encoding");
} }
...@@ -137,6 +102,7 @@ listTypeIterator *listTypeInitIterator(robj *subject, long index, unsigned char ...@@ -137,6 +102,7 @@ listTypeIterator *listTypeInitIterator(robj *subject, long index, unsigned char
/* Clean up the iterator. */ /* Clean up the iterator. */
void listTypeReleaseIterator(listTypeIterator *li) { void listTypeReleaseIterator(listTypeIterator *li) {
zfree(li->iter);
zfree(li); zfree(li);
} }
...@@ -148,24 +114,8 @@ int listTypeNext(listTypeIterator *li, listTypeEntry *entry) { ...@@ -148,24 +114,8 @@ int listTypeNext(listTypeIterator *li, listTypeEntry *entry) {
redisAssert(li->subject->encoding == li->encoding); redisAssert(li->subject->encoding == li->encoding);
entry->li = li; entry->li = li;
if (li->encoding == REDIS_ENCODING_ZIPLIST) { if (li->encoding == REDIS_ENCODING_QUICKLIST) {
entry->zi = li->zi; return quicklistNext(li->iter, &entry->entry);
if (entry->zi != NULL) {
if (li->direction == REDIS_TAIL)
li->zi = ziplistNext(li->subject->ptr,li->zi);
else
li->zi = ziplistPrev(li->subject->ptr,li->zi);
return 1;
}
} else if (li->encoding == REDIS_ENCODING_LINKEDLIST) {
entry->ln = li->ln;
if (entry->ln != NULL) {
if (li->direction == REDIS_TAIL)
li->ln = li->ln->next;
else
li->ln = li->ln->prev;
return 1;
}
} else { } else {
redisPanic("Unknown list encoding"); redisPanic("Unknown list encoding");
} }
...@@ -174,24 +124,14 @@ int listTypeNext(listTypeIterator *li, listTypeEntry *entry) { ...@@ -174,24 +124,14 @@ int listTypeNext(listTypeIterator *li, listTypeEntry *entry) {
/* Return entry or NULL at the current position of the iterator. */ /* Return entry or NULL at the current position of the iterator. */
robj *listTypeGet(listTypeEntry *entry) { robj *listTypeGet(listTypeEntry *entry) {
listTypeIterator *li = entry->li;
robj *value = NULL; robj *value = NULL;
if (li->encoding == REDIS_ENCODING_ZIPLIST) { if (entry->li->encoding == REDIS_ENCODING_QUICKLIST) {
unsigned char *vstr; if (entry->entry.value) {
unsigned int vlen; value = createStringObject((char *)entry->entry.value,
long long vlong; entry->entry.sz);
redisAssert(entry->zi != NULL); } else {
if (ziplistGet(entry->zi,&vstr,&vlen,&vlong)) { value = createStringObjectFromLongLong(entry->entry.longval);
if (vstr) {
value = createStringObject((char*)vstr,vlen);
} else {
value = createStringObjectFromLongLong(vlong);
}
} }
} else if (li->encoding == REDIS_ENCODING_LINKEDLIST) {
redisAssert(entry->ln != NULL);
value = listNodeValue(entry->ln);
incrRefCount(value);
} else { } else {
redisPanic("Unknown list encoding"); redisPanic("Unknown list encoding");
} }
...@@ -199,30 +139,18 @@ robj *listTypeGet(listTypeEntry *entry) { ...@@ -199,30 +139,18 @@ robj *listTypeGet(listTypeEntry *entry) {
} }
void listTypeInsert(listTypeEntry *entry, robj *value, int where) { void listTypeInsert(listTypeEntry *entry, robj *value, int where) {
robj *subject = entry->li->subject; if (entry->li->encoding == REDIS_ENCODING_QUICKLIST) {
if (entry->li->encoding == REDIS_ENCODING_ZIPLIST) {
value = getDecodedObject(value); value = getDecodedObject(value);
sds str = value->ptr;
size_t len = sdslen(str);
if (where == REDIS_TAIL) { if (where == REDIS_TAIL) {
unsigned char *next = ziplistNext(subject->ptr,entry->zi); quicklistInsertAfter((quicklist *)entry->entry.quicklist,
&entry->entry, str, len);
/* When we insert after the current element, but the current element } else if (where == REDIS_HEAD) {
* is the tail of the list, we need to do a push. */ quicklistInsertBefore((quicklist *)entry->entry.quicklist,
if (next == NULL) { &entry->entry, str, len);
subject->ptr = ziplistPush(subject->ptr,value->ptr,sdslen(value->ptr),REDIS_TAIL);
} else {
subject->ptr = ziplistInsert(subject->ptr,next,value->ptr,sdslen(value->ptr));
}
} else {
subject->ptr = ziplistInsert(subject->ptr,entry->zi,value->ptr,sdslen(value->ptr));
} }
decrRefCount(value); decrRefCount(value);
} else if (entry->li->encoding == REDIS_ENCODING_LINKEDLIST) {
if (where == REDIS_TAIL) {
listInsertNode(subject->ptr,entry->ln,value,AL_START_TAIL);
} else {
listInsertNode(subject->ptr,entry->ln,value,AL_START_HEAD);
}
incrRefCount(value);
} else { } else {
redisPanic("Unknown list encoding"); redisPanic("Unknown list encoding");
} }
...@@ -230,59 +158,33 @@ void listTypeInsert(listTypeEntry *entry, robj *value, int where) { ...@@ -230,59 +158,33 @@ void listTypeInsert(listTypeEntry *entry, robj *value, int where) {
/* Compare the given object with the entry at the current position. */ /* Compare the given object with the entry at the current position. */
int listTypeEqual(listTypeEntry *entry, robj *o) { int listTypeEqual(listTypeEntry *entry, robj *o) {
listTypeIterator *li = entry->li; if (entry->li->encoding == REDIS_ENCODING_QUICKLIST) {
if (li->encoding == REDIS_ENCODING_ZIPLIST) {
redisAssertWithInfo(NULL,o,sdsEncodedObject(o)); redisAssertWithInfo(NULL,o,sdsEncodedObject(o));
return ziplistCompare(entry->zi,o->ptr,sdslen(o->ptr)); return quicklistCompare(entry->entry.zi,o->ptr,sdslen(o->ptr));
} else if (li->encoding == REDIS_ENCODING_LINKEDLIST) {
return equalStringObjects(o,listNodeValue(entry->ln));
} else { } else {
redisPanic("Unknown list encoding"); redisPanic("Unknown list encoding");
} }
} }
/* Delete the element pointed to. */ /* Delete the element pointed to. */
void listTypeDelete(listTypeEntry *entry) { void listTypeDelete(listTypeIterator *iter, listTypeEntry *entry) {
listTypeIterator *li = entry->li; if (entry->li->encoding == REDIS_ENCODING_QUICKLIST) {
if (li->encoding == REDIS_ENCODING_ZIPLIST) { quicklistDelEntry(iter->iter, &entry->entry);
unsigned char *p = entry->zi;
li->subject->ptr = ziplistDelete(li->subject->ptr,&p);
/* Update position of the iterator depending on the direction */
if (li->direction == REDIS_TAIL)
li->zi = p;
else
li->zi = ziplistPrev(li->subject->ptr,p);
} else if (entry->li->encoding == REDIS_ENCODING_LINKEDLIST) {
listNode *next;
if (li->direction == REDIS_TAIL)
next = entry->ln->next;
else
next = entry->ln->prev;
listDelNode(li->subject->ptr,entry->ln);
li->ln = next;
} else { } else {
redisPanic("Unknown list encoding"); redisPanic("Unknown list encoding");
} }
} }
/* Create a quicklist from a single ziplist */
void listTypeConvert(robj *subject, int enc) { void listTypeConvert(robj *subject, int enc) {
listTypeIterator *li; redisAssertWithInfo(NULL,subject,subject->type==REDIS_LIST);
listTypeEntry entry; redisAssertWithInfo(NULL,subject,subject->encoding==REDIS_ENCODING_ZIPLIST);
redisAssertWithInfo(NULL,subject,subject->type == REDIS_LIST);
if (enc == REDIS_ENCODING_QUICKLIST) {
if (enc == REDIS_ENCODING_LINKEDLIST) { size_t zlen = server.list_max_ziplist_size;
list *l = listCreate(); int depth = server.list_compress_depth;
listSetFreeMethod(l,decrRefCountVoid); subject->ptr = quicklistCreateFromZiplist(zlen, depth, subject->ptr);
subject->encoding = REDIS_ENCODING_QUICKLIST;
/* listTypeGet returns a robj with incremented refcount */
li = listTypeInitIterator(subject,0,REDIS_TAIL);
while (listTypeNext(li,&entry)) listAddNodeTail(l,listTypeGet(&entry));
listTypeReleaseIterator(li);
subject->encoding = REDIS_ENCODING_LINKEDLIST;
zfree(subject->ptr);
subject->ptr = l;
} else { } else {
redisPanic("Unsupported list conversion"); redisPanic("Unsupported list conversion");
} }
...@@ -304,7 +206,9 @@ void pushGenericCommand(redisClient *c, int where) { ...@@ -304,7 +206,9 @@ void pushGenericCommand(redisClient *c, int where) {
for (j = 2; j < c->argc; j++) { for (j = 2; j < c->argc; j++) {
c->argv[j] = tryObjectEncoding(c->argv[j]); c->argv[j] = tryObjectEncoding(c->argv[j]);
if (!lobj) { if (!lobj) {
lobj = createZiplistObject(); lobj = createQuicklistObject();
quicklistSetOptions(lobj->ptr, server.list_max_ziplist_size,
server.list_compress_depth);
dbAdd(c->db,c->argv[1],lobj); dbAdd(c->db,c->argv[1],lobj);
} }
listTypePush(lobj,c->argv[j],where); listTypePush(lobj,c->argv[j],where);
...@@ -338,13 +242,6 @@ void pushxGenericCommand(redisClient *c, robj *refval, robj *val, int where) { ...@@ -338,13 +242,6 @@ void pushxGenericCommand(redisClient *c, robj *refval, robj *val, int where) {
checkType(c,subject,REDIS_LIST)) return; checkType(c,subject,REDIS_LIST)) return;
if (refval != NULL) { if (refval != NULL) {
/* We're not sure if this value can be inserted yet, but we cannot
* convert the list inside the iterator. We don't want to loop over
* the list twice (once to see if the value can be inserted and once
* to do the actual insert), so we assume this value can be inserted
* and convert the ziplist to a regular list if necessary. */
listTypeTryConversion(subject,val);
/* Seek refval from head to tail */ /* Seek refval from head to tail */
iter = listTypeInitIterator(subject,0,REDIS_TAIL); iter = listTypeInitIterator(subject,0,REDIS_TAIL);
while (listTypeNext(iter,&entry)) { while (listTypeNext(iter,&entry)) {
...@@ -357,10 +254,6 @@ void pushxGenericCommand(redisClient *c, robj *refval, robj *val, int where) { ...@@ -357,10 +254,6 @@ void pushxGenericCommand(redisClient *c, robj *refval, robj *val, int where) {
listTypeReleaseIterator(iter); listTypeReleaseIterator(iter);
if (inserted) { if (inserted) {
/* Check if the length exceeds the ziplist length threshold. */
if (subject->encoding == REDIS_ENCODING_ZIPLIST &&
ziplistLen(subject->ptr) > server.list_max_ziplist_entries)
listTypeConvert(subject,REDIS_ENCODING_LINKEDLIST);
signalModifiedKey(c->db,c->argv[1]); signalModifiedKey(c->db,c->argv[1]);
notifyKeyspaceEvent(REDIS_NOTIFY_LIST,"linsert", notifyKeyspaceEvent(REDIS_NOTIFY_LIST,"linsert",
c->argv[1],c->db->id); c->argv[1],c->db->id);
...@@ -418,31 +311,19 @@ void lindexCommand(redisClient *c) { ...@@ -418,31 +311,19 @@ void lindexCommand(redisClient *c) {
if ((getLongFromObjectOrReply(c, c->argv[2], &index, NULL) != REDIS_OK)) if ((getLongFromObjectOrReply(c, c->argv[2], &index, NULL) != REDIS_OK))
return; return;
if (o->encoding == REDIS_ENCODING_ZIPLIST) { if (o->encoding == REDIS_ENCODING_QUICKLIST) {
unsigned char *p; quicklistEntry entry;
unsigned char *vstr; if (quicklistIndex(o->ptr, index, &entry)) {
unsigned int vlen; if (entry.value) {
long long vlong; value = createStringObject((char*)entry.value,entry.sz);
p = ziplistIndex(o->ptr,index);
if (ziplistGet(p,&vstr,&vlen,&vlong)) {
if (vstr) {
value = createStringObject((char*)vstr,vlen);
} else { } else {
value = createStringObjectFromLongLong(vlong); value = createStringObjectFromLongLong(entry.longval);
} }
addReplyBulk(c,value); addReplyBulk(c,value);
decrRefCount(value); decrRefCount(value);
} else { } else {
addReply(c,shared.nullbulk); addReply(c,shared.nullbulk);
} }
} else if (o->encoding == REDIS_ENCODING_LINKEDLIST) {
listNode *ln = listIndex(o->ptr,index);
if (ln != NULL) {
value = listNodeValue(ln);
addReplyBulk(c,value);
} else {
addReply(c,shared.nullbulk);
}
} else { } else {
redisPanic("Unknown list encoding"); redisPanic("Unknown list encoding");
} }
...@@ -452,35 +333,18 @@ void lsetCommand(redisClient *c) { ...@@ -452,35 +333,18 @@ void lsetCommand(redisClient *c) {
robj *o = lookupKeyWriteOrReply(c,c->argv[1],shared.nokeyerr); robj *o = lookupKeyWriteOrReply(c,c->argv[1],shared.nokeyerr);
if (o == NULL || checkType(c,o,REDIS_LIST)) return; if (o == NULL || checkType(c,o,REDIS_LIST)) return;
long index; long index;
robj *value = (c->argv[3] = tryObjectEncoding(c->argv[3])); robj *value = c->argv[3];
if ((getLongFromObjectOrReply(c, c->argv[2], &index, NULL) != REDIS_OK)) if ((getLongFromObjectOrReply(c, c->argv[2], &index, NULL) != REDIS_OK))
return; return;
listTypeTryConversion(o,value); if (o->encoding == REDIS_ENCODING_QUICKLIST) {
if (o->encoding == REDIS_ENCODING_ZIPLIST) { quicklist *ql = o->ptr;
unsigned char *p, *zl = o->ptr; int replaced = quicklistReplaceAtIndex(ql, index,
p = ziplistIndex(zl,index); value->ptr, sdslen(value->ptr));
if (p == NULL) { if (!replaced) {
addReply(c,shared.outofrangeerr); addReply(c,shared.outofrangeerr);
} else { } else {
o->ptr = ziplistDelete(o->ptr,&p);
value = getDecodedObject(value);
o->ptr = ziplistInsert(o->ptr,p,value->ptr,sdslen(value->ptr));
decrRefCount(value);
addReply(c,shared.ok);
signalModifiedKey(c->db,c->argv[1]);
notifyKeyspaceEvent(REDIS_NOTIFY_LIST,"lset",c->argv[1],c->db->id);
server.dirty++;
}
} else if (o->encoding == REDIS_ENCODING_LINKEDLIST) {
listNode *ln = listIndex(o->ptr,index);
if (ln == NULL) {
addReply(c,shared.outofrangeerr);
} else {
decrRefCount((robj*)listNodeValue(ln));
listNodeValue(ln) = value;
incrRefCount(value);
addReply(c,shared.ok); addReply(c,shared.ok);
signalModifiedKey(c->db,c->argv[1]); signalModifiedKey(c->db,c->argv[1]);
notifyKeyspaceEvent(REDIS_NOTIFY_LIST,"lset",c->argv[1],c->db->id); notifyKeyspaceEvent(REDIS_NOTIFY_LIST,"lset",c->argv[1],c->db->id);
...@@ -549,43 +413,28 @@ void lrangeCommand(redisClient *c) { ...@@ -549,43 +413,28 @@ void lrangeCommand(redisClient *c) {
/* Return the result in form of a multi-bulk reply */ /* Return the result in form of a multi-bulk reply */
addReplyMultiBulkLen(c,rangelen); addReplyMultiBulkLen(c,rangelen);
if (o->encoding == REDIS_ENCODING_ZIPLIST) { if (o->encoding == REDIS_ENCODING_QUICKLIST) {
unsigned char *p = ziplistIndex(o->ptr,start); listTypeIterator *iter = listTypeInitIterator(o, start, REDIS_TAIL);
unsigned char *vstr;
unsigned int vlen;
long long vlong;
while(rangelen--) { while(rangelen--) {
ziplistGet(p,&vstr,&vlen,&vlong); listTypeEntry entry;
if (vstr) { listTypeNext(iter, &entry);
addReplyBulkCBuffer(c,vstr,vlen); quicklistEntry *qe = &entry.entry;
if (qe->value) {
addReplyBulkCBuffer(c,qe->value,qe->sz);
} else { } else {
addReplyBulkLongLong(c,vlong); addReplyBulkLongLong(c,qe->longval);
} }
p = ziplistNext(o->ptr,p);
}
} else if (o->encoding == REDIS_ENCODING_LINKEDLIST) {
listNode *ln;
/* If we are nearest to the end of the list, reach the element
* starting from tail and going backward, as it is faster. */
if (start > llen/2) start -= llen;
ln = listIndex(o->ptr,start);
while(rangelen--) {
addReplyBulk(c,ln->value);
ln = ln->next;
} }
listTypeReleaseIterator(iter);
} else { } else {
redisPanic("List encoding is not LINKEDLIST nor ZIPLIST!"); redisPanic("List encoding is not QUICKLIST!");
} }
} }
void ltrimCommand(redisClient *c) { void ltrimCommand(redisClient *c) {
robj *o; robj *o;
long start, end, llen, j, ltrim, rtrim; long start, end, llen, ltrim, rtrim;
list *list;
listNode *ln;
if ((getLongFromObjectOrReply(c, c->argv[2], &start, NULL) != REDIS_OK) || if ((getLongFromObjectOrReply(c, c->argv[2], &start, NULL) != REDIS_OK) ||
(getLongFromObjectOrReply(c, c->argv[3], &end, NULL) != REDIS_OK)) return; (getLongFromObjectOrReply(c, c->argv[3], &end, NULL) != REDIS_OK)) return;
...@@ -612,19 +461,9 @@ void ltrimCommand(redisClient *c) { ...@@ -612,19 +461,9 @@ void ltrimCommand(redisClient *c) {
} }
/* Remove list elements to perform the trim */ /* Remove list elements to perform the trim */
if (o->encoding == REDIS_ENCODING_ZIPLIST) { if (o->encoding == REDIS_ENCODING_QUICKLIST) {
o->ptr = ziplistDeleteRange(o->ptr,0,ltrim); quicklistDelRange(o->ptr,0,ltrim);
o->ptr = ziplistDeleteRange(o->ptr,-rtrim,rtrim); quicklistDelRange(o->ptr,-rtrim,rtrim);
} else if (o->encoding == REDIS_ENCODING_LINKEDLIST) {
list = o->ptr;
for (j = 0; j < ltrim; j++) {
ln = listFirst(list);
listDelNode(list,ln);
}
for (j = 0; j < rtrim; j++) {
ln = listLast(list);
listDelNode(list,ln);
}
} else { } else {
redisPanic("Unknown list encoding"); redisPanic("Unknown list encoding");
} }
...@@ -641,10 +480,9 @@ void ltrimCommand(redisClient *c) { ...@@ -641,10 +480,9 @@ void ltrimCommand(redisClient *c) {
void lremCommand(redisClient *c) { void lremCommand(redisClient *c) {
robj *subject, *obj; robj *subject, *obj;
obj = c->argv[3] = tryObjectEncoding(c->argv[3]); obj = c->argv[3];
long toremove; long toremove;
long removed = 0; long removed = 0;
listTypeEntry entry;
if ((getLongFromObjectOrReply(c, c->argv[2], &toremove, NULL) != REDIS_OK)) if ((getLongFromObjectOrReply(c, c->argv[2], &toremove, NULL) != REDIS_OK))
return; return;
...@@ -652,10 +490,6 @@ void lremCommand(redisClient *c) { ...@@ -652,10 +490,6 @@ void lremCommand(redisClient *c) {
subject = lookupKeyWriteOrReply(c,c->argv[1],shared.czero); subject = lookupKeyWriteOrReply(c,c->argv[1],shared.czero);
if (subject == NULL || checkType(c,subject,REDIS_LIST)) return; if (subject == NULL || checkType(c,subject,REDIS_LIST)) return;
/* Make sure obj is raw when we're dealing with a ziplist */
if (subject->encoding == REDIS_ENCODING_ZIPLIST)
obj = getDecodedObject(obj);
listTypeIterator *li; listTypeIterator *li;
if (toremove < 0) { if (toremove < 0) {
toremove = -toremove; toremove = -toremove;
...@@ -664,9 +498,10 @@ void lremCommand(redisClient *c) { ...@@ -664,9 +498,10 @@ void lremCommand(redisClient *c) {
li = listTypeInitIterator(subject,0,REDIS_TAIL); li = listTypeInitIterator(subject,0,REDIS_TAIL);
} }
listTypeEntry entry;
while (listTypeNext(li,&entry)) { while (listTypeNext(li,&entry)) {
if (listTypeEqual(&entry,obj)) { if (listTypeEqual(&entry,obj)) {
listTypeDelete(&entry); listTypeDelete(li, &entry);
server.dirty++; server.dirty++;
removed++; removed++;
if (toremove && removed == toremove) break; if (toremove && removed == toremove) break;
...@@ -674,11 +509,10 @@ void lremCommand(redisClient *c) { ...@@ -674,11 +509,10 @@ void lremCommand(redisClient *c) {
} }
listTypeReleaseIterator(li); listTypeReleaseIterator(li);
/* Clean up raw encoded object */ if (listTypeLength(subject) == 0) {
if (subject->encoding == REDIS_ENCODING_ZIPLIST) dbDelete(c->db,c->argv[1]);
decrRefCount(obj); }
if (listTypeLength(subject) == 0) dbDelete(c->db,c->argv[1]);
addReplyLongLong(c,removed); addReplyLongLong(c,removed);
if (removed) signalModifiedKey(c->db,c->argv[1]); if (removed) signalModifiedKey(c->db,c->argv[1]);
} }
...@@ -702,7 +536,9 @@ void lremCommand(redisClient *c) { ...@@ -702,7 +536,9 @@ void lremCommand(redisClient *c) {
void rpoplpushHandlePush(redisClient *c, robj *dstkey, robj *dstobj, robj *value) { void rpoplpushHandlePush(redisClient *c, robj *dstkey, robj *dstobj, robj *value) {
/* Create the list if the key does not exist */ /* Create the list if the key does not exist */
if (!dstobj) { if (!dstobj) {
dstobj = createZiplistObject(); dstobj = createQuicklistObject();
quicklistSetOptions(dstobj->ptr, server.list_max_ziplist_size,
server.list_compress_depth);
dbAdd(c->db,dstkey,dstobj); dbAdd(c->db,dstkey,dstobj);
} }
signalModifiedKey(c->db,dstkey); signalModifiedKey(c->db,dstkey);
...@@ -1010,7 +846,9 @@ void handleClientsBlockedOnLists(void) { ...@@ -1010,7 +846,9 @@ void handleClientsBlockedOnLists(void) {
} }
} }
if (listTypeLength(o) == 0) dbDelete(rl->db,rl->key); if (listTypeLength(o) == 0) {
dbDelete(rl->db,rl->key);
}
/* We don't call signalModifiedKey() as it was already called /* We don't call signalModifiedKey() as it was already called
* when an element was pushed on the list. */ * when an element was pushed on the list. */
} }
......
...@@ -1382,7 +1382,7 @@ void zremrangeGenericCommand(redisClient *c, int rangetype) { ...@@ -1382,7 +1382,7 @@ void zremrangeGenericCommand(redisClient *c, int rangetype) {
robj *key = c->argv[1]; robj *key = c->argv[1];
robj *zobj; robj *zobj;
int keyremoved = 0; int keyremoved = 0;
unsigned long deleted; unsigned long deleted = 0;
zrangespec range; zrangespec range;
zlexrangespec lexrange; zlexrangespec lexrange;
long start, end, llen; long start, end, llen;
......
...@@ -529,10 +529,10 @@ int pathIsBaseName(char *path) { ...@@ -529,10 +529,10 @@ int pathIsBaseName(char *path) {
return strchr(path,'/') == NULL && strchr(path,'\\') == NULL; return strchr(path,'/') == NULL && strchr(path,'\\') == NULL;
} }
#ifdef UTIL_TEST_MAIN #ifdef REDIS_TEST
#include <assert.h> #include <assert.h>
void test_string2ll(void) { static void test_string2ll(void) {
char buf[32]; char buf[32];
long long v; long long v;
...@@ -587,7 +587,7 @@ void test_string2ll(void) { ...@@ -587,7 +587,7 @@ void test_string2ll(void) {
assert(string2ll(buf,strlen(buf),&v) == 0); assert(string2ll(buf,strlen(buf),&v) == 0);
} }
void test_string2l(void) { static void test_string2l(void) {
char buf[32]; char buf[32];
long v; long v;
...@@ -636,9 +636,55 @@ void test_string2l(void) { ...@@ -636,9 +636,55 @@ void test_string2l(void) {
#endif #endif
} }
int main(int argc, char **argv) { static void test_ll2string(void) {
char buf[32];
long long v;
int sz;
v = 0;
sz = ll2string(buf, sizeof buf, v);
assert(sz == 1);
assert(!strcmp(buf, "0"));
v = -1;
sz = ll2string(buf, sizeof buf, v);
assert(sz == 2);
assert(!strcmp(buf, "-1"));
v = 99;
sz = ll2string(buf, sizeof buf, v);
assert(sz == 2);
assert(!strcmp(buf, "99"));
v = -99;
sz = ll2string(buf, sizeof buf, v);
assert(sz == 3);
assert(!strcmp(buf, "-99"));
v = -2147483648;
sz = ll2string(buf, sizeof buf, v);
assert(sz == 11);
assert(!strcmp(buf, "-2147483648"));
v = LLONG_MIN;
sz = ll2string(buf, sizeof buf, v);
assert(sz == 20);
assert(!strcmp(buf, "-9223372036854775808"));
v = LLONG_MAX;
sz = ll2string(buf, sizeof buf, v);
assert(sz == 19);
assert(!strcmp(buf, "9223372036854775807"));
}
#define UNUSED(x) (void)(x)
int utilTest(int argc, char **argv) {
UNUSED(argc);
UNUSED(argv);
test_string2ll(); test_string2ll();
test_string2l(); test_string2l();
test_ll2string();
return 0; return 0;
} }
#endif #endif
...@@ -42,4 +42,8 @@ int d2string(char *buf, size_t len, double value); ...@@ -42,4 +42,8 @@ int d2string(char *buf, size_t len, double value);
sds getAbsolutePath(char *filename); sds getAbsolutePath(char *filename);
int pathIsBaseName(char *path); int pathIsBaseName(char *path);
#ifdef REDIS_TEST
int utilTest(int argc, char **argv);
#endif
#endif #endif
...@@ -143,6 +143,7 @@ ...@@ -143,6 +143,7 @@
#define ZIPLIST_TAIL_OFFSET(zl) (*((uint32_t*)((zl)+sizeof(uint32_t)))) #define ZIPLIST_TAIL_OFFSET(zl) (*((uint32_t*)((zl)+sizeof(uint32_t))))
#define ZIPLIST_LENGTH(zl) (*((uint16_t*)((zl)+sizeof(uint32_t)*2))) #define ZIPLIST_LENGTH(zl) (*((uint16_t*)((zl)+sizeof(uint32_t)*2)))
#define ZIPLIST_HEADER_SIZE (sizeof(uint32_t)*2+sizeof(uint16_t)) #define ZIPLIST_HEADER_SIZE (sizeof(uint32_t)*2+sizeof(uint16_t))
#define ZIPLIST_END_SIZE (sizeof(uint8_t))
#define ZIPLIST_ENTRY_HEAD(zl) ((zl)+ZIPLIST_HEADER_SIZE) #define ZIPLIST_ENTRY_HEAD(zl) ((zl)+ZIPLIST_HEADER_SIZE)
#define ZIPLIST_ENTRY_TAIL(zl) ((zl)+intrev32ifbe(ZIPLIST_TAIL_OFFSET(zl))) #define ZIPLIST_ENTRY_TAIL(zl) ((zl)+intrev32ifbe(ZIPLIST_TAIL_OFFSET(zl)))
#define ZIPLIST_ENTRY_END(zl) ((zl)+intrev32ifbe(ZIPLIST_BYTES(zl))-1) #define ZIPLIST_ENTRY_END(zl) ((zl)+intrev32ifbe(ZIPLIST_BYTES(zl))-1)
...@@ -162,6 +163,13 @@ typedef struct zlentry { ...@@ -162,6 +163,13 @@ typedef struct zlentry {
unsigned char *p; unsigned char *p;
} zlentry; } zlentry;
#define ZIPLIST_ENTRY_ZERO(zle) { \
(zle)->prevrawlensize = (zle)->prevrawlen = 0; \
(zle)->lensize = (zle)->len = (zle)->headersize = 0; \
(zle)->encoding = 0; \
(zle)->p = NULL; \
}
/* Extract the encoding from the byte pointed by 'ptr' and set it into /* Extract the encoding from the byte pointed by 'ptr' and set it into
* 'encoding'. */ * 'encoding'. */
#define ZIP_ENTRY_ENCODING(ptr, encoding) do { \ #define ZIP_ENTRY_ENCODING(ptr, encoding) do { \
...@@ -169,6 +177,8 @@ typedef struct zlentry { ...@@ -169,6 +177,8 @@ typedef struct zlentry {
if ((encoding) < ZIP_STR_MASK) (encoding) &= ZIP_STR_MASK; \ if ((encoding) < ZIP_STR_MASK) (encoding) &= ZIP_STR_MASK; \
} while(0) } while(0)
void ziplistRepr(unsigned char *zl);
/* Return bytes needed to store integer encoded by 'encoding' */ /* Return bytes needed to store integer encoded by 'encoding' */
static unsigned int zipIntSize(unsigned char encoding) { static unsigned int zipIntSize(unsigned char encoding) {
switch(encoding) { switch(encoding) {
...@@ -404,14 +414,12 @@ static int64_t zipLoadInteger(unsigned char *p, unsigned char encoding) { ...@@ -404,14 +414,12 @@ static int64_t zipLoadInteger(unsigned char *p, unsigned char encoding) {
} }
/* Return a struct with all information about an entry. */ /* Return a struct with all information about an entry. */
static zlentry zipEntry(unsigned char *p) { static void zipEntry(unsigned char *p, zlentry *e) {
zlentry e;
ZIP_DECODE_PREVLEN(p, e->prevrawlensize, e->prevrawlen);
ZIP_DECODE_PREVLEN(p, e.prevrawlensize, e.prevrawlen); ZIP_DECODE_LENGTH(p + e->prevrawlensize, e->encoding, e->lensize, e->len);
ZIP_DECODE_LENGTH(p + e.prevrawlensize, e.encoding, e.lensize, e.len); e->headersize = e->prevrawlensize + e->lensize;
e.headersize = e.prevrawlensize + e.lensize; e->p = p;
e.p = p;
return e;
} }
/* Create a new empty ziplist. */ /* Create a new empty ziplist. */
...@@ -460,13 +468,13 @@ static unsigned char *__ziplistCascadeUpdate(unsigned char *zl, unsigned char *p ...@@ -460,13 +468,13 @@ static unsigned char *__ziplistCascadeUpdate(unsigned char *zl, unsigned char *p
zlentry cur, next; zlentry cur, next;
while (p[0] != ZIP_END) { while (p[0] != ZIP_END) {
cur = zipEntry(p); zipEntry(p, &cur);
rawlen = cur.headersize + cur.len; rawlen = cur.headersize + cur.len;
rawlensize = zipPrevEncodeLength(NULL,rawlen); rawlensize = zipPrevEncodeLength(NULL,rawlen);
/* Abort if there is no next entry. */ /* Abort if there is no next entry. */
if (p[rawlen] == ZIP_END) break; if (p[rawlen] == ZIP_END) break;
next = zipEntry(p+rawlen); zipEntry(p+rawlen, &next);
/* Abort when "prevlen" has not changed. */ /* Abort when "prevlen" has not changed. */
if (next.prevrawlen == rawlen) break; if (next.prevrawlen == rawlen) break;
...@@ -521,7 +529,7 @@ static unsigned char *__ziplistDelete(unsigned char *zl, unsigned char *p, unsig ...@@ -521,7 +529,7 @@ static unsigned char *__ziplistDelete(unsigned char *zl, unsigned char *p, unsig
int nextdiff = 0; int nextdiff = 0;
zlentry first, tail; zlentry first, tail;
first = zipEntry(p); zipEntry(p, &first);
for (i = 0; p[0] != ZIP_END && i < num; i++) { for (i = 0; p[0] != ZIP_END && i < num; i++) {
p += zipRawEntryLength(p); p += zipRawEntryLength(p);
deleted++; deleted++;
...@@ -545,7 +553,7 @@ static unsigned char *__ziplistDelete(unsigned char *zl, unsigned char *p, unsig ...@@ -545,7 +553,7 @@ static unsigned char *__ziplistDelete(unsigned char *zl, unsigned char *p, unsig
/* When the tail contains more than one entry, we need to take /* When the tail contains more than one entry, we need to take
* "nextdiff" in account as well. Otherwise, a change in the * "nextdiff" in account as well. Otherwise, a change in the
* size of prevlen doesn't have an effect on the *tail* offset. */ * size of prevlen doesn't have an effect on the *tail* offset. */
tail = zipEntry(p); zipEntry(p, &tail);
if (p[tail.headersize+tail.len] != ZIP_END) { if (p[tail.headersize+tail.len] != ZIP_END) {
ZIPLIST_TAIL_OFFSET(zl) = ZIPLIST_TAIL_OFFSET(zl) =
intrev32ifbe(intrev32ifbe(ZIPLIST_TAIL_OFFSET(zl))+nextdiff); intrev32ifbe(intrev32ifbe(ZIPLIST_TAIL_OFFSET(zl))+nextdiff);
...@@ -635,7 +643,7 @@ static unsigned char *__ziplistInsert(unsigned char *zl, unsigned char *p, unsig ...@@ -635,7 +643,7 @@ static unsigned char *__ziplistInsert(unsigned char *zl, unsigned char *p, unsig
/* When the tail contains more than one entry, we need to take /* When the tail contains more than one entry, we need to take
* "nextdiff" in account as well. Otherwise, a change in the * "nextdiff" in account as well. Otherwise, a change in the
* size of prevlen doesn't have an effect on the *tail* offset. */ * size of prevlen doesn't have an effect on the *tail* offset. */
tail = zipEntry(p+reqlen); zipEntry(p+reqlen, &tail);
if (p[reqlen+tail.headersize+tail.len] != ZIP_END) { if (p[reqlen+tail.headersize+tail.len] != ZIP_END) {
ZIPLIST_TAIL_OFFSET(zl) = ZIPLIST_TAIL_OFFSET(zl) =
intrev32ifbe(intrev32ifbe(ZIPLIST_TAIL_OFFSET(zl))+nextdiff); intrev32ifbe(intrev32ifbe(ZIPLIST_TAIL_OFFSET(zl))+nextdiff);
...@@ -665,6 +673,121 @@ static unsigned char *__ziplistInsert(unsigned char *zl, unsigned char *p, unsig ...@@ -665,6 +673,121 @@ static unsigned char *__ziplistInsert(unsigned char *zl, unsigned char *p, unsig
return zl; return zl;
} }
/* Merge ziplists 'first' and 'second' by appending 'second' to 'first'.
*
* NOTE: The larger ziplist is reallocated to contain the new merged ziplist.
* Either 'first' or 'second' can be used for the result. The parameter not
* used will be free'd and set to NULL.
*
* After calling this function, the input parameters are no longer valid since
* they are changed and free'd in-place.
*
* The result ziplist is the contents of 'first' followed by 'second'.
*
* On failure: returns NULL if the merge is impossible.
* On success: returns the merged ziplist (which is expanded version of either
* 'first' or 'second', also frees the other unused input ziplist, and sets the
* input ziplist argument equal to newly reallocated ziplist return value. */
unsigned char *ziplistMerge(unsigned char **first, unsigned char **second) {
/* If any params are null, we can't merge, so NULL. */
if (first == NULL || *first == NULL || second == NULL || *second == NULL)
return NULL;
/* Can't merge same list into itself. */
if (*first == *second)
return NULL;
size_t first_bytes = intrev32ifbe(ZIPLIST_BYTES(*first));
size_t first_len = intrev16ifbe(ZIPLIST_LENGTH(*first));
size_t second_bytes = intrev32ifbe(ZIPLIST_BYTES(*second));
size_t second_len = intrev16ifbe(ZIPLIST_LENGTH(*second));
int append;
unsigned char *source, *target;
size_t target_bytes, source_bytes;
/* Pick the largest ziplist so we can resize easily in-place.
* We must also track if we are now appending or prepending to
* the target ziplist. */
if (first_len >= second_len) {
/* retain first, append second to first. */
target = *first;
target_bytes = first_bytes;
source = *second;
source_bytes = second_bytes;
append = 1;
} else {
/* else, retain second, prepend first to second. */
target = *second;
target_bytes = second_bytes;
source = *first;
source_bytes = first_bytes;
append = 0;
}
/* Calculate final bytes (subtract one pair of metadata) */
size_t zlbytes = first_bytes + second_bytes -
ZIPLIST_HEADER_SIZE - ZIPLIST_END_SIZE;
size_t zllength = first_len + second_len;
/* Combined zl length should be limited within UINT16_MAX */
zllength = zllength < UINT16_MAX ? zllength : UINT16_MAX;
/* Save offset positions before we start ripping memory apart. */
size_t first_offset = intrev32ifbe(ZIPLIST_TAIL_OFFSET(*first));
size_t second_offset = intrev32ifbe(ZIPLIST_TAIL_OFFSET(*second));
/* Extend target to new zlbytes then append or prepend source. */
target = zrealloc(target, zlbytes);
if (append) {
/* append == appending to target */
/* Copy source after target (copying over original [END]):
* [TARGET - END, SOURCE - HEADER] */
memcpy(target + target_bytes - ZIPLIST_END_SIZE,
source + ZIPLIST_HEADER_SIZE,
source_bytes - ZIPLIST_HEADER_SIZE);
} else {
/* !append == prepending to target */
/* Move target *contents* exactly size of (source - [END]),
* then copy source into vacataed space (source - [END]):
* [SOURCE - END, TARGET - HEADER] */
memmove(target + source_bytes - ZIPLIST_END_SIZE,
target + ZIPLIST_HEADER_SIZE,
target_bytes - ZIPLIST_HEADER_SIZE);
memcpy(target, source, source_bytes - ZIPLIST_END_SIZE);
}
/* Update header metadata. */
ZIPLIST_BYTES(target) = intrev32ifbe(zlbytes);
ZIPLIST_LENGTH(target) = intrev16ifbe(zllength);
/* New tail offset is:
* + N bytes of first ziplist
* - 1 byte for [END] of first ziplist
* + M bytes for the offset of the original tail of the second ziplist
* - J bytes for HEADER because second_offset keeps no header. */
ZIPLIST_TAIL_OFFSET(target) = intrev32ifbe(
(first_bytes - ZIPLIST_END_SIZE) +
(second_offset - ZIPLIST_HEADER_SIZE));
/* __ziplistCascadeUpdate just fixes the prev length values until it finds a
* correct prev length value (then it assumes the rest of the list is okay).
* We tell CascadeUpdate to start at the first ziplist's tail element to fix
* the merge seam. */
target = __ziplistCascadeUpdate(target, target+first_offset);
/* Now free and NULL out what we didn't realloc */
if (append) {
zfree(*second);
*second = NULL;
*first = target;
} else {
zfree(*first);
*first = NULL;
*second = target;
}
return target;
}
unsigned char *ziplistPush(unsigned char *zl, unsigned char *s, unsigned int slen, int where) { unsigned char *ziplistPush(unsigned char *zl, unsigned char *s, unsigned int slen, int where) {
unsigned char *p; unsigned char *p;
p = (where == ZIPLIST_HEAD) ? ZIPLIST_ENTRY_HEAD(zl) : ZIPLIST_ENTRY_END(zl); p = (where == ZIPLIST_HEAD) ? ZIPLIST_ENTRY_HEAD(zl) : ZIPLIST_ENTRY_END(zl);
...@@ -748,7 +871,7 @@ unsigned int ziplistGet(unsigned char *p, unsigned char **sstr, unsigned int *sl ...@@ -748,7 +871,7 @@ unsigned int ziplistGet(unsigned char *p, unsigned char **sstr, unsigned int *sl
if (p == NULL || p[0] == ZIP_END) return 0; if (p == NULL || p[0] == ZIP_END) return 0;
if (sstr) *sstr = NULL; if (sstr) *sstr = NULL;
entry = zipEntry(p); zipEntry(p, &entry);
if (ZIP_IS_STR(entry.encoding)) { if (ZIP_IS_STR(entry.encoding)) {
if (sstr) { if (sstr) {
*slen = entry.len; *slen = entry.len;
...@@ -783,7 +906,7 @@ unsigned char *ziplistDelete(unsigned char *zl, unsigned char **p) { ...@@ -783,7 +906,7 @@ unsigned char *ziplistDelete(unsigned char *zl, unsigned char **p) {
} }
/* Delete a range of entries from the ziplist. */ /* Delete a range of entries from the ziplist. */
unsigned char *ziplistDeleteRange(unsigned char *zl, unsigned int index, unsigned int num) { unsigned char *ziplistDeleteRange(unsigned char *zl, int index, unsigned int num) {
unsigned char *p = ziplistIndex(zl,index); unsigned char *p = ziplistIndex(zl,index);
return (p == NULL) ? zl : __ziplistDelete(zl,p,num); return (p == NULL) ? zl : __ziplistDelete(zl,p,num);
} }
...@@ -796,7 +919,7 @@ unsigned int ziplistCompare(unsigned char *p, unsigned char *sstr, unsigned int ...@@ -796,7 +919,7 @@ unsigned int ziplistCompare(unsigned char *p, unsigned char *sstr, unsigned int
long long zval, sval; long long zval, sval;
if (p[0] == ZIP_END) return 0; if (p[0] == ZIP_END) return 0;
entry = zipEntry(p); zipEntry(p, &entry);
if (ZIP_IS_STR(entry.encoding)) { if (ZIP_IS_STR(entry.encoding)) {
/* Raw compare */ /* Raw compare */
if (entry.len == slen) { if (entry.len == slen) {
...@@ -913,7 +1036,7 @@ void ziplistRepr(unsigned char *zl) { ...@@ -913,7 +1036,7 @@ void ziplistRepr(unsigned char *zl) {
intrev32ifbe(ZIPLIST_TAIL_OFFSET(zl))); intrev32ifbe(ZIPLIST_TAIL_OFFSET(zl)));
p = ZIPLIST_ENTRY_HEAD(zl); p = ZIPLIST_ENTRY_HEAD(zl);
while(*p != ZIP_END) { while(*p != ZIP_END) {
entry = zipEntry(p); zipEntry(p, &entry);
printf( printf(
"{" "{"
"addr 0x%08lx, " "addr 0x%08lx, "
...@@ -952,14 +1075,14 @@ void ziplistRepr(unsigned char *zl) { ...@@ -952,14 +1075,14 @@ void ziplistRepr(unsigned char *zl) {
printf("{end}\n\n"); printf("{end}\n\n");
} }
#ifdef ZIPLIST_TEST_MAIN #ifdef REDIS_TEST
#include <sys/time.h> #include <sys/time.h>
#include "adlist.h" #include "adlist.h"
#include "sds.h" #include "sds.h"
#define debug(f, ...) { if (DEBUG) printf(f, __VA_ARGS__); } #define debug(f, ...) { if (DEBUG) printf(f, __VA_ARGS__); }
unsigned char *createList() { static unsigned char *createList() {
unsigned char *zl = ziplistNew(); unsigned char *zl = ziplistNew();
zl = ziplistPush(zl, (unsigned char*)"foo", 3, ZIPLIST_TAIL); zl = ziplistPush(zl, (unsigned char*)"foo", 3, ZIPLIST_TAIL);
zl = ziplistPush(zl, (unsigned char*)"quux", 4, ZIPLIST_TAIL); zl = ziplistPush(zl, (unsigned char*)"quux", 4, ZIPLIST_TAIL);
...@@ -968,7 +1091,7 @@ unsigned char *createList() { ...@@ -968,7 +1091,7 @@ unsigned char *createList() {
return zl; return zl;
} }
unsigned char *createIntList() { static unsigned char *createIntList() {
unsigned char *zl = ziplistNew(); unsigned char *zl = ziplistNew();
char buf[32]; char buf[32];
...@@ -987,13 +1110,13 @@ unsigned char *createIntList() { ...@@ -987,13 +1110,13 @@ unsigned char *createIntList() {
return zl; return zl;
} }
long long usec(void) { static long long usec(void) {
struct timeval tv; struct timeval tv;
gettimeofday(&tv,NULL); gettimeofday(&tv,NULL);
return (((long long)tv.tv_sec)*1000000)+tv.tv_usec; return (((long long)tv.tv_sec)*1000000)+tv.tv_usec;
} }
void stress(int pos, int num, int maxsize, int dnum) { static void stress(int pos, int num, int maxsize, int dnum) {
int i,j,k; int i,j,k;
unsigned char *zl; unsigned char *zl;
char posstr[2][5] = { "HEAD", "TAIL" }; char posstr[2][5] = { "HEAD", "TAIL" };
...@@ -1016,7 +1139,7 @@ void stress(int pos, int num, int maxsize, int dnum) { ...@@ -1016,7 +1139,7 @@ void stress(int pos, int num, int maxsize, int dnum) {
} }
} }
void pop(unsigned char *zl, int where) { static unsigned char *pop(unsigned char *zl, int where) {
unsigned char *p, *vstr; unsigned char *p, *vstr;
unsigned int vlen; unsigned int vlen;
long long vlong; long long vlong;
...@@ -1028,20 +1151,22 @@ void pop(unsigned char *zl, int where) { ...@@ -1028,20 +1151,22 @@ void pop(unsigned char *zl, int where) {
else else
printf("Pop tail: "); printf("Pop tail: ");
if (vstr) if (vstr) {
if (vlen && fwrite(vstr,vlen,1,stdout) == 0) perror("fwrite"); if (vlen && fwrite(vstr,vlen,1,stdout) == 0) perror("fwrite");
else }
else {
printf("%lld", vlong); printf("%lld", vlong);
}
printf("\n"); printf("\n");
ziplistDeleteRange(zl,-1,1); return ziplistDelete(zl,&p);
} else { } else {
printf("ERROR: Could not pop\n"); printf("ERROR: Could not pop\n");
exit(1); exit(1);
} }
} }
int randstring(char *target, unsigned int min, unsigned int max) { static int randstring(char *target, unsigned int min, unsigned int max) {
int p = 0; int p = 0;
int len = min+rand()%(max-min+1); int len = min+rand()%(max-min+1);
int minval, maxval; int minval, maxval;
...@@ -1067,23 +1192,24 @@ int randstring(char *target, unsigned int min, unsigned int max) { ...@@ -1067,23 +1192,24 @@ int randstring(char *target, unsigned int min, unsigned int max) {
return len; return len;
} }
void verify(unsigned char *zl, zlentry *e) { static void verify(unsigned char *zl, zlentry *e) {
int i;
int len = ziplistLen(zl); int len = ziplistLen(zl);
zlentry _e; zlentry _e;
for (i = 0; i < len; i++) { ZIPLIST_ENTRY_ZERO(&_e);
for (int i = 0; i < len; i++) {
memset(&e[i], 0, sizeof(zlentry)); memset(&e[i], 0, sizeof(zlentry));
e[i] = zipEntry(ziplistIndex(zl, i)); zipEntry(ziplistIndex(zl, i), &e[i]);
memset(&_e, 0, sizeof(zlentry)); memset(&_e, 0, sizeof(zlentry));
_e = zipEntry(ziplistIndex(zl, -len+i)); zipEntry(ziplistIndex(zl, -len+i), &_e);
assert(memcmp(&e[i], &_e, sizeof(zlentry)) == 0); assert(memcmp(&e[i], &_e, sizeof(zlentry)) == 0);
} }
} }
int main(int argc, char **argv) { int ziplistTest(int argc, char **argv) {
unsigned char *zl, *p; unsigned char *zl, *p;
unsigned char *entry; unsigned char *entry;
unsigned int elen; unsigned int elen;
...@@ -1096,21 +1222,25 @@ int main(int argc, char **argv) { ...@@ -1096,21 +1222,25 @@ int main(int argc, char **argv) {
zl = createIntList(); zl = createIntList();
ziplistRepr(zl); ziplistRepr(zl);
zfree(zl);
zl = createList(); zl = createList();
ziplistRepr(zl); ziplistRepr(zl);
pop(zl,ZIPLIST_TAIL); zl = pop(zl,ZIPLIST_TAIL);
ziplistRepr(zl); ziplistRepr(zl);
pop(zl,ZIPLIST_HEAD); zl = pop(zl,ZIPLIST_HEAD);
ziplistRepr(zl); ziplistRepr(zl);
pop(zl,ZIPLIST_TAIL); zl = pop(zl,ZIPLIST_TAIL);
ziplistRepr(zl); ziplistRepr(zl);
pop(zl,ZIPLIST_TAIL); zl = pop(zl,ZIPLIST_TAIL);
ziplistRepr(zl); ziplistRepr(zl);
zfree(zl);
printf("Get element at index 3:\n"); printf("Get element at index 3:\n");
{ {
zl = createList(); zl = createList();
...@@ -1126,6 +1256,7 @@ int main(int argc, char **argv) { ...@@ -1126,6 +1256,7 @@ int main(int argc, char **argv) {
printf("%lld\n", value); printf("%lld\n", value);
} }
printf("\n"); printf("\n");
zfree(zl);
} }
printf("Get element at index 4 (out of range):\n"); printf("Get element at index 4 (out of range):\n");
...@@ -1139,6 +1270,7 @@ int main(int argc, char **argv) { ...@@ -1139,6 +1270,7 @@ int main(int argc, char **argv) {
return 1; return 1;
} }
printf("\n"); printf("\n");
zfree(zl);
} }
printf("Get element at index -1 (last element):\n"); printf("Get element at index -1 (last element):\n");
...@@ -1156,6 +1288,7 @@ int main(int argc, char **argv) { ...@@ -1156,6 +1288,7 @@ int main(int argc, char **argv) {
printf("%lld\n", value); printf("%lld\n", value);
} }
printf("\n"); printf("\n");
zfree(zl);
} }
printf("Get element at index -4 (first element):\n"); printf("Get element at index -4 (first element):\n");
...@@ -1173,6 +1306,7 @@ int main(int argc, char **argv) { ...@@ -1173,6 +1306,7 @@ int main(int argc, char **argv) {
printf("%lld\n", value); printf("%lld\n", value);
} }
printf("\n"); printf("\n");
zfree(zl);
} }
printf("Get element at index -5 (reverse out of range):\n"); printf("Get element at index -5 (reverse out of range):\n");
...@@ -1186,6 +1320,7 @@ int main(int argc, char **argv) { ...@@ -1186,6 +1320,7 @@ int main(int argc, char **argv) {
return 1; return 1;
} }
printf("\n"); printf("\n");
zfree(zl);
} }
printf("Iterate list from 0 to end:\n"); printf("Iterate list from 0 to end:\n");
...@@ -1203,6 +1338,7 @@ int main(int argc, char **argv) { ...@@ -1203,6 +1338,7 @@ int main(int argc, char **argv) {
printf("\n"); printf("\n");
} }
printf("\n"); printf("\n");
zfree(zl);
} }
printf("Iterate list from 1 to end:\n"); printf("Iterate list from 1 to end:\n");
...@@ -1220,6 +1356,7 @@ int main(int argc, char **argv) { ...@@ -1220,6 +1356,7 @@ int main(int argc, char **argv) {
printf("\n"); printf("\n");
} }
printf("\n"); printf("\n");
zfree(zl);
} }
printf("Iterate list from 2 to end:\n"); printf("Iterate list from 2 to end:\n");
...@@ -1237,6 +1374,7 @@ int main(int argc, char **argv) { ...@@ -1237,6 +1374,7 @@ int main(int argc, char **argv) {
printf("\n"); printf("\n");
} }
printf("\n"); printf("\n");
zfree(zl);
} }
printf("Iterate starting out of range:\n"); printf("Iterate starting out of range:\n");
...@@ -1249,6 +1387,7 @@ int main(int argc, char **argv) { ...@@ -1249,6 +1387,7 @@ int main(int argc, char **argv) {
printf("ERROR\n"); printf("ERROR\n");
} }
printf("\n"); printf("\n");
zfree(zl);
} }
printf("Iterate from back to front:\n"); printf("Iterate from back to front:\n");
...@@ -1266,6 +1405,7 @@ int main(int argc, char **argv) { ...@@ -1266,6 +1405,7 @@ int main(int argc, char **argv) {
printf("\n"); printf("\n");
} }
printf("\n"); printf("\n");
zfree(zl);
} }
printf("Iterate from back to front, deleting all items:\n"); printf("Iterate from back to front, deleting all items:\n");
...@@ -1284,6 +1424,7 @@ int main(int argc, char **argv) { ...@@ -1284,6 +1424,7 @@ int main(int argc, char **argv) {
printf("\n"); printf("\n");
} }
printf("\n"); printf("\n");
zfree(zl);
} }
printf("Delete inclusive range 0,0:\n"); printf("Delete inclusive range 0,0:\n");
...@@ -1291,6 +1432,7 @@ int main(int argc, char **argv) { ...@@ -1291,6 +1432,7 @@ int main(int argc, char **argv) {
zl = createList(); zl = createList();
zl = ziplistDeleteRange(zl, 0, 1); zl = ziplistDeleteRange(zl, 0, 1);
ziplistRepr(zl); ziplistRepr(zl);
zfree(zl);
} }
printf("Delete inclusive range 0,1:\n"); printf("Delete inclusive range 0,1:\n");
...@@ -1298,6 +1440,7 @@ int main(int argc, char **argv) { ...@@ -1298,6 +1440,7 @@ int main(int argc, char **argv) {
zl = createList(); zl = createList();
zl = ziplistDeleteRange(zl, 0, 2); zl = ziplistDeleteRange(zl, 0, 2);
ziplistRepr(zl); ziplistRepr(zl);
zfree(zl);
} }
printf("Delete inclusive range 1,2:\n"); printf("Delete inclusive range 1,2:\n");
...@@ -1305,6 +1448,7 @@ int main(int argc, char **argv) { ...@@ -1305,6 +1448,7 @@ int main(int argc, char **argv) {
zl = createList(); zl = createList();
zl = ziplistDeleteRange(zl, 1, 2); zl = ziplistDeleteRange(zl, 1, 2);
ziplistRepr(zl); ziplistRepr(zl);
zfree(zl);
} }
printf("Delete with start index out of range:\n"); printf("Delete with start index out of range:\n");
...@@ -1312,6 +1456,7 @@ int main(int argc, char **argv) { ...@@ -1312,6 +1456,7 @@ int main(int argc, char **argv) {
zl = createList(); zl = createList();
zl = ziplistDeleteRange(zl, 5, 1); zl = ziplistDeleteRange(zl, 5, 1);
ziplistRepr(zl); ziplistRepr(zl);
zfree(zl);
} }
printf("Delete with num overflow:\n"); printf("Delete with num overflow:\n");
...@@ -1319,6 +1464,7 @@ int main(int argc, char **argv) { ...@@ -1319,6 +1464,7 @@ int main(int argc, char **argv) {
zl = createList(); zl = createList();
zl = ziplistDeleteRange(zl, 1, 5); zl = ziplistDeleteRange(zl, 1, 5);
ziplistRepr(zl); ziplistRepr(zl);
zfree(zl);
} }
printf("Delete foo while iterating:\n"); printf("Delete foo while iterating:\n");
...@@ -1343,11 +1489,12 @@ int main(int argc, char **argv) { ...@@ -1343,11 +1489,12 @@ int main(int argc, char **argv) {
} }
printf("\n"); printf("\n");
ziplistRepr(zl); ziplistRepr(zl);
zfree(zl);
} }
printf("Regression test for >255 byte strings:\n"); printf("Regression test for >255 byte strings:\n");
{ {
char v1[257],v2[257]; char v1[257] = {0}, v2[257] = {0};
memset(v1,'x',256); memset(v1,'x',256);
memset(v2,'y',256); memset(v2,'y',256);
zl = ziplistNew(); zl = ziplistNew();
...@@ -1362,13 +1509,15 @@ int main(int argc, char **argv) { ...@@ -1362,13 +1509,15 @@ int main(int argc, char **argv) {
assert(ziplistGet(p,&entry,&elen,&value)); assert(ziplistGet(p,&entry,&elen,&value));
assert(strncmp(v2,(char*)entry,elen) == 0); assert(strncmp(v2,(char*)entry,elen) == 0);
printf("SUCCESS\n\n"); printf("SUCCESS\n\n");
zfree(zl);
} }
printf("Regression test deleting next to last entries:\n"); printf("Regression test deleting next to last entries:\n");
{ {
char v[3][257]; char v[3][257] = {{0}};
zlentry e[3]; zlentry e[3] = {{.prevrawlensize = 0, .prevrawlen = 0, .lensize = 0,
int i; .len = 0, .headersize = 0, .encoding = 0, .p = NULL}};
size_t i;
for (i = 0; i < (sizeof(v)/sizeof(v[0])); i++) { for (i = 0; i < (sizeof(v)/sizeof(v[0])); i++) {
memset(v[i], 'a' + i, sizeof(v[0])); memset(v[i], 'a' + i, sizeof(v[0]));
...@@ -1399,6 +1548,7 @@ int main(int argc, char **argv) { ...@@ -1399,6 +1548,7 @@ int main(int argc, char **argv) {
assert(e[1].prevrawlensize == 5); assert(e[1].prevrawlensize == 5);
printf("SUCCESS\n\n"); printf("SUCCESS\n\n");
zfree(zl);
} }
printf("Create long list and check indices:\n"); printf("Create long list and check indices:\n");
...@@ -1420,6 +1570,7 @@ int main(int argc, char **argv) { ...@@ -1420,6 +1570,7 @@ int main(int argc, char **argv) {
assert(999-i == value); assert(999-i == value);
} }
printf("SUCCESS\n\n"); printf("SUCCESS\n\n");
zfree(zl);
} }
printf("Compare strings with ziplist entries:\n"); printf("Compare strings with ziplist entries:\n");
...@@ -1445,6 +1596,82 @@ int main(int argc, char **argv) { ...@@ -1445,6 +1596,82 @@ int main(int argc, char **argv) {
return 1; return 1;
} }
printf("SUCCESS\n\n"); printf("SUCCESS\n\n");
zfree(zl);
}
printf("Merge test:\n");
{
/* create list gives us: [hello, foo, quux, 1024] */
zl = createList();
unsigned char *zl2 = createList();
unsigned char *zl3 = ziplistNew();
unsigned char *zl4 = ziplistNew();
if (ziplistMerge(&zl4, &zl4)) {
printf("ERROR: Allowed merging of one ziplist into itself.\n");
return 1;
}
/* Merge two empty ziplists, get empty result back. */
zl4 = ziplistMerge(&zl3, &zl4);
ziplistRepr(zl4);
if (ziplistLen(zl4)) {
printf("ERROR: Merging two empty ziplists created entries.\n");
return 1;
}
zfree(zl4);
zl2 = ziplistMerge(&zl, &zl2);
/* merge gives us: [hello, foo, quux, 1024, hello, foo, quux, 1024] */
ziplistRepr(zl2);
if (ziplistLen(zl2) != 8) {
printf("ERROR: Merged length not 8, but: %u\n", ziplistLen(zl2));
return 1;
}
p = ziplistIndex(zl2,0);
if (!ziplistCompare(p,(unsigned char*)"hello",5)) {
printf("ERROR: not \"hello\"\n");
return 1;
}
if (ziplistCompare(p,(unsigned char*)"hella",5)) {
printf("ERROR: \"hella\"\n");
return 1;
}
p = ziplistIndex(zl2,3);
if (!ziplistCompare(p,(unsigned char*)"1024",4)) {
printf("ERROR: not \"1024\"\n");
return 1;
}
if (ziplistCompare(p,(unsigned char*)"1025",4)) {
printf("ERROR: \"1025\"\n");
return 1;
}
p = ziplistIndex(zl2,4);
if (!ziplistCompare(p,(unsigned char*)"hello",5)) {
printf("ERROR: not \"hello\"\n");
return 1;
}
if (ziplistCompare(p,(unsigned char*)"hella",5)) {
printf("ERROR: \"hella\"\n");
return 1;
}
p = ziplistIndex(zl2,7);
if (!ziplistCompare(p,(unsigned char*)"1024",4)) {
printf("ERROR: not \"1024\"\n");
return 1;
}
if (ziplistCompare(p,(unsigned char*)"1025",4)) {
printf("ERROR: \"1025\"\n");
return 1;
}
printf("SUCCESS\n\n");
zfree(zl);
} }
printf("Stress with random payloads of different encoding:\n"); printf("Stress with random payloads of different encoding:\n");
...@@ -1464,7 +1691,7 @@ int main(int argc, char **argv) { ...@@ -1464,7 +1691,7 @@ int main(int argc, char **argv) {
for (i = 0; i < 20000; i++) { for (i = 0; i < 20000; i++) {
zl = ziplistNew(); zl = ziplistNew();
ref = listCreate(); ref = listCreate();
listSetFreeMethod(ref,sdsfree); listSetFreeMethod(ref,(void (*)(void*))sdsfree);
len = rand() % 256; len = rand() % 256;
/* Create lists */ /* Create lists */
...@@ -1532,5 +1759,4 @@ int main(int argc, char **argv) { ...@@ -1532,5 +1759,4 @@ int main(int argc, char **argv) {
return 0; return 0;
} }
#endif #endif
...@@ -32,6 +32,7 @@ ...@@ -32,6 +32,7 @@
#define ZIPLIST_TAIL 1 #define ZIPLIST_TAIL 1
unsigned char *ziplistNew(void); unsigned char *ziplistNew(void);
unsigned char *ziplistMerge(unsigned char **first, unsigned char **second);
unsigned char *ziplistPush(unsigned char *zl, unsigned char *s, unsigned int slen, int where); unsigned char *ziplistPush(unsigned char *zl, unsigned char *s, unsigned int slen, int where);
unsigned char *ziplistIndex(unsigned char *zl, int index); unsigned char *ziplistIndex(unsigned char *zl, int index);
unsigned char *ziplistNext(unsigned char *zl, unsigned char *p); unsigned char *ziplistNext(unsigned char *zl, unsigned char *p);
...@@ -39,8 +40,12 @@ unsigned char *ziplistPrev(unsigned char *zl, unsigned char *p); ...@@ -39,8 +40,12 @@ unsigned char *ziplistPrev(unsigned char *zl, unsigned char *p);
unsigned int ziplistGet(unsigned char *p, unsigned char **sval, unsigned int *slen, long long *lval); unsigned int ziplistGet(unsigned char *p, unsigned char **sval, unsigned int *slen, long long *lval);
unsigned char *ziplistInsert(unsigned char *zl, unsigned char *p, unsigned char *s, unsigned int slen); unsigned char *ziplistInsert(unsigned char *zl, unsigned char *p, unsigned char *s, unsigned int slen);
unsigned char *ziplistDelete(unsigned char *zl, unsigned char **p); unsigned char *ziplistDelete(unsigned char *zl, unsigned char **p);
unsigned char *ziplistDeleteRange(unsigned char *zl, unsigned int index, unsigned int num); unsigned char *ziplistDeleteRange(unsigned char *zl, int index, unsigned int num);
unsigned int ziplistCompare(unsigned char *p, unsigned char *s, unsigned int slen); unsigned int ziplistCompare(unsigned char *p, unsigned char *s, unsigned int slen);
unsigned char *ziplistFind(unsigned char *p, unsigned char *vstr, unsigned int vlen, unsigned int skip); unsigned char *ziplistFind(unsigned char *p, unsigned char *vstr, unsigned int vlen, unsigned int skip);
unsigned int ziplistLen(unsigned char *zl); unsigned int ziplistLen(unsigned char *zl);
size_t ziplistBlobLen(unsigned char *zl); size_t ziplistBlobLen(unsigned char *zl);
#ifdef REDIS_TEST
int ziplistTest(int argc, char *argv[]);
#endif
...@@ -370,8 +370,8 @@ size_t zipmapBlobLen(unsigned char *zm) { ...@@ -370,8 +370,8 @@ size_t zipmapBlobLen(unsigned char *zm) {
return totlen; return totlen;
} }
#ifdef ZIPMAP_TEST_MAIN #ifdef REDIS_TEST
void zipmapRepr(unsigned char *p) { static void zipmapRepr(unsigned char *p) {
unsigned int l; unsigned int l;
printf("{status %u}",*p++); printf("{status %u}",*p++);
...@@ -404,9 +404,13 @@ void zipmapRepr(unsigned char *p) { ...@@ -404,9 +404,13 @@ void zipmapRepr(unsigned char *p) {
printf("\n"); printf("\n");
} }
int main(void) { #define UNUSED(x) (void)(x)
int zipmapTest(int argc, char *argv[]) {
unsigned char *zm; unsigned char *zm;
UNUSED(argc);
UNUSED(argv);
zm = zipmapNew(); zm = zipmapNew();
zm = zipmapSet(zm,(unsigned char*) "name",4, (unsigned char*) "foo",3,NULL); zm = zipmapSet(zm,(unsigned char*) "name",4, (unsigned char*) "foo",3,NULL);
......
...@@ -46,4 +46,8 @@ unsigned int zipmapLen(unsigned char *zm); ...@@ -46,4 +46,8 @@ unsigned int zipmapLen(unsigned char *zm);
size_t zipmapBlobLen(unsigned char *zm); size_t zipmapBlobLen(unsigned char *zm);
void zipmapRepr(unsigned char *p); void zipmapRepr(unsigned char *p);
#ifdef REDIS_TEST
int zipmapTest(int argc, char *argv[]);
#endif
#endif #endif
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment