Unverified Commit 45129059 authored by sundb's avatar sundb Committed by GitHub
Browse files

Replace ziplist with listpack in quicklist (#9740)



Part three of implementing #8702, following #8887 and #9366 .

## Description of the feature
1. Replace the ziplist container of quicklist with listpack.
2. Convert existing quicklist ziplists on RDB loading time. an O(n) operation.

## Interface changes
1. New `list-max-listpack-size` config is an alias for `list-max-ziplist-size`.
2. Replace `debug ziplist` command with `debug listpack`.

## Internal changes
1. Add `lpMerge` to merge two listpacks . (same as `ziplistMerge`)
2. Add `lpRepr` to print info of listpack which is used in debugCommand and `quicklistRepr`. (same as `ziplistRepr`)
3. Replace `QUICKLIST_NODE_CONTAINER_ZIPLIST` with `QUICKLIST_NODE_CONTAINER_PACKED`(following #9357 ).
    It represent that a quicklistNode is a packed node, as opposed to a plain node.
4. Remove `createZiplistObject` method, which is never used.
5. Calculate listpack entry size using overhead overestimation in `quicklistAllowInsert`.
    We prefer an overestimation, which would at worse lead to a few bytes below the lowest limit of 4k.

## Improvements
1. Calling `lpShrinkToFit` after converting Ziplist to listpack, which was missed at #9366.
2. Optimize `quicklistAppendPlainNode` to avoid memcpy data.

## Bugfix
1. Fix crash in `quicklistRepr` when ziplist is compressed, introduced from #9366.

## Test
1. Add unittest for `lpMerge`.
2. Modify the old quicklist ziplist corrupt dump test.
Co-authored-by: default avatarOran Agra <oran@redislabs.com>
parent fb4f7be2
......@@ -1737,7 +1737,7 @@ hash-max-listpack-value 64
# per list node.
# The highest performing option is usually -2 (8 Kb size) or -1 (4 Kb size),
# but if your use case is unique, adjust the settings as necessary.
list-max-ziplist-size -2
list-max-listpack-size -2
# Lists may also be compressed.
# Compress depth is the number of quicklist ziplist nodes from *each* side of
......
......@@ -2603,7 +2603,7 @@ standardConfig configs[] = {
createIntConfig("io-threads", NULL, DEBUG_CONFIG | IMMUTABLE_CONFIG, 1, 128, server.io_threads_num, 1, INTEGER_CONFIG, NULL, NULL), /* Single threaded by default */
createIntConfig("auto-aof-rewrite-percentage", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.aof_rewrite_perc, 100, INTEGER_CONFIG, NULL, NULL),
createIntConfig("cluster-replica-validity-factor", "cluster-slave-validity-factor", MODIFIABLE_CONFIG, 0, INT_MAX, server.cluster_slave_validity_factor, 10, INTEGER_CONFIG, NULL, NULL), /* Slave max data age factor. */
createIntConfig("list-max-ziplist-size", NULL, MODIFIABLE_CONFIG, INT_MIN, INT_MAX, server.list_max_ziplist_size, -2, INTEGER_CONFIG, NULL, NULL),
createIntConfig("list-max-listpack-size", "list-max-ziplist-size", MODIFIABLE_CONFIG, INT_MIN, INT_MAX, server.list_max_listpack_size, -2, INTEGER_CONFIG, NULL, NULL),
createIntConfig("tcp-keepalive", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.tcpkeepalive, 300, INTEGER_CONFIG, NULL, NULL),
createIntConfig("cluster-migration-barrier", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.cluster_migration_barrier, 1, INTEGER_CONFIG, NULL, NULL),
createIntConfig("active-defrag-cycle-min", NULL, MODIFIABLE_CONFIG, 1, 99, server.active_defrag_cycle_min, 1, INTEGER_CONFIG, NULL, NULL), /* Default: 1% CPU min (at lower threshold) */
......
......@@ -847,7 +847,7 @@ void scanGenericCommand(client *c, robj *o, unsigned long cursor) {
/* Step 2: Iterate the collection.
*
* Note that if the object is encoded with a ziplist, intset, or any other
* Note that if the object is encoded with a listpack, intset, or any other
* representation that is not a hash table, we are sure that it is also
* composed of a small number of elements. So to avoid taking state we
* just return everything inside the object in a single call, setting the
......
......@@ -473,8 +473,8 @@ void debugCommand(client *c) {
" Run a fuzz tester against the stringmatchlen() function.",
"STRUCTSIZE",
" Return the size of different Redis core C structures.",
"ZIPLIST <key>",
" Show low level info about the ziplist encoding of <key>.",
"LISTPACK <key>",
" Show low level info about the listpack encoding of <key>.",
"QUICKLIST <key> [<0|1>]",
" Show low level info about the quicklist encoding of <key>."
" The optional argument (0 by default) sets the level of detail",
......@@ -602,8 +602,8 @@ NULL
used = snprintf(nextra, remaining, " ql_avg_node:%.2f", avg);
nextra += used;
remaining -= used;
/* Add quicklist fill level / max ziplist size */
used = snprintf(nextra, remaining, " ql_ziplist_max:%d", ql->fill);
/* Add quicklist fill level / max listpack size */
used = snprintf(nextra, remaining, " ql_listpack_max:%d", ql->fill);
nextra += used;
remaining -= used;
/* Add isCompressed? */
......@@ -653,17 +653,17 @@ NULL
(long long) sdsavail(val->ptr),
(long long) getStringObjectSdsUsedMemory(val));
}
} else if (!strcasecmp(c->argv[1]->ptr,"ziplist") && c->argc == 3) {
} else if (!strcasecmp(c->argv[1]->ptr,"listpack") && c->argc == 3) {
robj *o;
if ((o = objectCommandLookupOrReply(c,c->argv[2],shared.nokeyerr))
== NULL) return;
if (o->encoding != OBJ_ENCODING_ZIPLIST) {
addReplyError(c,"Not a ziplist encoded object.");
if (o->encoding != OBJ_ENCODING_LISTPACK) {
addReplyError(c,"Not a listpack encoded object.");
} else {
ziplistRepr(o->ptr);
addReplyStatus(c,"Ziplist structure printed on stdout");
lpRepr(o->ptr);
addReplyStatus(c,"Listpack structure printed on stdout");
}
} else if (!strcasecmp(c->argv[1]->ptr,"quicklist") && (c->argc == 3 || c->argc == 4)) {
robj *o;
......
......@@ -1021,7 +1021,7 @@ unsigned char *lpDeleteRangeWithEntry(unsigned char *lp, unsigned char **p, unsi
* address again after a reallocation. */
unsigned long poff = first-lp;
/* Move tail to the front of the ziplist */
/* Move tail to the front of the listpack */
memmove(first, tail, eofptr - tail + 1);
lpSetTotalBytes(lp, bytes - (tail - first));
uint32_t numele = lpGetNumElements(lp);
......@@ -1064,6 +1064,103 @@ unsigned char *lpDeleteRange(unsigned char *lp, long index, unsigned long num) {
return lp;
}
/* Merge listpacks 'first' and 'second' by appending 'second' to 'first'.
*
* NOTE: The larger listpack is reallocated to contain the new merged listpack.
* Either 'first' or 'second' can be used for the result. The parameter not
* used will be free'd and set to NULL.
*
* After calling this function, the input parameters are no longer valid since
* they are changed and free'd in-place.
*
* The result listpack is the contents of 'first' followed by 'second'.
*
* On failure: returns NULL if the merge is impossible.
* On success: returns the merged listpack (which is expanded version of either
* 'first' or 'second', also frees the other unused input listpack, and sets the
* input listpack argument equal to newly reallocated listpack return value. */
unsigned char *lpMerge(unsigned char **first, unsigned char **second) {
/* If any params are null, we can't merge, so NULL. */
if (first == NULL || *first == NULL || second == NULL || *second == NULL)
return NULL;
/* Can't merge same list into itself. */
if (*first == *second)
return NULL;
size_t first_bytes = lpBytes(*first);
unsigned long first_len = lpLength(*first);
size_t second_bytes = lpBytes(*second);
unsigned long second_len = lpLength(*second);
int append;
unsigned char *source, *target;
size_t target_bytes, source_bytes;
/* Pick the largest listpack so we can resize easily in-place.
* We must also track if we are now appending or prepending to
* the target listpack. */
if (first_bytes >= second_bytes) {
/* retain first, append second to first. */
target = *first;
target_bytes = first_bytes;
source = *second;
source_bytes = second_bytes;
append = 1;
} else {
/* else, retain second, prepend first to second. */
target = *second;
target_bytes = second_bytes;
source = *first;
source_bytes = first_bytes;
append = 0;
}
/* Calculate final bytes (subtract one pair of metadata) */
unsigned long long lpbytes = (unsigned long long)first_bytes + second_bytes - LP_HDR_SIZE - 1;
assert(lpbytes < UINT32_MAX); /* larger values can't be stored */
unsigned long lplength = first_len + second_len;
/* Combined lp length should be limited within UINT16_MAX */
lplength = lplength < UINT16_MAX ? lplength : UINT16_MAX;
/* Extend target to new lpbytes then append or prepend source. */
target = zrealloc(target, lpbytes);
if (append) {
/* append == appending to target */
/* Copy source after target (copying over original [END]):
* [TARGET - END, SOURCE - HEADER] */
memcpy(target + target_bytes - 1,
source + LP_HDR_SIZE,
source_bytes - LP_HDR_SIZE);
} else {
/* !append == prepending to target */
/* Move target *contents* exactly size of (source - [END]),
* then copy source into vacated space (source - [END]):
* [SOURCE - END, TARGET - HEADER] */
memmove(target + source_bytes - 1,
target + LP_HDR_SIZE,
target_bytes - LP_HDR_SIZE);
memcpy(target, source, source_bytes - 1);
}
lpSetNumElements(target, lplength);
lpSetTotalBytes(target, lpbytes);
/* Now free and NULL out what we didn't realloc */
if (append) {
zfree(*second);
*second = NULL;
*first = target;
} else {
zfree(*first);
*first = NULL;
*second = target;
}
return target;
}
/* Return the total number of bytes the listpack is composed of. */
size_t lpBytes(unsigned char *lp) {
return lpGetTotalBytes(lp);
......@@ -1377,6 +1474,57 @@ unsigned int lpRandomPairsUnique(unsigned char *lp, unsigned int count, listpack
return picked;
}
/* Print info of listpack which is used in debugCommand */
void lpRepr(unsigned char *lp) {
unsigned char *p, *vstr;
int64_t vlen;
unsigned char intbuf[LP_INTBUF_SIZE];
int index = 0;
printf("{total bytes %zu} {num entries %lu}\n", lpBytes(lp), lpLength(lp));
p = lpFirst(lp);
while(p) {
uint32_t encoded_size_bytes = lpCurrentEncodedSizeBytes(p);
uint32_t encoded_size = lpCurrentEncodedSizeUnsafe(p);
unsigned long back_len = lpEncodeBacklen(NULL, encoded_size);
printf(
"{\n"
"\taddr: 0x%08lx,\n"
"\tindex: %2d,\n"
"\toffset: %1lu,\n"
"\thdr+entrylen+backlen: %2lu,\n"
"\thdrlen: %3u,\n"
"\tbacklen: %2lu,\n"
"\tpayload: %1u\n",
(long unsigned)p,
index,
(unsigned long) (p-lp),
encoded_size + back_len,
encoded_size_bytes,
back_len,
encoded_size - encoded_size_bytes);
printf("\tbytes: ");
for (unsigned int i = 0; i < (encoded_size + back_len); i++) {
printf("%02x|",p[i]);
}
printf("\n");
vstr = lpGet(p, &vlen, intbuf);
printf("\t[str]");
if (vlen > 40) {
if (fwrite(vstr, 40, 1, stdout) == 0) perror("fwrite");
printf("...");
} else {
if (fwrite(vstr, vlen, 1, stdout) == 0) perror("fwrite");
}
printf("\n}\n");
index++;
p = lpNext(lp, p);
}
printf("{end}\n\n");
}
#ifdef REDIS_TEST
#include <sys/time.h>
......@@ -1845,6 +1993,58 @@ int listpackTest(int argc, char *argv[], int flags) {
lpFree(lp);
}
TEST("lpMerge two empty listpacks") {
unsigned char *lp1 = lpNew(0);
unsigned char *lp2 = lpNew(0);
/* Merge two empty listpacks, get empty result back. */
lp1 = lpMerge(&lp1, &lp2);
assert(lpLength(lp1) == 0);
zfree(lp1);
}
TEST("lpMerge two listpacks - first larger than second") {
unsigned char *lp1 = createIntList();
unsigned char *lp2 = createList();
size_t lp1_bytes = lpBytes(lp1);
size_t lp2_bytes = lpBytes(lp2);
unsigned long lp1_len = lpLength(lp1);
unsigned long lp2_len = lpLength(lp2);
unsigned char *lp3 = lpMerge(&lp1, &lp2);
assert(lp3 == lp1);
assert(lp2 == NULL);
assert(lpLength(lp3) == (lp1_len + lp2_len));
assert(lpBytes(lp3) == (lp1_bytes + lp2_bytes - LP_HDR_SIZE - 1));
verifyEntry(lpSeek(lp3, 0), (unsigned char*)"4294967296", 10);
verifyEntry(lpSeek(lp3, 5), (unsigned char*)"much much longer non integer", 28);
verifyEntry(lpSeek(lp3, 6), (unsigned char*)"hello", 5);
verifyEntry(lpSeek(lp3, -1), (unsigned char*)"1024", 4);
zfree(lp3);
}
TEST("lpMerge two listpacks - second larger than first") {
unsigned char *lp1 = createList();
unsigned char *lp2 = createIntList();
size_t lp1_bytes = lpBytes(lp1);
size_t lp2_bytes = lpBytes(lp2);
unsigned long lp1_len = lpLength(lp1);
unsigned long lp2_len = lpLength(lp2);
unsigned char *lp3 = lpMerge(&lp1, &lp2);
assert(lp3 == lp2);
assert(lp1 == NULL);
assert(lpLength(lp3) == (lp1_len + lp2_len));
assert(lpBytes(lp3) == (lp1_bytes + lp2_bytes - LP_HDR_SIZE - 1));
verifyEntry(lpSeek(lp3, 0), (unsigned char*)"hello", 5);
verifyEntry(lpSeek(lp3, 3), (unsigned char*)"1024", 4);
verifyEntry(lpSeek(lp3, 4), (unsigned char*)"4294967296", 10);
verifyEntry(lpSeek(lp3, -1), (unsigned char*)"much much longer non integer", 28);
zfree(lp3);
}
TEST("Random pair with one element") {
listpackEntry key, val;
unsigned char *lp = lpNew(0);
......
......@@ -68,6 +68,7 @@ unsigned char *lpReplaceInteger(unsigned char *lp, unsigned char **p, long long
unsigned char *lpDelete(unsigned char *lp, unsigned char *p, unsigned char **newp);
unsigned char *lpDeleteRangeWithEntry(unsigned char *lp, unsigned char **p, unsigned long num);
unsigned char *lpDeleteRange(unsigned char *lp, long index, unsigned long num);
unsigned char *lpMerge(unsigned char **first, unsigned char **second);
unsigned long lpLength(unsigned char *lp);
unsigned char *lpGet(unsigned char *p, int64_t *count, unsigned char *intbuf);
unsigned char *lpGetValue(unsigned char *p, unsigned int *slen, long long *lval);
......@@ -88,6 +89,7 @@ void lpRandomPair(unsigned char *lp, unsigned long total_count, listpackEntry *k
void lpRandomPairs(unsigned char *lp, unsigned int count, listpackEntry *keys, listpackEntry *vals);
unsigned int lpRandomPairsUnique(unsigned char *lp, unsigned int count, listpackEntry *keys, listpackEntry *vals);
int lpSafeToAdd(unsigned char* lp, size_t add);
void lpRepr(unsigned char *lp);
#ifdef REDIS_TEST
int listpackTest(int argc, char *argv[], int flags);
......
......@@ -526,7 +526,7 @@ int moduleCreateEmptyKey(RedisModuleKey *key, int type) {
switch(type) {
case REDISMODULE_KEYTYPE_LIST:
obj = createQuicklistObject();
quicklistSetOptions(obj->ptr, server.list_max_ziplist_size,
quicklistSetOptions(obj->ptr, server.list_max_listpack_size,
server.list_compress_depth);
break;
case REDISMODULE_KEYTYPE_ZSET:
......
......@@ -233,13 +233,6 @@ robj *createQuicklistObject(void) {
return o;
}
robj *createZiplistObject(void) {
unsigned char *zl = ziplistNew();
robj *o = createObject(OBJ_LIST,zl);
o->encoding = OBJ_ENCODING_ZIPLIST;
return o;
}
robj *createSetObject(void) {
dict *d = dictCreate(&setDictType);
robj *o = createObject(OBJ_SET,d);
......
This diff is collapsed.
......@@ -35,11 +35,11 @@
/* Node, quicklist, and Iterator are the only data structures used currently. */
/* quicklistNode is a 32 byte struct describing a ziplist for a quicklist.
/* quicklistNode is a 32 byte struct describing a listpack for a quicklist.
* We use bit fields keep the quicklistNode at 32 bytes.
* count: 16 bits, max 65536 (max zl bytes is 65k, so max count actually < 32k).
* count: 16 bits, max 65536 (max lp bytes is 65k, so max count actually < 32k).
* encoding: 2 bits, RAW=1, LZF=2.
* container: 2 bits, NONE=1, ZIPLIST=2.
* container: 2 bits, PLAIN=1, PACKED=2.
* recompress: 1 bit, bool, true if node is temporary decompressed for usage.
* attempted_compress: 1 bit, boolean, used for verifying during testing.
* extra: 10 bits, free for future use; pads out the remainder of 32 bits */
......@@ -48,9 +48,9 @@ typedef struct quicklistNode {
struct quicklistNode *next;
unsigned char *entry;
size_t sz; /* entry size in bytes */
unsigned int count : 16; /* count of items in ziplist */
unsigned int count : 16; /* count of items in listpack */
unsigned int encoding : 2; /* RAW==1 or LZF==2 */
unsigned int container : 2; /* NONE==1 or ZIPLIST==2 */
unsigned int container : 2; /* PLAIN==1 or PACKED==2 */
unsigned int recompress : 1; /* was this node previous compressed? */
unsigned int attempted_compress : 1; /* node can't compress; too small */
unsigned int extra : 10; /* more bits to steal for future usage */
......@@ -105,7 +105,7 @@ typedef struct quicklistBookmark {
typedef struct quicklist {
quicklistNode *head;
quicklistNode *tail;
unsigned long count; /* total count of all entries in all ziplists */
unsigned long count; /* total count of all entries in all listpacks */
unsigned long len; /* number of quicklistNodes */
int fill : QL_FILL_BITS; /* fill factor for individual nodes */
unsigned int compress : QL_COMP_BITS; /* depth of end nodes not to compress;0=off */
......@@ -117,7 +117,7 @@ typedef struct quicklistIter {
const quicklist *quicklist;
quicklistNode *current;
unsigned char *zi;
long offset; /* offset in current ziplist */
long offset; /* offset in current listpack */
int direction;
} quicklistIter;
......@@ -143,7 +143,7 @@ typedef struct quicklistEntry {
/* quicklist container formats */
#define QUICKLIST_NODE_CONTAINER_PLAIN 1
#define QUICKLIST_NODE_CONTAINER_ZIPLIST 2
#define QUICKLIST_NODE_CONTAINER_PACKED 2
#define QL_NODE_IS_PLAIN(node) ((node)->container == QUICKLIST_NODE_CONTAINER_PLAIN)
......@@ -161,12 +161,8 @@ int quicklistPushHead(quicklist *quicklist, void *value, const size_t sz);
int quicklistPushTail(quicklist *quicklist, void *value, const size_t sz);
void quicklistPush(quicklist *quicklist, void *value, const size_t sz,
int where);
void quicklistAppendZiplist(quicklist *quicklist, unsigned char *zl);
void quicklistAppendListpack(quicklist *quicklist, unsigned char *zl);
void quicklistAppendPlainNode(quicklist *quicklist, unsigned char *data, size_t sz);
quicklist *quicklistAppendValuesFromZiplist(quicklist *quicklist,
unsigned char *zl);
quicklist *quicklistCreateFromZiplist(int fill, int compress,
unsigned char *zl);
void quicklistInsertAfter(quicklist *quicklist, quicklistEntry *entry,
void *value, const size_t sz);
void quicklistInsertBefore(quicklist *quicklist, quicklistEntry *entry,
......
......@@ -1593,6 +1593,45 @@ int ziplistPairsConvertAndValidateIntegrity(unsigned char *zl, size_t size, unsi
return ret;
}
/* callback for ziplistValidateIntegrity.
* The ziplist element pointed by 'p' will be converted and stored into listpack. */
static int _ziplistEntryConvertAndValidate(unsigned char *p, unsigned int head_count, void *userdata) {
UNUSED(head_count);
unsigned char *str;
unsigned int slen;
long long vll;
unsigned char **lp = (unsigned char**)userdata;
if (!ziplistGet(p, &str, &slen, &vll)) return 0;
if (str)
*lp = lpAppend(*lp, (unsigned char*)str, slen);
else
*lp = lpAppendInteger(*lp, vll);
return 1;
}
/* callback for ziplistValidateIntegrity.
* The ziplist element pointed by 'p' will be converted and stored into quicklist. */
static int _listZiplistEntryConvertAndValidate(unsigned char *p, unsigned int head_count, void *userdata) {
UNUSED(head_count);
unsigned char *str;
unsigned int slen;
long long vll;
char longstr[32] = {0};
quicklist *ql = (quicklist*)userdata;
if (!ziplistGet(p, &str, &slen, &vll)) return 0;
if (!str) {
/* Write the longval as a string so we can re-add it */
slen = ll2string(longstr, sizeof(longstr), vll);
str = (unsigned char *)longstr;
}
quicklistPushTail(ql, str, slen);
return 1;
}
/* callback for to check the listpack doesn't have duplicate records */
static int _lpPairsEntryValidation(unsigned char *p, unsigned int head_count, void *userdata) {
struct {
......@@ -1680,7 +1719,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) {
if (len == 0) goto emptykey;
o = createQuicklistObject();
quicklistSetOptions(o->ptr, server.list_max_ziplist_size,
quicklistSetOptions(o->ptr, server.list_max_listpack_size,
server.list_compress_depth);
/* Load every single element of the list */
......@@ -1950,10 +1989,11 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) {
if (len == 0) goto emptykey;
o = createQuicklistObject();
quicklistSetOptions(o->ptr, server.list_max_ziplist_size,
quicklistSetOptions(o->ptr, server.list_max_listpack_size,
server.list_compress_depth);
uint64_t container = QUICKLIST_NODE_CONTAINER_ZIPLIST;
uint64_t container = QUICKLIST_NODE_CONTAINER_PACKED;
while (len--) {
unsigned char *lp;
size_t encoded_len;
if (rdbtype == RDB_TYPE_LIST_QUICKLIST_2) {
......@@ -1962,7 +2002,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) {
return NULL;
}
if (container != QUICKLIST_NODE_CONTAINER_ZIPLIST && container != QUICKLIST_NODE_CONTAINER_PLAIN) {
if (container != QUICKLIST_NODE_CONTAINER_PACKED && container != QUICKLIST_NODE_CONTAINER_PLAIN) {
rdbReportCorruptRDB("Quicklist integrity check failed.");
decrRefCount(o);
return NULL;
......@@ -1979,24 +2019,39 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) {
if (container == QUICKLIST_NODE_CONTAINER_PLAIN) {
quicklistAppendPlainNode(o->ptr, data, encoded_len);
zfree(data);
continue;
}
if (deep_integrity_validation) server.stat_dump_payload_sanitizations++;
if (!ziplistValidateIntegrity(data, encoded_len, deep_integrity_validation, NULL, NULL)) {
rdbReportCorruptRDB("Ziplist integrity check failed.");
decrRefCount(o);
if (rdbtype == RDB_TYPE_LIST_QUICKLIST_2) {
lp = data;
if (deep_integrity_validation) server.stat_dump_payload_sanitizations++;
if (!lpValidateIntegrity(lp, encoded_len, deep_integrity_validation, NULL, NULL)) {
rdbReportCorruptRDB("Listpack integrity check failed.");
decrRefCount(o);
zfree(lp);
return NULL;
}
} else {
lp = lpNew(encoded_len);
if (!ziplistValidateIntegrity(data, encoded_len, 1,
_ziplistEntryConvertAndValidate, &lp))
{
rdbReportCorruptRDB("Ziplist integrity check failed.");
decrRefCount(o);
zfree(data);
zfree(lp);
return NULL;
}
zfree(data);
return NULL;
lp = lpShrinkToFit(lp);
}
/* Silently skip empty ziplists, if we'll end up with empty quicklist we'll fail later. */
if (ziplistLen(data) == 0) {
zfree(data);
if (lpLength(lp) == 0) {
zfree(lp);
continue;
} else {
quicklistAppendZiplist(o->ptr, data);
quicklistAppendListpack(o->ptr, lp);
}
}
......@@ -2080,27 +2135,36 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) {
}
}
break;
case RDB_TYPE_LIST_ZIPLIST:
if (deep_integrity_validation) server.stat_dump_payload_sanitizations++;
if (!ziplistValidateIntegrity(encoded, encoded_len, deep_integrity_validation, NULL, NULL)) {
rdbReportCorruptRDB("List ziplist integrity check failed.");
zfree(encoded);
o->ptr = NULL;
decrRefCount(o);
return NULL;
}
case RDB_TYPE_LIST_ZIPLIST:
{
quicklist *ql = quicklistNew(server.list_max_listpack_size,
server.list_compress_depth);
if (!ziplistValidateIntegrity(encoded, encoded_len, 1,
_listZiplistEntryConvertAndValidate, ql))
{
rdbReportCorruptRDB("List ziplist integrity check failed.");
zfree(encoded);
o->ptr = NULL;
decrRefCount(o);
quicklistRelease(ql);
return NULL;
}
if (ql->len == 0) {
zfree(encoded);
o->ptr = NULL;
decrRefCount(o);
quicklistRelease(ql);
goto emptykey;
}
if (ziplistLen(encoded) == 0) {
zfree(encoded);
o->ptr = NULL;
decrRefCount(o);
goto emptykey;
o->type = OBJ_LIST;
o->ptr = ql;
o->encoding = OBJ_ENCODING_QUICKLIST;
break;
}
o->type = OBJ_LIST;
o->encoding = OBJ_ENCODING_ZIPLIST;
listTypeConvert(o,OBJ_ENCODING_QUICKLIST);
break;
case RDB_TYPE_SET_INTSET:
if (deep_integrity_validation) server.stat_dump_payload_sanitizations++;
if (!intsetValidateIntegrity(encoded, encoded_len, deep_integrity_validation)) {
......@@ -2138,6 +2202,8 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) {
if (zsetLength(o) > server.zset_max_listpack_entries)
zsetConvert(o,OBJ_ENCODING_SKIPLIST);
else
o->ptr = lpShrinkToFit(o->ptr);
break;
}
case RDB_TYPE_ZSET_LISTPACK:
......@@ -2180,9 +2246,10 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) {
goto emptykey;
}
if (hashTypeLength(o) > server.hash_max_listpack_entries) {
if (hashTypeLength(o) > server.hash_max_listpack_entries)
hashTypeConvert(o, OBJ_ENCODING_HT);
}
else
o->ptr = lpShrinkToFit(o->ptr);
break;
}
case RDB_TYPE_HASH_LISTPACK:
......
......@@ -2378,7 +2378,7 @@ dictType commandTableDictType = {
NULL /* allow to expand */
};
/* Hash type hash table (note that small hashes are represented with ziplists) */
/* Hash type hash table (note that small hashes are represented with listpacks) */
dictType hashDictType = {
dictSdsHash, /* hash function */
NULL, /* key dup */
......
......@@ -730,7 +730,7 @@ typedef struct RedisModuleDigest {
#define OBJ_ENCODING_INTSET 6 /* Encoded as intset */
#define OBJ_ENCODING_SKIPLIST 7 /* Encoded as skiplist */
#define OBJ_ENCODING_EMBSTR 8 /* Embedded sds string encoding */
#define OBJ_ENCODING_QUICKLIST 9 /* Encoded as linked list of ziplists */
#define OBJ_ENCODING_QUICKLIST 9 /* Encoded as linked list of listpacks */
#define OBJ_ENCODING_STREAM 10 /* Encoded as a radix tree of listpacks */
#define OBJ_ENCODING_LISTPACK 11 /* Encoded as a listpack */
......@@ -1679,7 +1679,7 @@ struct redisServer {
size_t stream_node_max_bytes;
long long stream_node_max_entries;
/* List parameters */
int list_max_ziplist_size;
int list_max_listpack_size;
int list_compress_depth;
/* time cache */
redisAtomic time_t unixtime; /* Unix time sampled every cron cycle. */
......@@ -2208,7 +2208,6 @@ void listTypeInsert(listTypeEntry *entry, robj *value, int where);
void listTypeReplace(listTypeEntry *entry, robj *value);
int listTypeEqual(listTypeEntry *entry, robj *o);
void listTypeDelete(listTypeIterator *iter, listTypeEntry *entry);
void listTypeConvert(robj *subject, int enc);
robj *listTypeDup(robj *o);
int listTypeDelRange(robj *o, long start, long stop);
void unblockClientWaitingData(client *c);
......@@ -2260,7 +2259,6 @@ robj *createStringObjectFromLongLong(long long value);
robj *createStringObjectFromLongLongForValue(long long value);
robj *createStringObjectFromLongDouble(long double value, int humanfriendly);
robj *createQuicklistObject(void);
robj *createZiplistObject(void);
robj *createSetObject(void);
robj *createIntsetObject(void);
robj *createHashObject(void);
......
......@@ -199,21 +199,6 @@ void listTypeDelete(listTypeIterator *iter, listTypeEntry *entry) {
}
}
/* Create a quicklist from a single ziplist */
void listTypeConvert(robj *subject, int enc) {
serverAssertWithInfo(NULL,subject,subject->type==OBJ_LIST);
serverAssertWithInfo(NULL,subject,subject->encoding==OBJ_ENCODING_ZIPLIST);
if (enc == OBJ_ENCODING_QUICKLIST) {
size_t zlen = server.list_max_ziplist_size;
int depth = server.list_compress_depth;
subject->ptr = quicklistCreateFromZiplist(zlen, depth, subject->ptr);
subject->encoding = enc;
} else {
serverPanic("Unsupported list conversion");
}
}
/* This is a helper function for the COPY command.
* Duplicate a list object, with the guarantee that the returned object
* has the same encoding as the original one.
......@@ -263,7 +248,7 @@ void pushGenericCommand(client *c, int where, int xx) {
}
lobj = createQuicklistObject();
quicklistSetOptions(lobj->ptr, server.list_max_ziplist_size,
quicklistSetOptions(lobj->ptr, server.list_max_listpack_size,
server.list_compress_depth);
dbAdd(c->db,c->argv[1],lobj);
}
......@@ -823,7 +808,7 @@ void lmoveHandlePush(client *c, robj *dstkey, robj *dstobj, robj *value,
/* Create the list if the key does not exist */
if (!dstobj) {
dstobj = createQuicklistObject();
quicklistSetOptions(dstobj->ptr, server.list_max_ziplist_size,
quicklistSetOptions(dstobj->ptr, server.list_max_listpack_size,
server.list_compress_depth);
dbAdd(c->db,dstkey,dstobj);
}
......
......@@ -1377,7 +1377,7 @@ int zsetAdd(robj *zobj, double score, sds ele, int in_flags, int *out_flags, dou
}
}
/* Note that the above block handling ziplist would have either returned or
/* Note that the above block handling listpack would have either returned or
* converted the key to skiplist. */
if (zobj->encoding == OBJ_ENCODING_SKIPLIST) {
zset *zs = zobj->ptr;
......
......@@ -552,7 +552,7 @@ int string2d(const char *s, size_t slen, double *dp) {
* required. The representation should always be parsable by strtod(3).
* This function does not support human-friendly formatting like ld2string
* does. It is intended mainly to be used inside t_zset.c when writing scores
* into a ziplist representing a sorted set. */
* into a listpack representing a sorted set. */
int d2string(char *buf, size_t len, double value) {
if (isnan(value)) {
len = snprintf(buf,len,"nan");
......
......@@ -29,26 +29,6 @@ test {corrupt payload: #7445 - with sanitize} {
}
}
test {corrupt payload: #7445 - without sanitize - 1} {
start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
r config set sanitize-dump-payload no
r restore key 0 $corrupt_payload_7445
catch {r lindex key 2}
assert_equal [count_log_message 0 "crashed by signal"] 0
assert_equal [count_log_message 0 "ASSERTION FAILED"] 1
}
}
test {corrupt payload: #7445 - without sanitize - 2} {
start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
r config set sanitize-dump-payload no
r restore key 0 $corrupt_payload_7445
catch {r lset key 2 "BEEF"}
assert_equal [count_log_message 0 "crashed by signal"] 0
assert_equal [count_log_message 0 "ASSERTION FAILED"] 1
}
}
test {corrupt payload: hash with valid zip list header, invalid entry len} {
start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
catch {
......@@ -82,10 +62,9 @@ test {corrupt payload: valid zipped hash header, dup records} {
test {corrupt payload: quicklist big ziplist prev len} {
start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
r config set sanitize-dump-payload no
r restore key 0 "\x0e\x01\x1b\x1b\x00\x00\x00\x16\x00\x00\x00\x04\x00\x00\x02\x61\x00\x04\x02\x62\x00\x04\x02\x63\x00\x19\x02\x64\x00\xff\x09\x00\xec\x42\xe9\xf5\xd6\x19\x9e\xbd"
catch {r lindex key -2}
assert_equal [count_log_message 0 "crashed by signal"] 0
assert_equal [count_log_message 0 "ASSERTION FAILED"] 1
catch {r restore key 0 "\x0E\x01\x13\x13\x00\x00\x00\x0E\x00\x00\x00\x02\x00\x00\x02\x61\x00\x0E\x02\x62\x00\xFF\x09\x00\x49\x97\x30\xB2\x0D\xA1\xED\xAA"} err
assert_match "*Bad data format*" $err
verify_log_message 0 "*integrity check failed*" 0
}
}
......@@ -103,13 +82,9 @@ test {corrupt payload: quicklist small ziplist prev len} {
test {corrupt payload: quicklist ziplist wrong count} {
start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
r config set sanitize-dump-payload no
r restore key 0 "\x0E\x01\x13\x13\x00\x00\x00\x0E\x00\x00\x00\x03\x00\x00\x02\x61\x00\x04\x02\x62\x00\xFF\x09\x00\x4D\xE2\x0A\x2F\x08\x25\xDF\x91"
# we'll be able to push, but iterating on the list will assert
r lpush key header
r rpush key footer
catch { [r lrange key 0 -1] }
assert_equal [count_log_message 0 "crashed by signal"] 0
assert_equal [count_log_message 0 "ASSERTION FAILED"] 1
catch {r restore key 0 "\x0E\x01\x13\x13\x00\x00\x00\x0E\x00\x00\x00\x03\x00\x00\x02\x61\x00\x04\x02\x62\x00\xFF\x09\x00\x4D\xE2\x0A\x2F\x08\x25\xDF\x91"} err
assert_match "*Bad data format*" $err
verify_log_message 0 "*integrity check failed*" 0
}
}
......@@ -335,10 +310,9 @@ test {corrupt payload: fuzzer findings - NPD in quicklistIndex} {
r debug set-skip-checksum-validation 1
catch {
r RESTORE key 0 "\x0E\x01\x13\x13\x00\x00\x00\x10\x00\x00\x00\x03\x12\x00\xF3\x02\x02\x5F\x31\x04\xF1\xFF\x09\x00\xC9\x4B\x31\xFE\x61\xC0\x96\xFE"
r LSET key 290 290
}
assert_equal [count_log_message 0 "crashed by signal"] 0
assert_equal [count_log_message 0 "ASSERTION FAILED"] 1
} err
assert_match "*Bad data format*" $err
verify_log_message 0 "*integrity check failed*" 0
}
}
......@@ -429,12 +403,9 @@ test {corrupt payload: fuzzer findings - valgrind ziplist prevlen reaches outsid
start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
r config set sanitize-dump-payload no
r debug set-skip-checksum-validation 1
r RESTORE _listbig 0 "\x0E\x02\x1B\x1B\x00\x00\x00\x16\x00\x00\x00\x05\x00\x00\x02\x5F\x39\x04\xF9\x02\x02\x5F\x37\x04\xF7\x02\x02\x5F\x35\xFF\x19\x19\x00\x00\x00\x16\x00\x00\x00\x05\x00\x00\xF5\x02\x02\x5F\x33\x04\xF3\x95\x02\x5F\x31\x04\xF1\xFF\x09\x00\x0C\xFC\x99\x2C\x23\x45\x15\x60"
catch { r RPOP _listbig }
catch { r RPOP _listbig }
catch { r RPUSH _listbig 949682325 }
assert_equal [count_log_message 0 "crashed by signal"] 0
assert_equal [count_log_message 0 "ASSERTION FAILED"] 1
catch {r RESTORE _listbig 0 "\x0E\x02\x1B\x1B\x00\x00\x00\x16\x00\x00\x00\x05\x00\x00\x02\x5F\x39\x04\xF9\x02\x02\x5F\x37\x04\xF7\x02\x02\x5F\x35\xFF\x19\x19\x00\x00\x00\x16\x00\x00\x00\x05\x00\x00\xF5\x02\x02\x5F\x33\x04\xF3\x95\x02\x5F\x31\x04\xF1\xFF\x09\x00\x0C\xFC\x99\x2C\x23\x45\x15\x60"} err
assert_match "*Bad data format*" $err
verify_log_message 0 "*integrity check failed*" 0
}
}
......@@ -451,11 +422,9 @@ test {corrupt payload: fuzzer findings - valgrind ziplist prev too big} {
start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
r config set sanitize-dump-payload no
r debug set-skip-checksum-validation 1
r RESTORE _list 0 "\x0E\x01\x13\x13\x00\x00\x00\x10\x00\x00\x00\x03\x00\x00\xF3\x02\x02\x5F\x31\xC1\xF1\xFF\x09\x00\xC9\x4B\x31\xFE\x61\xC0\x96\xFE"
catch { r RPUSHX _list -45 }
catch { r LREM _list -748 -840}
assert_equal [count_log_message 0 "crashed by signal"] 0
assert_equal [count_log_message 0 "ASSERTION FAILED"] 1
catch {r RESTORE _list 0 "\x0E\x01\x13\x13\x00\x00\x00\x10\x00\x00\x00\x03\x00\x00\xF3\x02\x02\x5F\x31\xC1\xF1\xFF\x09\x00\xC9\x4B\x31\xFE\x61\xC0\x96\xFE"} err
assert_match "*Bad data format*" $err
verify_log_message 0 "*integrity check failed*" 0
}
}
......@@ -778,10 +747,9 @@ test {corrupt payload: fuzzer findings - invalid access in ziplist tail prevlen
start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
r debug set-skip-checksum-validation 1
r config set sanitize-dump-payload no
r restore _listbig 0 "\x12\x02\x02\x1B\x1B\x00\x00\x00\x16\x00\x00\x00\x05\x00\x00\x02\x5F\x39\x04\xF9\x02\x02\x5F\x37\x04\xF7\x02\x02\x5F\x35\xFF\x02\x19\x19\x00\x00\x00\x16\x00\x00\x00\x05\x00\x00\xF5\x02\x02\x5F\x33\x04\xF3\x02\x02\x5F\x31\xFE\xF1\xFF\x0A\x00\x64\x0C\xEB\x03\xDF\x36\x61\xCE"
catch { r RPOPLPUSH _listbig _listbig }
assert_equal [count_log_message 0 "crashed by signal"] 0
assert_equal [count_log_message 0 "ASSERTION FAILED"] 1
catch {r restore _listbig 0 "\x0e\x02\x1B\x1B\x00\x00\x00\x16\x00\x00\x00\x05\x00\x00\x02\x5F\x39\x04\xF9\x02\x02\x5F\x37\x04\xF7\x02\x02\x5F\x35\xFF\x19\x19\x00\x00\x00\x16\x00\x00\x00\x05\x00\x00\xF5\x02\x02\x5F\x33\x04\xF3\x02\x02\x5F\x31\xFE\xF1\xFF\x0A\x00\x6B\x43\x32\x2F\xBB\x29\x0a\xBE"} err
assert_match "*Bad data format*" $err
r ping
}
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment